diff --git a/CHANGELOG.md b/CHANGELOG.md index 18d277303..e008d98fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,12 +26,24 @@ And it also includes fixes for all reported issues so far example: `python cli/sdapi.py /sdapi/v1/sd-models` - memory: add ram usage monitoring in addition to gpu memory usage monitoring - updated core requirements +- **Compile** + - new option: **fused projections** + pretty much free 5% performance boost for compatible models + enable in settings -> compute settings + - new option: **dynamic quantization** (experimental) + reduces memory usage and increases performance + enable in settings -> compute settings + best used together with torch compile: *inductor* + this feature is highly experimental and will evolve over time + requires nightly versions of `torch` and `torchao` + > pip install -U --pre torch torchvision torchaudio --index-url + > pip install git+ - **IPEX**, thanks @disty0 - - rewrote IPEX hijacks to get rid of CondFunc - - dropped IPEX 2.0 specific fixes, update to IPEX 2.1 - - add `IPEX_SDPA_SLICE_TRIGGER_RATE` and `IPEX_ATTENTION_SLICE_RATE` env variables + - update to IPEX 2.1 + - add `IPEX_SDPA_SLICE_TRIGGER_RATE` and `IPEX_ATTENTION_SLICE_RATE` env variables - **Fixes** - ipadapter: allow changing of model/image on-the-fly + - ipadapter: fix fallback of cross-attention on unload - python: fix python 3.9 compatibility - img2img: clip and blip interrogate - sampler: guard against invalid sampler index diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index bd23a0ecb..9b6a2a0ba 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit bd23a0ecb63f3b4108e5518f24b0226a9fd5b5ac +Subproject commit 9b6a2a0ba8cecf06ffde471f93fa40595f3398da diff --git a/modules/control/units/ipadapter.py b/modules/control/units/ipadapter.py index ed12ca63c..7ea1cc814 100644 --- a/modules/control/units/ipadapter.py +++ b/modules/control/units/ipadapter.py @@ -42,7 +42,6 @@ def apply_ip_adapter(pipe, p: processing.StableDiffusionProcessing, adapter, sca pipe.set_ip_adapter_scale(0) if loaded is not None: shared.log.debug('IP adapter: unload attention processor') - pipe.unet.set_default_attn_processor() pipe.unet.config.encoder_hid_dim_type = None loaded = None return False @@ -74,7 +73,6 @@ def apply_ip_adapter(pipe, p: processing.StableDiffusionProcessing, adapter, sca t0 = time.time() if loaded is not None: # shared.log.debug('IP adapter: reset attention processor') - pipe.unet.set_default_attn_processor() loaded = None else: shared.log.debug('IP adapter: load attention processor') diff --git a/modules/sd_models.py b/modules/sd_models.py index 0e1669233..5fb822bac 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -760,7 +760,9 @@ def set_diffuser_options(sd_model, vae = None, op: str = 'model'): sd_model.vqvae.to(torch.float32) # vqvae is producing nans in fp16 if shared.opts.cross_attention_optimization == "xFormers" and hasattr(sd_model, 'enable_xformers_memory_efficient_attention'): sd_model.enable_xformers_memory_efficient_attention() - + if shared.opts.diffusers_fuse_projections and hasattr(sd_model, 'fuse_qkv_projections'): + shared.log.debug(f'Setting {op}: enable fused projections') + sd_model.fuse_qkv_projections() if shared.opts.diffusers_eval: if hasattr(sd_model, "unet") and hasattr(sd_model.unet, "requires_grad_"): sd_model.unet.requires_grad_(False) @@ -771,6 +773,8 @@ def set_diffuser_options(sd_model, vae = None, op: str = 'model'): if hasattr(sd_model, "text_encoder") and hasattr(sd_model.text_encoder, "requires_grad_"): sd_model.text_encoder.requires_grad_(False) sd_model.text_encoder.eval() + if shared.opts.diffusers_quantization: + sd_model = sd_models_compile.dynamic_quantization(sd_model) if shared.opts.opt_channelslast and hasattr(sd_model, 'unet'): shared.log.debug(f'Setting {op}: enable channels last') diff --git a/modules/sd_models_compile.py b/modules/sd_models_compile.py index a909a3763..99e863b0a 100644 --- a/modules/sd_models_compile.py +++ b/modules/sd_models_compile.py @@ -130,6 +130,17 @@ def compile_torch(sd_model): torch._logging.set_logs(dynamo=log_level, aot=log_level, inductor=log_level) # pylint: disable=protected-access torch._dynamo.config.verbose = shared.opts.cuda_compile_verbose # pylint: disable=protected-access torch._dynamo.config.suppress_errors = shared.opts.cuda_compile_errors # pylint: disable=protected-access + + try: + torch._inductor.config.conv_1x1_as_mm = True # pylint: disable=protected-access + torch._inductor.config.coordinate_descent_tuning = True # pylint: disable=protected-access + torch._inductor.config.epilogue_fusion = False # pylint: disable=protected-access + torch._inductor.config.coordinate_descent_check_all_directions = True # pylint: disable=protected-access + torch._inductor.config.use_mixed_mm = True # pylint: disable=protected-access + # torch._inductor.config.force_fuse_int_mm_with_mul = True # pylint: disable=protected-access + except Exception as e: + shared.log.error(f"Torch inductor config error: {e}") + t0 = time.time() if shared.opts.cuda_compile: if shared.opts.cuda_compile and (not hasattr(sd_model, 'unet') or not hasattr(sd_model.unet, 'config')): @@ -169,3 +180,28 @@ def compile_diffusers(sd_model): else: sd_model = compile_torch(sd_model) return sd_model + + +def dynamic_quantization(sd_model): + try: + from torchao.quantization import quant_api + except Exception as e: + shared.log.error(f"Model dynamic quantization not supported: {e}") + return sd_model + + def dynamic_quant_filter_fn(mod, *args): # pylint: disable=unused-argument + return (isinstance(mod, torch.nn.Linear) and mod.in_features > 16 and (mod.in_features, mod.out_features) + not in [(1280, 640), (1920, 1280), (1920, 640), (2048, 1280), (2048, 2560), (2560, 1280), (256, 128), (2816, 1280), (320, 640), (512, 1536), (512, 256), (512, 512), (640, 1280), (640, 1920), (640, 320), (640, 5120), (640, 640), (960, 320), (960, 640)]) + + def conv_filter_fn(mod, *args): # pylint: disable=unused-argument + return (isinstance(mod, torch.nn.Conv2d) and mod.kernel_size == (1, 1) and 128 in [mod.in_channels, mod.out_channels]) + + shared.log.info(f"Model dynamic quantization: pipeline={sd_model.__class__.__name__}") + try: + quant_api.swap_conv2d_1x1_to_linear(sd_model.unet, conv_filter_fn) + quant_api.swap_conv2d_1x1_to_linear(sd_model.vae, conv_filter_fn) + quant_api.apply_dynamic_quant(sd_model.unet, dynamic_quant_filter_fn) + quant_api.apply_dynamic_quant(sd_model.vae, dynamic_quant_filter_fn) + except Exception as e: + shared.log.error(f"Model dynamic quantization error: {e}") + return sd_model diff --git a/modules/shared.py b/modules/shared.py index fefd0554c..c3d5b3120 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -317,6 +317,7 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "other_sep": OptionInfo("

Execution precision

", "", gr.HTML), "opt_channelslast": OptionInfo(False, "Use channels last as torch memory format "), "cudnn_benchmark": OptionInfo(False, "Enable full-depth cuDNN benchmark feature"), + "diffusers_fuse_projections": OptionInfo(False, "Enable fused projections"), "torch_gc_threshold": OptionInfo(80, "Memory usage threshold before running Torch GC", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1}), "cuda_compile_sep": OptionInfo("

Model Compile

", "", gr.HTML), @@ -329,6 +330,7 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "cuda_compile_precompile": OptionInfo(False, "Model compile precompile"), "cuda_compile_verbose": OptionInfo(False, "Model compile verbose mode"), "cuda_compile_errors": OptionInfo(True, "Model compile suppress errors"), + "diffusers_quantization": OptionInfo(False, "Enable dynamic quantization"), "ipex_sep": OptionInfo("

IPEX, DirectML and OpenVINO

", "", gr.HTML), "ipex_optimize": OptionInfo(False if not devices.backend == "ipex" else True, "Enable IPEX Optimize for Intel GPUs"), diff --git a/scripts/animatediff.py b/scripts/animatediff.py index 074d31d6b..b08272370 100644 --- a/scripts/animatediff.py +++ b/scripts/animatediff.py @@ -72,7 +72,7 @@ def set_adapter(adapter_name: str = 'None'): if getattr(shared.sd_model, 'image_encoder', None) is not None: shared.log.debug('AnimateDiff: unloading IP adapter') # shared.sd_model.image_encoder = None - shared.sd_model.unet.set_default_attn_processor() + # shared.sd_model.unet.set_default_attn_processor() shared.sd_model.unet.config.encoder_hid_dim_type = None if adapter_name.endswith('.ckpt') or adapter_name.endswith('.safetensors'): import huggingface_hub as hf diff --git a/scripts/ipadapter.py b/scripts/ipadapter.py index 2360da901..edb326667 100644 --- a/scripts/ipadapter.py +++ b/scripts/ipadapter.py @@ -74,7 +74,6 @@ class Script(scripts.Script): shared.sd_model.set_ip_adapter_scale(0) if loaded is not None: shared.log.debug('IP adapter: unload attention processor') - shared.sd_model.unet.set_default_attn_processor() shared.sd_model.unet.config.encoder_hid_dim_type = None loaded = None return @@ -104,7 +103,6 @@ class Script(scripts.Script): t0 = time.time() if loaded is not None: shared.log.debug('IP adapter: reset attention processor') - shared.sd_model.unet.set_default_attn_processor() loaded = None else: shared.log.debug('IP adapter: load attention processor')