diff --git a/CHANGELOG.md b/CHANGELOG.md index d1a3aecf2..189e0083b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,9 +3,10 @@ ## Update for 2025-11-08 - **Features** - - allow recursive inline wildcards using curly braces syntax - - simplify SDNQ pre-quantization saved config - - refactor settings and improve handling of attention mechanisms + - **wildcards**: allow recursive inline wildcards using curly braces syntax + - **sdnq**: simplify pre-quantization saved config + - **attention**: refactor settings and improve handling of attention mechanisms + - **lora**: separate fuse setting for native-vs-diffuser implementations - **Fixes** - hires strength save/load in metadata, thanks @awsr - fix imgi2img initial scale tab, thanks @awsr diff --git a/modules/lora/extra_networks_lora.py b/modules/lora/extra_networks_lora.py index 99ab20d15..e6e6a37b2 100644 --- a/modules/lora/extra_networks_lora.py +++ b/modules/lora/extra_networks_lora.py @@ -235,7 +235,7 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): infotext(p) prompt(p) if has_changed and len(include) == 0: # print only once - shared.log.info(f'Network load: type=LoRA apply={[n.name for n in l.loaded_networks]} method={load_method} mode={"fuse" if shared.opts.lora_fuse_diffusers else "backup"} te={te_multipliers} unet={unet_multipliers} time={l.timer.summary}') + shared.log.info(f'Network load: type=LoRA apply={[n.name for n in l.loaded_networks]} method={load_method} mode={"fuse" if shared.opts.lora_fuse_native else "backup"} te={te_multipliers} unet={unet_multipliers} time={l.timer.summary}') def deactivate(self, p): if len(lora_diffusers.diffuser_loaded) > 0: diff --git a/modules/lora/lora_apply.py b/modules/lora/lora_apply.py index f18922040..9ded1a590 100644 --- a/modules/lora/lora_apply.py +++ b/modules/lora/lora_apply.py @@ -20,7 +20,7 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n weights_backup = getattr(self, "network_weights_backup", None) bias_backup = getattr(self, "network_bias_backup", None) if weights_backup is not None or bias_backup is not None: - if (shared.opts.lora_fuse_diffusers and not isinstance(weights_backup, bool)) or (not shared.opts.lora_fuse_diffusers and isinstance(weights_backup, bool)): # invalidate so we can change direct/backup on-the-fly + if (shared.opts.lora_fuse_native and not isinstance(weights_backup, bool)) or (not shared.opts.lora_fuse_native and isinstance(weights_backup, bool)): # invalidate so we can change direct/backup on-the-fly weights_backup = None bias_backup = None self.network_weights_backup = weights_backup @@ -33,15 +33,15 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n if bnb is None: bnb = model_quant.load_bnb('Network load: type=LoRA', silent=True) if bnb is not None: - if shared.opts.lora_fuse_diffusers: + if shared.opts.lora_fuse_native: self.network_weights_backup = True else: self.network_weights_backup = bnb.functional.dequantize_4bit(weight, quant_state=weight.quant_state, quant_type=weight.quant_type, blocksize=weight.blocksize,) self.quant_state, self.quant_type, self.blocksize = weight.quant_state, weight.quant_type, weight.blocksize else: - self.network_weights_backup = weight.clone().to(devices.cpu) if not shared.opts.lora_fuse_diffusers else True + self.network_weights_backup = weight.clone().to(devices.cpu) if not shared.opts.lora_fuse_native else True else: - if shared.opts.lora_fuse_diffusers: + if shared.opts.lora_fuse_native: self.network_weights_backup = True else: self.network_weights_backup = weight.clone().to(devices.cpu) @@ -61,7 +61,7 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n if bias_backup is None: if getattr(self, 'bias', None) is not None: - if shared.opts.lora_fuse_diffusers: + if shared.opts.lora_fuse_native: self.network_bias_backup = True else: bias_backup = self.bias.clone() diff --git a/modules/lora/lora_diffusers.py b/modules/lora/lora_diffusers.py index a97272370..eb1515ca0 100644 --- a/modules/lora/lora_diffusers.py +++ b/modules/lora/lora_diffusers.py @@ -54,7 +54,7 @@ def load_diffusers(name: str, network_on_disk: network.NetworkOnDisk, lora_scale t0 = time.time() name = name.replace(".", "_") sd_model: diffusers.DiffusionPipeline = getattr(shared.sd_model, "pipe", shared.sd_model) - shared.log.debug(f'Network load: type=LoRA name="{name}" file="{network_on_disk.filename}" detected={network_on_disk.sd_version} method=diffusers scale={lora_scale} fuse={shared.opts.lora_fuse_diffusers}') + shared.log.debug(f'Network load: type=LoRA name="{name}" file="{network_on_disk.filename}" detected={network_on_disk.sd_version} method=diffusers scale={lora_scale} fuse={shared.opts.lora_fuse_native}:{shared.opts.lora_fuse_diffusers}') if not hasattr(sd_model, 'load_lora_weights'): shared.log.error(f'Network load: type=LoRA class={sd_model.__class__} does not implement load lora') return None diff --git a/modules/lora/lora_load.py b/modules/lora/lora_load.py index 85c66208d..de3e9bfe0 100644 --- a/modules/lora/lora_load.py +++ b/modules/lora/lora_load.py @@ -128,7 +128,7 @@ def load_safetensors(name, network_on_disk: network.NetworkOnDisk) -> Union[netw if l.debug: shared.log.debug(f'Network load: type=LoRA name="{name}" unmatched={keys_failed_to_match}') else: - shared.log.debug(f'Network load: type=LoRA name="{name}" type={set(network_types)} keys={len(matched_networks)} dtypes={dtypes} fuse={shared.opts.lora_fuse_diffusers}') + shared.log.debug(f'Network load: type=LoRA name="{name}" type={set(network_types)} keys={len(matched_networks)} dtypes={dtypes} fuse={shared.opts.lora_fuse_native}:{shared.opts.lora_fuse_diffusers}') if len(matched_networks) == 0: return None lora_cache[name] = net @@ -303,7 +303,7 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non errors.display(e, 'LoRA') if len(l.loaded_networks) > 0 and l.debug: - shared.log.debug(f'Network load: type=LoRA loaded={[n.name for n in l.loaded_networks]} cache={list(lora_cache)}') + shared.log.debug(f'Network load: type=LoRA loaded={[n.name for n in l.loaded_networks]} cache={list(lora_cache)} fuse={shared.opts.lora_fuse_native}:{shared.opts.lora_fuse_diffusers}') if recompile_model: shared.log.info("Network load: type=LoRA recompiling model") diff --git a/modules/lora/networks.py b/modules/lora/networks.py index f03063e2d..4294615c9 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -49,7 +49,7 @@ def network_activate(include=[], exclude=[]): continue backup_size += network_backup_weights(module, network_layer_name, wanted_names) batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name) - if shared.opts.lora_fuse_diffusers: + if shared.opts.lora_fuse_native: network_apply_direct(module, batch_updown, batch_ex_bias, device=device) else: network_apply_weights(module, batch_updown, batch_ex_bias, device=device) @@ -68,14 +68,14 @@ def network_activate(include=[], exclude=[]): pbar.remove_task(task) # hide progress bar for no action l.timer.activate += time.time() - t0 if l.debug and len(l.loaded_networks) > 0: - shared.log.debug(f'Network load: type=LoRA networks={[n.name for n in l.loaded_networks]} modules={active_components} layers={total} weights={applied_weight} bias={applied_bias} backup={round(backup_size/1024/1024/1024, 2)} fuse={shared.opts.lora_fuse_diffusers} device={device} time={l.timer.summary}') + shared.log.debug(f'Network load: type=LoRA networks={[n.name for n in l.loaded_networks]} modules={active_components} layers={total} weights={applied_weight} bias={applied_bias} backup={round(backup_size/1024/1024/1024, 2)} fuse={shared.opts.lora_fuse_native}:{shared.opts.lora_fuse_diffusers} device={device} time={l.timer.summary}') modules.clear() if len(applied_layers) > 0 or shared.opts.diffusers_offload_mode == "sequential": sd_models.set_diffuser_offload(sd_model, op="model") def network_deactivate(include=[], exclude=[]): - if not shared.opts.lora_fuse_diffusers or shared.opts.lora_force_diffusers: + if not shared.opts.lora_fuse_native or shared.opts.lora_force_diffusers: return if len(l.previously_loaded_networks) == 0: return @@ -112,7 +112,7 @@ def network_deactivate(include=[], exclude=[]): pbar.update(task, advance=1) continue batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name, use_previous=True) - if shared.opts.lora_fuse_diffusers: + if shared.opts.lora_fuse_native: network_apply_direct(module, batch_updown, batch_ex_bias, device=device, deactivate=True) else: network_apply_weights(module, batch_updown, batch_ex_bias, device=device, deactivate=True) @@ -125,7 +125,7 @@ def network_deactivate(include=[], exclude=[]): l.timer.deactivate = time.time() - t0 if l.debug and len(l.previously_loaded_networks) > 0: - shared.log.debug(f'Network deactivate: type=LoRA networks={[n.name for n in l.previously_loaded_networks]} modules={active_components} layers={total} apply={len(applied_layers)} fuse={shared.opts.lora_fuse_diffusers} time={l.timer.summary}') + shared.log.debug(f'Network deactivate: type=LoRA networks={[n.name for n in l.previously_loaded_networks]} modules={active_components} layers={total} apply={len(applied_layers)} fuse={shared.opts.lora_fuse_native}:{shared.opts.lora_fuse_diffusers} time={l.timer.summary}') modules.clear() if len(applied_layers) > 0 or shared.opts.diffusers_offload_mode == "sequential": sd_models.set_diffuser_offload(sd_model, op="model") diff --git a/modules/shared.py b/modules/shared.py index 66165e385..8af5135e2 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -711,14 +711,15 @@ options_templates.update(options_section(('extra_networks', "Networks"), { "extra_networks_lora_sep": OptionInfo("