diff --git a/extensions-builtin/Lora/network_lora.py b/extensions-builtin/Lora/network_lora.py index 5e6eaef6c..8ebda2e22 100644 --- a/extensions-builtin/Lora/network_lora.py +++ b/extensions-builtin/Lora/network_lora.py @@ -22,7 +22,6 @@ class NetworkModuleLora(network.NetworkModule): self.dim = weights.w["lora_down.weight"].shape[0] def create_module(self, weights, key, none_ok=False): - from modules.shared import opts weight = weights.get(key) if weight is None and none_ok: return None @@ -49,8 +48,6 @@ class NetworkModuleLora(network.NetworkModule): if weight.shape != module.weight.shape: weight = weight.reshape(module.weight.shape) module.weight.copy_(weight) - if opts.lora_load_gpu: - module = module.to(device=devices.device, dtype=devices.dtype) module.weight.requires_grad_(False) return module diff --git a/mkdocs.yml b/mkdocs.yml deleted file mode 100644 index 7fdeb24db..000000000 --- a/mkdocs.yml +++ /dev/null @@ -1,49 +0,0 @@ -site_name: SD.Next Documentation -site_url: https://vladmandic.github.io/automatic -repo_url: https://github.com/vladmandic/automatic -repo_name: vladmandic/automatic -docs_dir: wiki - -theme: - name: material - - features: - - navigation.footer - - navigation.instant - - navigation.instant.prefetch - - navigation.instant.progress - - navigation.tracking - - palette: - - media: "(prefers-color-scheme: light)" - scheme: default - toggle: - icon: material/weather-night - name: Switch to dark mode - primary: teal - accent: pink - - - media: "(prefers-color-scheme: dark)" - scheme: slate - toggle: - icon: material/weather-sunny - name: Switch to light mode - primary: green - accent: yellow - - logo: assets/favicon.svg - favicon: assets/favicon.svg - -extra: - social: - - icon: fontawesome/brands/discord - link: https://discord.gg/VjvR2tabEX - name: Discord - - icon: fontawesome/brands/youtube - link: https://www.youtube.com/@SDNext - name: Youtube - -markdown_extensions: - - admonition - - pymdownx.details - - pymdownx.superfences \ No newline at end of file diff --git a/modules/extra_networks.py b/modules/extra_networks.py index fca48e21c..e96d2e5b7 100644 --- a/modules/extra_networks.py +++ b/modules/extra_networks.py @@ -74,9 +74,12 @@ def is_stepwise(en_obj): return any([len(str(x).split("@")) > 1 for x in all_args]) # noqa C419 # pylint: disable=use-a-generator -def activate(p, extra_network_data, step=0): +def activate(p, extra_network_data=None, step=0): """call activate for extra networks in extra_network_data in specified order, then call activate for all remaining registered networks with an empty argument list""" - if extra_network_data is None: + if p.disable_extra_networks: + return + extra_network_data = extra_network_data or p.network_data + if extra_network_data is None or len(extra_network_data) == 0: return stepwise = False for extra_network_args in extra_network_data.values(): @@ -106,15 +109,18 @@ def activate(p, extra_network_data, step=0): except Exception as e: errors.display(e, f"Activating network: type={extra_network_name}") - p.extra_network_data = extra_network_data + p.network_data = extra_network_data if stepwise: p.stepwise_lora = True shared.opts.data['lora_functional'] = functional -def deactivate(p, extra_network_data): +def deactivate(p, extra_network_data=None): """call deactivate for extra networks in extra_network_data in specified order, then call deactivate for all remaining registered networks""" - if extra_network_data is None: + if p.disable_extra_networks: + return + extra_network_data = extra_network_data or p.network_data + if extra_network_data is None or len(extra_network_data) == 0: return for extra_network_name in extra_network_data: extra_network = extra_network_registry.get(extra_network_name, None) diff --git a/modules/face/faceid.py b/modules/face/faceid.py index b74e15dc5..4a4f07531 100644 --- a/modules/face/faceid.py +++ b/modules/face/faceid.py @@ -204,7 +204,6 @@ def face_id( ip_model_dict["face_image"] = face_images ip_model_dict["faceid_embeds"] = face_embeds # overwrite placeholder faceid_model.set_scale(scale) - extra_network_data = None if p.all_prompts is None or len(p.all_prompts) == 0: processing.process_init(p) @@ -215,11 +214,9 @@ def face_id( p.negative_prompts = p.all_negative_prompts[n * p.batch_size:(n+1) * p.batch_size] p.seeds = p.all_seeds[n * p.batch_size:(n+1) * p.batch_size] p.subseeds = p.all_subseeds[n * p.batch_size:(n+1) * p.batch_size] - p.prompts, extra_network_data = extra_networks.parse_prompts(p.prompts) + p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts) - if not p.disable_extra_networks: - with devices.autocast(): - extra_networks.activate(p, extra_network_data) + extra_networks.activate(p, p.network_data) ip_model_dict.update({ "prompt": p.prompts[0], "negative_prompt": p.negative_prompts[0], @@ -239,8 +236,7 @@ def face_id( devices.torch_gc() ipadapter.unapply(p.sd_model) - if not p.disable_extra_networks: - extra_networks.deactivate(p, extra_network_data) + extra_networks.deactivate(p, p.network_data) p.extra_generation_params["IP Adapter"] = f"{basename}:{scale}" finally: diff --git a/modules/lora/extra_networks_lora.py b/modules/lora/extra_networks_lora.py index 57966550a..4ce7a94a9 100644 --- a/modules/lora/extra_networks_lora.py +++ b/modules/lora/extra_networks_lora.py @@ -1,5 +1,4 @@ import re -import time import numpy as np import modules.lora.networks as networks from modules import extra_networks, shared @@ -128,10 +127,9 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): if len(networks.loaded_networks) > 0 and step == 0: infotext(p) prompt(p) - shared.log.info(f'Load network: type=LoRA apply={[n.name for n in networks.loaded_networks]} te={te_multipliers} unet={unet_multipliers} time={networks.get_timers()}') + shared.log.info(f'Load network: type=LoRA apply={[n.name for n in networks.loaded_networks]} te={te_multipliers} unet={unet_multipliers} time={networks.timer.summary}') def deactivate(self, p): - t0 = time.time() if shared.native and len(networks.diffuser_loaded) > 0: if hasattr(shared.sd_model, "unload_lora_weights") and hasattr(shared.sd_model, "text_encoder"): if not (shared.compiled_model_state is not None and shared.compiled_model_state.is_compiled is True): @@ -142,10 +140,8 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): except Exception: pass networks.network_deactivate() - t1 = time.time() - networks.timer['restore'] += t1 - t0 if self.active and networks.debug: - shared.log.debug(f"Network end: type=LoRA load={networks.timer['load']:.2f} apply={networks.timer['apply']:.2f} restore={networks.timer['restore']:.2f}") + shared.log.debug(f"Network end: type=LoRA time={networks.timer.summary}") if self.errors: for k, v in self.errors.items(): shared.log.error(f'LoRA: name="{k}" errors={v}') diff --git a/modules/lora/lora_timers.py b/modules/lora/lora_timers.py new file mode 100644 index 000000000..30c35a728 --- /dev/null +++ b/modules/lora/lora_timers.py @@ -0,0 +1,38 @@ +class Timer(): + list: float = 0 + load: float = 0 + backup: float = 0 + calc: float = 0 + apply: float = 0 + move: float = 0 + restore: float = 0 + activate: float = 0 + deactivate: float = 0 + + @property + def total(self): + return round(self.activate + self.deactivate, 2) + + @property + def summary(self): + t = {} + for k, v in self.__dict__.items(): + if v > 0.1: + t[k] = round(v, 2) + return t + + def clear(self, complete: bool = False): + self.backup = 0 + self.calc = 0 + self.apply = 0 + self.move = 0 + self.restore = 0 + if complete: + self.activate = 0 + self.deactivate = 0 + + def add(self, name, t): + self.__dict__[name] += t + + def __str__(self): + return f'{self.__class__.__name__}({self.summary})' diff --git a/modules/lora/network_lora.py b/modules/lora/network_lora.py index 6c1d7ea3f..8bf475ebc 100644 --- a/modules/lora/network_lora.py +++ b/modules/lora/network_lora.py @@ -22,7 +22,6 @@ class NetworkModuleLora(network.NetworkModule): self.dim = weights.w["lora_down.weight"].shape[0] def create_module(self, weights, key, none_ok=False): - from modules.shared import opts weight = weights.get(key) if weight is None and none_ok: return None @@ -49,8 +48,6 @@ class NetworkModuleLora(network.NetworkModule): if weight.shape != module.weight.shape: weight = weight.reshape(module.weight.shape) module.weight.copy_(weight) - if opts.lora_load_gpu: - module = module.to(device=devices.device, dtype=devices.dtype) module.weight.requires_grad_(False) return module diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 9618f01a9..805b24b52 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -1,24 +1,15 @@ from typing import Union, List +from contextlib import nullcontext import os import re import time import concurrent -from contextlib import nullcontext import torch import diffusers.models.lora import rich.progress as rp -import modules.lora.network as network -import modules.lora.network_lora as network_lora -import modules.lora.network_hada as network_hada -import modules.lora.network_ia3 as network_ia3 -import modules.lora.network_oft as network_oft -import modules.lora.network_lokr as network_lokr -import modules.lora.network_full as network_full -import modules.lora.network_norm as network_norm -import modules.lora.network_glora as network_glora -import modules.lora.network_overrides as network_overrides -import modules.lora.lora_convert as lora_convert +from modules.lora import lora_timers, network, lora_convert, network_overrides +from modules.lora import network_lora, network_hada, network_ia3, network_oft, network_lokr, network_full, network_norm, network_glora from modules.lora.extra_networks_lora import ExtraNetworkLora from modules import shared, devices, sd_models, sd_models_compile, errors, files_cache, model_quant @@ -28,7 +19,6 @@ extra_network_lora = ExtraNetworkLora() available_networks = {} available_network_aliases = {} loaded_networks: List[network.Network] = [] -timer = { 'list': 0, 'load': 0, 'backup': 0, 'calc': 0, 'apply': 0, 'move': 0, 'restore': 0, 'deactivate': 0 } bnb = None lora_cache = {} diffuser_loaded = [] @@ -36,6 +26,7 @@ diffuser_scales = [] available_network_hash_lookup = {} forbidden_network_aliases = {} re_network_name = re.compile(r"(.*)\s*\([0-9a-fA-F]+\)") +timer = lora_timers.Timer() module_types = [ network_lora.ModuleTypeLora(), network_hada.ModuleTypeHada(), @@ -47,19 +38,6 @@ module_types = [ network_glora.ModuleTypeGLora(), ] - -def total_time(): - return sum(timer.values()) - - -def get_timers(): - t = { 'total': round(sum(timer.values()), 2) } - for k, v in timer.items(): - if v > 0.1: - t[k] = round(v, 2) - return t - - # section: load networks from disk def load_diffusers(name, network_on_disk, lora_scale=shared.opts.extra_networks_default_multiplier) -> Union[network.Network, None]: @@ -154,7 +132,7 @@ def load_safetensors(name, network_on_disk) -> Union[network.Network, None]: if debug: shared.log.debug(f'LoRA name="{name}" unmatched={keys_failed_to_match}') else: - shared.log.debug(f'LoRA name="{name}" type={set(network_types)} keys={len(matched_networks)}') + shared.log.debug(f'LoRA name="{name}" type={set(network_types)} keys={len(matched_networks)} direct={shared.opts.lora_fuse_diffusers}') if len(matched_networks) == 0: return None lora_cache[name] = net @@ -222,12 +200,11 @@ def list_available_networks(): for fn in candidates: executor.submit(add_network, fn) t1 = time.time() - timer['list'] = t1 - t0 + timer.list = t1 - t0 shared.log.info(f'Available LoRAs: path="{shared.cmd_opts.lora_dir}" items={len(available_networks)} folders={len(forbidden_network_aliases)} time={t1 - t0:.2f}') def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None): - timer['list'] = 0 networks_on_disk: list[network.NetworkOnDisk] = [available_network_aliases.get(name, None) for name in names] if any(x is None for x in networks_on_disk): list_available_networks() @@ -301,13 +278,12 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non if len(loaded_networks) > 0: devices.torch_gc() - t1 = time.time() - timer['load'] = t1 - t0 + timer.load = time.time() - t0 # section: process loaded networks -def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], weight: torch.Tensor, network_layer_name: str, wanted_names: tuple): +def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], network_layer_name: str, wanted_names: tuple): global bnb # pylint: disable=W0603 backup_size = 0 if len(loaded_networks) > 0 and network_layer_name is not None and any([net.modules.get(network_layer_name, None) for net in loaded_networks]): # noqa: C419 # pylint: disable=R1729 @@ -315,9 +291,10 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n weights_backup = getattr(self, "network_weights_backup", None) if weights_backup is None and wanted_names != (): # pylint: disable=C1803 + weight = getattr(self, 'weight', None) self.network_weights_backup = None - if shared.opts.lora_fuse_diffusers or shared.opts.lora_low_memory: - weights_backup = True + if shared.opts.lora_fuse_diffusers: + self.network_weights_backup = True elif getattr(weight, "quant_type", None) in ['nf4', 'fp4']: if bnb is None: bnb = model_quant.load_bnb('Load network: type=LoRA', silent=True) @@ -329,86 +306,112 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n self.blocksize = weight.blocksize else: weights_backup = weight.clone() + weights_backup = weights_backup.to(devices.cpu) else: weights_backup = weight.clone() - if shared.opts.lora_offload_backup and weights_backup is not None and isinstance(weights_backup, torch.Tensor): weights_backup = weights_backup.to(devices.cpu) - self.network_weights_backup = weights_backup bias_backup = getattr(self, "network_bias_backup", None) if bias_backup is None: if getattr(self, 'bias', None) is not None: - if shared.opts.lora_fuse_diffusers or shared.opts.lora_low_memory: - bias_backup = True + if shared.opts.lora_fuse_diffusers: + self.network_bias_backup = True else: bias_backup = self.bias.clone() - else: - bias_backup = None - if shared.opts.lora_offload_backup and bias_backup is not None and isinstance(bias_backup, torch.Tensor): - bias_backup = bias_backup.to(devices.cpu) - self.network_bias_backup = bias_backup + bias_backup = bias_backup.to(devices.cpu) if getattr(self, 'network_weights_backup', None) is not None: backup_size += self.network_weights_backup.numel() * self.network_weights_backup.element_size() if isinstance(self.network_weights_backup, torch.Tensor) else 0 if getattr(self, 'network_bias_backup', None) is not None: backup_size += self.network_bias_backup.numel() * self.network_bias_backup.element_size() if isinstance(self.network_bias_backup, torch.Tensor) else 0 - t1 = time.time() - timer['backup'] += t1 - t0 + timer.backup += time.time() - t0 return backup_size -def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], weight: torch.Tensor, network_layer_name: str): +def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], network_layer_name: str): if shared.opts.diffusers_offload_mode == "none": self.to(devices.device) batch_updown = None batch_ex_bias = None for net in loaded_networks: module = net.modules.get(network_layer_name, None) - if module is not None and hasattr(self, 'weight'): - try: - t0 = time.time() - updown, ex_bias = module.calc_updown(weight) - t1 = time.time() - if batch_updown is not None and updown is not None: - batch_updown += updown.to(batch_updown.device) - else: - batch_updown = updown - if batch_ex_bias is not None and ex_bias is not None: - batch_ex_bias += ex_bias.to(batch_ex_bias.device) - else: - batch_ex_bias = ex_bias - timer['calc'] += t1 - t0 - if shared.opts.diffusers_offload_mode != "none": - t0 = time.time() - if batch_updown is not None: - batch_updown = batch_updown.to(devices.cpu) - if batch_ex_bias is not None: - batch_ex_bias = batch_ex_bias.to(devices.cpu) - t1 = time.time() - timer['move'] += t1 - t0 - except RuntimeError as e: - extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 - if debug: - module_name = net.modules.get(network_layer_name, None) - shared.log.error(f'LoRA apply weight name="{net.name}" module="{module_name}" layer="{network_layer_name}" {e}') - errors.display(e, 'LoRA') - raise RuntimeError('LoRA apply weight') from e + if module is None: continue + try: + t0 = time.time() + weight = self.weight.to(devices.device) + updown, ex_bias = module.calc_updown(weight) + if batch_updown is not None and updown is not None: + batch_updown += updown.to(batch_updown.device) + else: + batch_updown = updown + if batch_ex_bias is not None and ex_bias is not None: + batch_ex_bias += ex_bias.to(batch_ex_bias.device) + else: + batch_ex_bias = ex_bias + timer.calc += time.time() - t0 + if shared.opts.diffusers_offload_mode == "sequential": + t0 = time.time() + if batch_updown is not None: + batch_updown = batch_updown.to(devices.cpu) + if batch_ex_bias is not None: + batch_ex_bias = batch_ex_bias.to(devices.cpu) + t1 = time.time() + timer.move += t1 - t0 + except RuntimeError as e: + extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 + if debug: + module_name = net.modules.get(network_layer_name, None) + shared.log.error(f'LoRA apply weight name="{net.name}" module="{module_name}" layer="{network_layer_name}" {e}') + errors.display(e, 'LoRA') + raise RuntimeError('LoRA apply weight') from e + continue return batch_updown, batch_ex_bias -def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], updown: torch.Tensor, ex_bias: torch.Tensor, orig_device: torch.device, deactivate: bool = False): +def network_apply_direct(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], updown: torch.Tensor, ex_bias: torch.Tensor, deactivate: bool = False): + weights_backup = getattr(self, "network_weights_backup", False) + bias_backup = getattr(self, "network_bias_backup", False) + if not weights_backup and not bias_backup: + return None, None t0 = time.time() + + if weights_backup: + if updown is not None and len(self.weight.shape) == 4 and self.weight.shape[1] == 9: # inpainting model. zero pad updown to make channel[1] 4 to 9 + updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) # pylint: disable=not-callable + if updown is not None: + if deactivate: + updown *= -1 + new_weight = self.weight.to(devices.device) + updown.to(devices.device) + if getattr(self, "quant_type", None) in ['nf4', 'fp4'] and bnb is not None: + self.weight = bnb.nn.Params4bit(new_weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) + else: + self.weight = torch.nn.Parameter(new_weight, requires_grad=False) + del new_weight + if hasattr(self, "qweight") and hasattr(self, "freeze"): + self.freeze() + + if bias_backup: + if ex_bias is not None: + if deactivate: + ex_bias *= -1 + new_weight = bias_backup.to(devices.device) + ex_bias.to(devices.device) + self.bias = torch.nn.Parameter(new_weight, requires_grad=False) + del new_weight + + timer.apply += time.time() - t0 + return self.weight.device, self.weight.dtype + + +def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], updown: torch.Tensor, ex_bias: torch.Tensor, orig_device: torch.device, deactivate: bool = False): weights_backup = getattr(self, "network_weights_backup", None) bias_backup = getattr(self, "network_bias_backup", None) if weights_backup is None and bias_backup is None: return None, None + t0 = time.time() if weights_backup is not None: - if isinstance(weights_backup, bool): - weights_backup = self.weight - else: - self.weight = None + self.weight = None if updown is not None and len(weights_backup.shape) == 4 and weights_backup.shape[1] == 9: # inpainting model. zero pad updown to make channel[1] 4 to 9 updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) # pylint: disable=not-callable if updown is not None: @@ -426,10 +429,7 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn self.freeze() if bias_backup is not None: - if isinstance(bias_backup, bool): - bias_backup = self.bias - else: - self.bias = None + self.bias = None if ex_bias is not None: if deactivate: ex_bias *= -1 @@ -438,19 +438,16 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn del new_weight else: self.bias = torch.nn.Parameter(bias_backup.to(device=orig_device), requires_grad=False) - else: - self.bias = None - t1 = time.time() - timer['apply'] += t1 - t0 + timer.apply += time.time() - t0 return self.weight.device, self.weight.dtype def network_deactivate(): - if not shared.opts.lora_low_memory: + if not shared.opts.lora_fuse_diffusers: return - timer['deactivate'] = 0 t0 = time.time() + timer.clear() sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility if shared.opts.diffusers_offload_mode == "sequential": sd_models.disable_offload(sd_model) @@ -462,15 +459,13 @@ def network_deactivate(): modules[component_name] = list(component.named_modules()) total = sum(len(x) for x in modules.values()) if len(loaded_networks) > 0: - pbar = rp.Progress(rp.TextColumn('[cyan]Deactivate network: type=LoRA'), rp.BarColumn(), rp.TaskProgressColumn(), - rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), - console=shared.console) + pbar = rp.Progress(rp.TextColumn('[cyan]Network: type=LoRA action=deactivate'), rp.BarColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), console=shared.console) task = pbar.add_task(description='', total=total) else: task = None pbar = nullcontext() with devices.inference_context(), pbar: - applied = 0 + applied_layers = [] weights_devices = [] weights_dtypes = [] for component in modules.keys(): @@ -479,36 +474,33 @@ def network_deactivate(): network_layer_name = getattr(module, 'network_layer_name', None) if shared.state.interrupted or network_layer_name is None: if task is not None: - pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} skip') + pbar.update(task, advance=1) continue - weight = getattr(module, 'weight', None) - weight = weight.to(devices.device) if weight is not None else None - batch_updown, batch_ex_bias = network_calc_weights(module, weight, network_layer_name) - weights_device, weights_dtype = network_apply_weights(module, batch_updown, batch_ex_bias, orig_device, deactivate=True) + batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name) + if shared.opts.lora_fuse_diffusers: + weights_device, weights_dtype = network_apply_direct(module, batch_updown, batch_ex_bias, deactivate=True) + else: + weights_device, weights_dtype = network_apply_weights(module, batch_updown, batch_ex_bias, orig_device, deactivate=True) weights_devices.append(weights_device) weights_dtypes.append(weights_dtype) if batch_updown is not None or batch_ex_bias is not None: - applied += 1 - del weight, batch_updown, batch_ex_bias + applied_layers.append(network_layer_name) + del batch_updown, batch_ex_bias module.network_current_names = () if task is not None: - pbar.update(task, advance=1, - description=f'networks={len(loaded_networks)} modules={len(modules)} deactivate={applied}') + pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} modules={len(modules)} deactivate={len(applied_layers)}') weights_devices, weights_dtypes = list(set([x for x in weights_devices if x is not None])), list(set([x for x in weights_dtypes if x is not None])) # noqa: C403 # pylint: disable=R1718 + timer.deactivate = time.time() - t0 if debug and len(loaded_networks) > 0: - shared.log.debug( - f'Deactivate network: type=LoRA networks={len(loaded_networks)} modules={total} deactivate={applied} device={weights_devices} dtype={weights_dtypes} fuse={shared.opts.lora_fuse_diffusers} time={get_timers()}') + shared.log.debug(f'Deactivate network: type=LoRA networks={len(loaded_networks)} modules={total} deactivate={len(applied_layers)} device={weights_devices} dtype={weights_dtypes} fuse={shared.opts.lora_fuse_diffusers} time={timer.summary}') modules.clear() if shared.opts.diffusers_offload_mode == "sequential": sd_models.set_diffuser_offload(sd_model, op="model") - t1 = time.time() - timer['deactivate'] += t1 - t0 + def network_activate(): - timer['backup'] = 0 - timer['calc'] = 0 - timer['apply'] = 0 - timer['move'] = 0 + t0 = time.time() + timer.clear(complete=True) sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility if shared.opts.diffusers_offload_mode == "sequential": sd_models.disable_offload(sd_model) @@ -520,14 +512,14 @@ def network_activate(): modules[component_name] = list(component.named_modules()) total = sum(len(x) for x in modules.values()) if len(loaded_networks) > 0: - pbar = rp.Progress(rp.TextColumn('[cyan]Apply network: type=LoRA'), rp.BarColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), console=shared.console) + pbar = rp.Progress(rp.TextColumn('[cyan]Network: type=LoRA action=activate'), rp.BarColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), console=shared.console) task = pbar.add_task(description='' , total=total) else: task = None pbar = nullcontext() with devices.inference_context(), pbar: wanted_names = tuple((x.name, x.te_multiplier, x.unet_multiplier, x.dyn_dim) for x in loaded_networks) if len(loaded_networks) > 0 else () - applied = 0 + applied_layers = [] backup_size = 0 weights_devices = [] weights_dtypes = [] @@ -536,26 +528,28 @@ def network_activate(): for _, module in modules[component]: network_layer_name = getattr(module, 'network_layer_name', None) current_names = getattr(module, "network_current_names", ()) - if shared.state.interrupted or network_layer_name is None or current_names == wanted_names: + if getattr(module, 'weight', None) is None or shared.state.interrupted or network_layer_name is None or current_names == wanted_names: if task is not None: - pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} skip') + pbar.update(task, advance=1) continue - weight = getattr(module, 'weight', None) - weight = weight.to(devices.device) if weight is not None else None - backup_size += network_backup_weights(module, weight, network_layer_name, wanted_names) - batch_updown, batch_ex_bias = network_calc_weights(module, weight, network_layer_name) - weights_device, weights_dtype = network_apply_weights(module, batch_updown, batch_ex_bias, orig_device) + backup_size += network_backup_weights(module, network_layer_name, wanted_names) + batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name) + if shared.opts.lora_fuse_diffusers: + weights_device, weights_dtype = network_apply_direct(module, batch_updown, batch_ex_bias) + else: + weights_device, weights_dtype = network_apply_weights(module, batch_updown, batch_ex_bias, orig_device) weights_devices.append(weights_device) weights_dtypes.append(weights_dtype) if batch_updown is not None or batch_ex_bias is not None: - applied += 1 - del weight, batch_updown, batch_ex_bias + applied_layers.append(network_layer_name) + del batch_updown, batch_ex_bias module.network_current_names = wanted_names if task is not None: - pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} modules={len(modules)} apply={applied} backup={backup_size}') + pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} modules={total} apply={len(applied_layers)} backup={backup_size}') weights_devices, weights_dtypes = list(set([x for x in weights_devices if x is not None])), list(set([x for x in weights_dtypes if x is not None])) # noqa: C403 # pylint: disable=R1718 + timer.activate = time.time() - t0 if debug and len(loaded_networks) > 0: - shared.log.debug(f'Load network: type=LoRA networks={len(loaded_networks)} modules={total} apply={applied} device={weights_devices} dtype={weights_dtypes} backup={backup_size} fuse={shared.opts.lora_fuse_diffusers} time={get_timers()}') + shared.log.debug(f'Load network: type=LoRA networks={len(loaded_networks)} modules={total} apply={len(applied_layers)} device={weights_devices} dtype={weights_dtypes} backup={backup_size} fuse={shared.opts.lora_fuse_diffusers} time={timer.summary}') modules.clear() if shared.opts.diffusers_offload_mode == "sequential": sd_models.set_diffuser_offload(sd_model, op="model") diff --git a/modules/processing.py b/modules/processing.py index 57512850a..7ae397538 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -286,7 +286,6 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: t0 = time.time() if not hasattr(p, 'skip_init'): p.init(p.all_prompts, p.all_seeds, p.all_subseeds) - extra_network_data = None debug(f'Processing inner: args={vars(p)}') for n in range(p.n_iter): pag.apply(p) @@ -311,9 +310,9 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: p.scripts.before_process_batch(p, batch_number=n, prompts=p.prompts, seeds=p.seeds, subseeds=p.subseeds) if len(p.prompts) == 0: break - p.prompts, extra_network_data = extra_networks.parse_prompts(p.prompts) - if not p.disable_extra_networks: - extra_networks.activate(p, extra_network_data) + p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts) + if not shared.native: + extra_networks.activate(p, p.network_data) if p.scripts is not None and isinstance(p.scripts, scripts.ScriptRunner): p.scripts.process_batch(p, batch_number=n, prompts=p.prompts, seeds=p.seeds, subseeds=p.subseeds) @@ -417,6 +416,10 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: timer.process.record('post') del samples + + if not shared.native: + extra_networks.deactivate(p, p.network_data) + devices.torch_gc() if hasattr(shared.sd_model, 'restore_pipeline') and shared.sd_model.restore_pipeline is not None: @@ -445,9 +448,6 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: from modules import ipadapter ipadapter.unapply(shared.sd_model) - if not p.disable_extra_networks: - extra_networks.deactivate(p, extra_network_data) - if shared.opts.include_mask: if shared.opts.mask_apply_overlay and p.overlay_images is not None and len(p.overlay_images): p.image_mask = create_binary_mask(p.overlay_images[0]) diff --git a/modules/processing_args.py b/modules/processing_args.py index d73762d29..93b0bf9b2 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -101,6 +101,7 @@ def task_specific_kwargs(p, model): def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2: typing.Optional[list]=None, negative_prompts_2: typing.Optional[list]=None, desc:str='', **kwargs): t0 = time.time() + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) apply_circular(p.tiling, model) if hasattr(model, "set_progress_bar_config"): model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba') diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index f3eb0bc37..0b4c7dfe1 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -67,7 +67,7 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {} raise AssertionError('Interrupted...') time.sleep(0.1) if hasattr(p, "stepwise_lora") and shared.native: - extra_networks.activate(p, p.extra_network_data, step=step) + extra_networks.activate(p, step=step) if latents is None: return kwargs elif shared.opts.nan_skip: diff --git a/modules/processing_class.py b/modules/processing_class.py index 21e86c1b0..2cbc07cc2 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -139,6 +139,7 @@ class StableDiffusionProcessing: self.negative_pooleds = [] self.disable_extra_networks = False self.iteration = 0 + self.network_data = {} # initializers self.prompt = prompt diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 0341cac4d..d22a9de97 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -4,7 +4,7 @@ import time import numpy as np import torch import torchvision.transforms.functional as TF -from modules import shared, devices, processing, sd_models, errors, sd_hijack_hypertile, processing_vae, sd_models_compile, hidiffusion, timer, modelstats +from modules import shared, devices, processing, sd_models, errors, sd_hijack_hypertile, processing_vae, sd_models_compile, hidiffusion, timer, modelstats, extra_networks from modules.processing_helpers import resize_hires, calculate_base_steps, calculate_hires_steps, calculate_refiner_steps, save_intermediate, update_sampler, is_txt2img, is_refiner_enabled from modules.processing_args import set_pipeline_args from modules.onnx_impl import preprocess_pipeline as preprocess_onnx_pipeline, check_parameters_changed as olive_check_parameters_changed @@ -89,6 +89,7 @@ def process_base(p: processing.StableDiffusionProcessing): sd_models.move_model(shared.sd_model.unet, devices.device) if hasattr(shared.sd_model, 'transformer'): sd_models.move_model(shared.sd_model.transformer, devices.device) + extra_networks.activate(p) hidiffusion.apply(p, shared.sd_model_type) # if 'image' in base_args: # base_args['image'] = set_latents(p) @@ -223,11 +224,14 @@ def process_hires(p: processing.StableDiffusionProcessing, output): shared.state.job = 'HiRes' shared.state.sampling_steps = hires_args.get('prior_num_inference_steps', None) or p.steps or hires_args.get('num_inference_steps', None) try: + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) sd_models.move_model(shared.sd_model, devices.device) if hasattr(shared.sd_model, 'unet'): sd_models.move_model(shared.sd_model.unet, devices.device) if hasattr(shared.sd_model, 'transformer'): sd_models.move_model(shared.sd_model.transformer, devices.device) + if 'base' in p.skip: + extra_networks.activate(p) sd_models_compile.check_deepcache(enable=True) output = shared.sd_model(**hires_args) # pylint: disable=not-callable if isinstance(output, dict): @@ -345,6 +349,7 @@ def process_refine(p: processing.StableDiffusionProcessing, output): def process_decode(p: processing.StableDiffusionProcessing, output): + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) if output is not None: if not hasattr(output, 'images') and hasattr(output, 'frames'): shared.log.debug(f'Generated: frames={len(output.frames[0])}') @@ -405,8 +410,6 @@ def process_diffusers(p: processing.StableDiffusionProcessing): shared.sd_model = orig_pipeline return results - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - # sanitize init_images if hasattr(p, 'init_images') and getattr(p, 'init_images', None) is None: del p.init_images @@ -453,13 +456,13 @@ def process_diffusers(p: processing.StableDiffusionProcessing): shared.sd_model = orig_pipeline return results - results = process_decode(p, output) + extra_networks.deactivate(p) + timer.process.add('lora', networks.timer.total) + results = process_decode(p, output) timer.process.record('decode') - timer.process.add('lora', networks.total_time()) shared.sd_model = orig_pipeline - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) if p.state == '': diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 06c0b6012..8c140e0d6 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -92,7 +92,7 @@ class PromptEmbedder: return [x for xs in xss for x in xs] # unpack EN data in case of TE LoRA - en_data = p.extra_network_data + en_data = p.network_data en_data = [idx.items for item in en_data.values() for idx in item] effective_batch = 1 if self.allsame else self.batchsize key = str([self.prompts, self.negative_prompts, effective_batch, self.clip_skip, self.steps, en_data]) diff --git a/modules/shared.py b/modules/shared.py index f8a989270..6e8f2f3fd 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -905,15 +905,12 @@ options_templates.update(options_section(('extra_networks', "Networks"), { "extra_networks_default_multiplier": OptionInfo(1.0, "Default strength", gr.Slider, {"minimum": 0.0, "maximum": 2.0, "step": 0.01}), "lora_preferred_name": OptionInfo("filename", "LoRA preferred name", gr.Radio, {"choices": ["filename", "alias"], "visible": False}), "lora_add_hashes_to_infotext": OptionInfo(False, "LoRA add hash info"), - "lora_fuse_diffusers": OptionInfo(False if not cmd_opts.use_openvino else True, "LoRA fuse directly to model"), - "lora_load_gpu": OptionInfo(True if not (cmd_opts.lowvram or cmd_opts.medvram) else False, "LoRA load directly to GPU"), - "lora_offload_backup": OptionInfo(True, "LoRA offload backup weights"), + "lora_fuse_diffusers": OptionInfo(True, "LoRA fuse directly to model"), "lora_force_diffusers": OptionInfo(False if not cmd_opts.use_openvino else True, "LoRA force loading of all models using Diffusers"), "lora_maybe_diffusers": OptionInfo(False, "LoRA force loading of specific models using Diffusers"), "lora_apply_tags": OptionInfo(0, "LoRA auto-apply tags", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), "lora_in_memory_limit": OptionInfo(0, "LoRA memory cache", gr.Slider, {"minimum": 0, "maximum": 24, "step": 1}), "lora_quant": OptionInfo("NF4","LoRA precision in quantized models", gr.Radio, {"choices": ["NF4", "FP4"]}), - "lora_low_memory": OptionInfo(False, "LoRA low memory mode"), })) options_templates.update(options_section((None, "Internal options"), { diff --git a/wiki b/wiki index c5d484397..20c9fe52f 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit c5d484397f7504fdea098d5e24c843a69c9fd2a2 +Subproject commit 20c9fe52f253c23e736227787ddebd4cbfcbfe68