diff --git a/CHANGELOG.md b/CHANGELOG.md index 57c09eda4..62f49b783 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,17 +1,17 @@ # Change Log for SD.Next -## Update for 2023-12-23 +## Update for 2023-12-24 *Note*: based on `diffusers==0.25.0.dev0` -- **Control** +- **Control** - native implementation of **ControlNet**, **ControlNet XS**, **T2I Adapters** and **IP Adapters** - - top-level **Control** next to **Text** and **Image** generate - - supports all variations of **SD15** and **SD-XL** models + - top-level **Control** next to **Text** and **Image** generate + - supports all variations of **SD15** and **SD-XL** models - supports *Text*, *Image*, *Batch* and *Video* processing - for details and list of supported models and workflows, see Wiki documentation: - -- **Diffusers** + +- **Diffusers** - **AnimateDiff** - can now be used with *second pass* - enhance, upscale and hires your videos! - **IP Adapter** @@ -58,6 +58,8 @@ - add support for block weights, thanks @AI-Casanova example `` - add support for LyCORIS GLora networks + - add support for LoRA PEFT (*Diffusers*) networks + - add support for Lora-OFT (*Kohya*) and Lyco-OFT (*Kohaku*) networks - reintroduce alternative loading method in settings: `lora_force_diffusers` - add support for `lora_fuse_diffusers` if using alternative method use if you have multiple complex loras that may be causing performance degradation diff --git a/extensions-builtin/Lora/extra_networks_lora.py b/extensions-builtin/Lora/extra_networks_lora.py index 5ef27e0fc..6cdfd03a8 100644 --- a/extensions-builtin/Lora/extra_networks_lora.py +++ b/extensions-builtin/Lora/extra_networks_lora.py @@ -62,7 +62,7 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): if network_hashes: p.extra_generation_params["Lora hashes"] = ", ".join(network_hashes) if len(names) > 0: - shared.log.info(f'Applying LoRA: {names} patch={t1-t0:.2f} load={t2-t1:.2f}') + shared.log.info(f'LoRA apply: {names} patch={t1-t0:.2f} load={t2-t1:.2f}') elif self.active: self.active = False diff --git a/extensions-builtin/Lora/lora_convert.py b/extensions-builtin/Lora/lora_convert.py index fb314f258..65f6a0adb 100644 --- a/extensions-builtin/Lora/lora_convert.py +++ b/extensions-builtin/Lora/lora_convert.py @@ -1,9 +1,11 @@ -from typing import Dict +import os import re import bisect +from typing import Dict from modules import shared +debug = os.environ.get('SD_LORA_DEBUG', None) is not None suffix_conversion = { "attentions": {}, "resnets": { @@ -144,12 +146,13 @@ class KeyConvert: map_keys = list(self.UNET_CONVERSION_MAP.keys()) # prefix of U-Net modules map_keys.sort() search_key = key.replace(self.LORA_PREFIX_UNET, "").replace(self.OFT_PREFIX_UNET, "").replace(self.LORA_PREFIX_TEXT_ENCODER1, "").replace(self.LORA_PREFIX_TEXT_ENCODER2, "") - position = bisect.bisect_right(map_keys, search_key) map_key = map_keys[position - 1] if search_key.startswith(map_key): - key = key.replace(map_key, self.UNET_CONVERSION_MAP[map_key]).replace("oft","lora") # pylint: disable=unsubscriptable-object + key = key.replace(map_key, self.UNET_CONVERSION_MAP[map_key]).replace("oft", "lora") # pylint: disable=unsubscriptable-object sd_module = shared.sd_model.network_layer_mapping.get(key, None) + if debug and sd_module is None: + raise RuntimeError(f"LoRA key not found in network_layer_mapping: key={key} mapping={shared.sd_model.network_layer_mapping.keys()}") return key, sd_module def __call__(self, key): diff --git a/extensions-builtin/Lora/lyco_helpers.py b/extensions-builtin/Lora/lyco_helpers.py index 279b34bc9..1679a0ce6 100644 --- a/extensions-builtin/Lora/lyco_helpers.py +++ b/extensions-builtin/Lora/lyco_helpers.py @@ -19,3 +19,50 @@ def rebuild_cp_decomposition(up, down, mid): up = up.reshape(up.size(0), -1) down = down.reshape(down.size(0), -1) return torch.einsum('n m k l, i n, m j -> i j k l', mid, up, down) + + +# copied from https://github.com/KohakuBlueleaf/LyCORIS/blob/dev/lycoris/modules/lokr.py +def factorization(dimension: int, factor:int=-1) -> tuple[int, int]: + ''' + return a tuple of two value of input dimension decomposed by the number closest to factor + second value is higher or equal than first value. + + In LoRA with Kroneckor Product, first value is a value for weight scale. + secon value is a value for weight. + + Becuase of non-commutative property, A⊗B ≠ B⊗A. Meaning of two matrices is slightly different. + + examples) + factor + -1 2 4 8 16 ... + 127 -> 1, 127 127 -> 1, 127 127 -> 1, 127 127 -> 1, 127 127 -> 1, 127 + 128 -> 8, 16 128 -> 2, 64 128 -> 4, 32 128 -> 8, 16 128 -> 8, 16 + 250 -> 10, 25 250 -> 2, 125 250 -> 2, 125 250 -> 5, 50 250 -> 10, 25 + 360 -> 8, 45 360 -> 2, 180 360 -> 4, 90 360 -> 8, 45 360 -> 12, 30 + 512 -> 16, 32 512 -> 2, 256 512 -> 4, 128 512 -> 8, 64 512 -> 16, 32 + 1024 -> 32, 32 1024 -> 2, 512 1024 -> 4, 256 1024 -> 8, 128 1024 -> 16, 64 + ''' + + if factor > 0 and (dimension % factor) == 0: + m = factor + n = dimension // factor + if m > n: + n, m = m, n + return m, n + if factor < 0: + factor = dimension + m, n = 1, dimension + length = m + n + while m length or new_m>factor: + break + else: + m, n = new_m, new_n + if m > n: + n, m = m, n + return m, n + diff --git a/extensions-builtin/Lora/network_full.py b/extensions-builtin/Lora/network_full.py index bf6930e96..233791712 100644 --- a/extensions-builtin/Lora/network_full.py +++ b/extensions-builtin/Lora/network_full.py @@ -16,12 +16,12 @@ class NetworkModuleFull(network.NetworkModule): self.weight = weights.w.get("diff") self.ex_bias = weights.w.get("diff_b") - def calc_updown(self, orig_weight): + def calc_updown(self, target): output_shape = self.weight.shape - updown = self.weight.to(orig_weight.device, dtype=orig_weight.dtype) + updown = self.weight.to(target.device, dtype=target.dtype) if self.ex_bias is not None: - ex_bias = self.ex_bias.to(orig_weight.device, dtype=orig_weight.dtype) + ex_bias = self.ex_bias.to(target.device, dtype=target.dtype) else: ex_bias = None - return self.finalize_updown(updown, orig_weight, output_shape, ex_bias) + return self.finalize_updown(updown, target, output_shape, ex_bias) diff --git a/extensions-builtin/Lora/network_glora.py b/extensions-builtin/Lora/network_glora.py index 3c54a14cc..ce6ceaa1b 100644 --- a/extensions-builtin/Lora/network_glora.py +++ b/extensions-builtin/Lora/network_glora.py @@ -20,11 +20,11 @@ class NetworkModuleGLora(network.NetworkModule): # pylint: disable=abstract-meth self.w2a = weights.w["a2.weight"] self.w2b = weights.w["b2.weight"] - def calc_updown(self, orig_weight): # pylint: disable=arguments-differ - w1a = self.w1a.to(orig_weight.device, dtype=orig_weight.dtype) - w1b = self.w1b.to(orig_weight.device, dtype=orig_weight.dtype) - w2a = self.w2a.to(orig_weight.device, dtype=orig_weight.dtype) - w2b = self.w2b.to(orig_weight.device, dtype=orig_weight.dtype) + def calc_updown(self, target): # pylint: disable=arguments-differ + w1a = self.w1a.to(target.device, dtype=target.dtype) + w1b = self.w1b.to(target.device, dtype=target.dtype) + w2a = self.w2a.to(target.device, dtype=target.dtype) + w2b = self.w2b.to(target.device, dtype=target.dtype) output_shape = [w1a.size(0), w1b.size(1)] - updown = (w2b @ w1b) + ((orig_weight @ w2a) @ w1a) - return self.finalize_updown(updown, orig_weight, output_shape) + updown = (w2b @ w1b) + ((target @ w2a) @ w1a) + return self.finalize_updown(updown, target, output_shape) diff --git a/extensions-builtin/Lora/network_hada.py b/extensions-builtin/Lora/network_hada.py index 78e8c1569..0feda761e 100644 --- a/extensions-builtin/Lora/network_hada.py +++ b/extensions-builtin/Lora/network_hada.py @@ -22,15 +22,15 @@ class NetworkModuleHada(network.NetworkModule): self.t1 = weights.w.get("hada_t1") self.t2 = weights.w.get("hada_t2") - def calc_updown(self, orig_weight): - w1a = self.w1a.to(orig_weight.device, dtype=orig_weight.dtype) - w1b = self.w1b.to(orig_weight.device, dtype=orig_weight.dtype) - w2a = self.w2a.to(orig_weight.device, dtype=orig_weight.dtype) - w2b = self.w2b.to(orig_weight.device, dtype=orig_weight.dtype) + def calc_updown(self, target): + w1a = self.w1a.to(target.device, dtype=target.dtype) + w1b = self.w1b.to(target.device, dtype=target.dtype) + w2a = self.w2a.to(target.device, dtype=target.dtype) + w2b = self.w2b.to(target.device, dtype=target.dtype) output_shape = [w1a.size(0), w1b.size(1)] if self.t1 is not None: output_shape = [w1a.size(1), w1b.size(1)] - t1 = self.t1.to(orig_weight.device, dtype=orig_weight.dtype) + t1 = self.t1.to(target.device, dtype=target.dtype) updown1 = lyco_helpers.make_weight_cp(t1, w1a, w1b) output_shape += t1.shape[2:] else: @@ -38,9 +38,9 @@ class NetworkModuleHada(network.NetworkModule): output_shape += w1b.shape[2:] updown1 = lyco_helpers.rebuild_conventional(w1a, w1b, output_shape) if self.t2 is not None: - t2 = self.t2.to(orig_weight.device, dtype=orig_weight.dtype) + t2 = self.t2.to(target.device, dtype=target.dtype) updown2 = lyco_helpers.make_weight_cp(t2, w2a, w2b) else: updown2 = lyco_helpers.rebuild_conventional(w2a, w2b, output_shape) updown = updown1 * updown2 - return self.finalize_updown(updown, orig_weight, output_shape) + return self.finalize_updown(updown, target, output_shape) diff --git a/extensions-builtin/Lora/network_ia3.py b/extensions-builtin/Lora/network_ia3.py index f8d86926f..cb39df228 100644 --- a/extensions-builtin/Lora/network_ia3.py +++ b/extensions-builtin/Lora/network_ia3.py @@ -15,12 +15,12 @@ class NetworkModuleIa3(network.NetworkModule): self.w = weights.w["weight"] self.on_input = weights.w["on_input"].item() - def calc_updown(self, orig_weight): - w = self.w.to(orig_weight.device, dtype=orig_weight.dtype) - output_shape = [w.size(0), orig_weight.size(1)] + def calc_updown(self, target): + w = self.w.to(target.device, dtype=target.dtype) + output_shape = [w.size(0), target.size(1)] if self.on_input: output_shape.reverse() else: w = w.reshape(-1, 1) - updown = orig_weight * w - return self.finalize_updown(updown, orig_weight, output_shape) + updown = target * w + return self.finalize_updown(updown, target, output_shape) diff --git a/extensions-builtin/Lora/network_lokr.py b/extensions-builtin/Lora/network_lokr.py index 426d64308..20387efee 100644 --- a/extensions-builtin/Lora/network_lokr.py +++ b/extensions-builtin/Lora/network_lokr.py @@ -32,26 +32,26 @@ class NetworkModuleLokr(network.NetworkModule): self.dim = self.w2b.shape[0] if self.w2b is not None else self.dim self.t2 = weights.w.get("lokr_t2") - def calc_updown(self, orig_weight): + def calc_updown(self, target): if self.w1 is not None: - w1 = self.w1.to(orig_weight.device, dtype=orig_weight.dtype) + w1 = self.w1.to(target.device, dtype=target.dtype) else: - w1a = self.w1a.to(orig_weight.device, dtype=orig_weight.dtype) - w1b = self.w1b.to(orig_weight.device, dtype=orig_weight.dtype) + w1a = self.w1a.to(target.device, dtype=target.dtype) + w1b = self.w1b.to(target.device, dtype=target.dtype) w1 = w1a @ w1b if self.w2 is not None: - w2 = self.w2.to(orig_weight.device, dtype=orig_weight.dtype) + w2 = self.w2.to(target.device, dtype=target.dtype) elif self.t2 is None: - w2a = self.w2a.to(orig_weight.device, dtype=orig_weight.dtype) - w2b = self.w2b.to(orig_weight.device, dtype=orig_weight.dtype) + w2a = self.w2a.to(target.device, dtype=target.dtype) + w2b = self.w2b.to(target.device, dtype=target.dtype) w2 = w2a @ w2b else: - t2 = self.t2.to(orig_weight.device, dtype=orig_weight.dtype) - w2a = self.w2a.to(orig_weight.device, dtype=orig_weight.dtype) - w2b = self.w2b.to(orig_weight.device, dtype=orig_weight.dtype) + t2 = self.t2.to(target.device, dtype=target.dtype) + w2a = self.w2a.to(target.device, dtype=target.dtype) + w2b = self.w2b.to(target.device, dtype=target.dtype) w2 = lyco_helpers.make_weight_cp(t2, w2a, w2b) output_shape = [w1.size(0) * w2.size(0), w1.size(1) * w2.size(1)] - if len(orig_weight.shape) == 4: - output_shape = orig_weight.shape + if len(target.shape) == 4: + output_shape = target.shape updown = make_kron(output_shape, w1, w2) - return self.finalize_updown(updown, orig_weight, output_shape) + return self.finalize_updown(updown, target, output_shape) diff --git a/extensions-builtin/Lora/network_lora.py b/extensions-builtin/Lora/network_lora.py index 5dcb05322..8c2c4c8a5 100644 --- a/extensions-builtin/Lora/network_lora.py +++ b/extensions-builtin/Lora/network_lora.py @@ -51,20 +51,20 @@ class NetworkModuleLora(network.NetworkModule): module.weight.requires_grad_(False) return module - def calc_updown(self, orig_weight): # pylint: disable=W0237 - up = self.up_model.weight.to(orig_weight.device, dtype=orig_weight.dtype) - down = self.down_model.weight.to(orig_weight.device, dtype=orig_weight.dtype) + def calc_updown(self, target): # pylint: disable=W0237 + up = self.up_model.weight.to(target.device, dtype=target.dtype) + down = self.down_model.weight.to(target.device, dtype=target.dtype) output_shape = [up.size(0), down.size(1)] if self.mid_model is not None: # cp-decomposition - mid = self.mid_model.weight.to(orig_weight.device, dtype=orig_weight.dtype) + mid = self.mid_model.weight.to(target.device, dtype=target.dtype) updown = lyco_helpers.rebuild_cp_decomposition(up, down, mid) output_shape += mid.shape[2:] else: if len(down.shape) == 4: output_shape += down.shape[2:] updown = lyco_helpers.rebuild_conventional(up, down, output_shape, self.network.dyn_dim) - return self.finalize_updown(updown, orig_weight, output_shape) + return self.finalize_updown(updown, target, output_shape) def forward(self, x, y): self.up_model.to(device=devices.device) diff --git a/extensions-builtin/Lora/network_norm.py b/extensions-builtin/Lora/network_norm.py index a291fbad3..f327b9754 100644 --- a/extensions-builtin/Lora/network_norm.py +++ b/extensions-builtin/Lora/network_norm.py @@ -14,11 +14,11 @@ class NetworkModuleNorm(network.NetworkModule): self.w_norm = weights.w.get("w_norm") self.b_norm = weights.w.get("b_norm") - def calc_updown(self, orig_weight): + def calc_updown(self, target): output_shape = self.w_norm.shape - updown = self.w_norm.to(orig_weight.device, dtype=orig_weight.dtype) + updown = self.w_norm.to(target.device, dtype=target.dtype) if self.b_norm is not None: - ex_bias = self.b_norm.to(orig_weight.device, dtype=orig_weight.dtype) + ex_bias = self.b_norm.to(target.device, dtype=target.dtype) else: ex_bias = None - return self.finalize_updown(updown, orig_weight, output_shape, ex_bias) + return self.finalize_updown(updown, target, output_shape, ex_bias) diff --git a/extensions-builtin/Lora/network_oft.py b/extensions-builtin/Lora/network_oft.py index 6d350671a..6cadc36d0 100644 --- a/extensions-builtin/Lora/network_oft.py +++ b/extensions-builtin/Lora/network_oft.py @@ -1,49 +1,85 @@ import torch -import diffusers.models.lora as diffusers_lora import network -from modules import devices +from lyco_helpers import factorization +from einops import rearrange + class ModuleTypeOFT(network.ModuleType): def create_module(self, net: network.Network, weights: network.NetworkWeights): - """ - weights.w.items() - - alpha : tensor(0.0010, dtype=torch.bfloat16) - oft_blocks : tensor([[[ 0.0000e+00, 1.4400e-04, 1.7319e-03, ..., -8.8882e-04, - 5.7373e-03, -4.4250e-03], - [-1.4400e-04, 0.0000e+00, 8.6594e-04, ..., 1.5945e-03, - -8.5449e-04, 1.9684e-03], ...etc... - , dtype=torch.bfloat16)""" - - if "oft_blocks" in weights.w.keys(): - module = NetworkModuleOFT(net, weights) - return module - else: - return None + if all(x in weights.w for x in ["oft_blocks"]) or all(x in weights.w for x in ["oft_diag"]): + return NetworkModuleOFT(net, weights) + return None +# Supports both kohya-ss' implementation of COFT https://github.com/kohya-ss/sd-scripts/blob/main/networks/oft.py +# and KohakuBlueleaf's implementation of OFT/COFT https://github.com/KohakuBlueleaf/LyCORIS/blob/dev/lycoris/modules/diag_oft.py class NetworkModuleOFT(network.NetworkModule): - def __init__(self, net: network.Network, weights: network.NetworkWeights): + def __init__(self, net: network.Network, weights: network.NetworkWeights): + super().__init__(net, weights) - self.weights = weights.w.get("oft_blocks").to(device=devices.device) - self.dim = self.weights.shape[0] # num blocks - self.alpha = self.multiplier() - self.block_size = self.weights.shape[-1] + self.lin_module = None + self.org_module: list[torch.Module] = [self.sd_module] - def get_weight(self): - block_Q = self.weights - self.weights.transpose(1, 2) - I = torch.eye(self.block_size, device=devices.device).unsqueeze(0).repeat(self.dim, 1, 1) - block_R = torch.matmul(I + block_Q, (I - block_Q).inverse()) - block_R_weighted = self.alpha * block_R + (1 - self.alpha) * I - R = torch.block_diag(*block_R_weighted) - return R + self.scale = 1.0 - def calc_updown(self, orig_weight): - R = self.get_weight().to(device=devices.device, dtype=orig_weight.dtype) - if orig_weight.dim() == 4: - updown = torch.einsum("oihw, op -> pihw", orig_weight, R) * self.calc_scale() + # kohya-ss + if "oft_blocks" in weights.w.keys(): + self.is_kohya = True + self.oft_blocks = weights.w["oft_blocks"] # (num_blocks, block_size, block_size) + self.alpha = weights.w["alpha"] # alpha is constraint + self.dim = self.oft_blocks.shape[0] # lora dim + # LyCORIS + elif "oft_diag" in weights.w.keys(): + self.is_kohya = False + self.oft_blocks = weights.w["oft_diag"] + # self.alpha is unused + self.dim = self.oft_blocks.shape[1] # (num_blocks, block_size, block_size) + + is_linear = type(self.sd_module) in [torch.nn.Linear, torch.nn.modules.linear.NonDynamicallyQuantizableLinear] + is_conv = type(self.sd_module) in [torch.nn.Conv2d] + is_other_linear = type(self.sd_module) in [torch.nn.MultiheadAttention] # unsupported + + if is_linear: + self.out_dim = self.sd_module.out_features + elif is_conv: + self.out_dim = self.sd_module.out_channels + elif is_other_linear: + self.out_dim = self.sd_module.embed_dim + + if self.is_kohya: + self.constraint = self.alpha * self.out_dim + self.num_blocks = self.dim + self.block_size = self.out_dim // self.dim else: - updown = torch.einsum("oi, op -> pi", orig_weight, R) * self.calc_scale() + self.constraint = None + self.block_size, self.num_blocks = factorization(self.out_dim, self.dim) - return self.finalize_updown(updown, orig_weight, orig_weight.shape) + def calc_updown(self, target): + oft_blocks = self.oft_blocks.to(target.device, dtype=target.dtype) + eye = torch.eye(self.block_size, device=target.device) + constraint = self.constraint.to(target.device) + + if self.is_kohya: + block_Q = oft_blocks - oft_blocks.transpose(1, 2) # ensure skew-symmetric orthogonal matrix + norm_Q = torch.norm(block_Q.flatten()).to(target.device) + new_norm_Q = torch.clamp(norm_Q, max=constraint) + block_Q = block_Q * ((new_norm_Q + 1e-8) / (norm_Q + 1e-8)) + mat1 = eye + block_Q + mat2 = (eye - block_Q).float().inverse() + oft_blocks = torch.matmul(mat1, mat2) + + R = oft_blocks.to(target.device, dtype=target.dtype) + + # This errors out for MultiheadAttention, might need to be handled up-stream + merged_weight = rearrange(target, '(k n) ... -> k n ...', k=self.num_blocks, n=self.block_size) + merged_weight = torch.einsum( + 'k n m, k n ... -> k m ...', + R, + merged_weight + ) + merged_weight = rearrange(merged_weight, 'k m ... -> (k m) ...') + + updown = merged_weight.to(target.device, dtype=target.dtype) - target + output_shape = target.shape + return self.finalize_updown(updown, target, output_shape) diff --git a/extensions-builtin/Lora/networks.py b/extensions-builtin/Lora/networks.py index fb8b9e138..4ad17df1b 100644 --- a/extensions-builtin/Lora/networks.py +++ b/extensions-builtin/Lora/networks.py @@ -1,4 +1,4 @@ -from typing import Union +from typing import Union, List import os import re import time @@ -18,12 +18,12 @@ import diffusers.models.lora from modules import shared, devices, sd_models, sd_models_compile, errors, scripts, sd_hijack -debug = os.environ.get('SD_LORA_DEBUG', None) +debug = os.environ.get('SD_LORA_DEBUG', None) is not None originals: lora_patches.LoraPatches = None extra_network_lora = None available_networks = {} available_network_aliases = {} -loaded_networks = [] +loaded_networks: List[network.Network] = [] timer = { 'load': 0, 'apply': 0, 'restore': 0 } # networks_in_memory = {} lora_cache = {} @@ -76,7 +76,7 @@ def assign_network_names_to_compvis_modules(sd_model): sd_model.network_layer_mapping = network_layer_mapping -def load_diffusers(name, network_on_disk, lora_scale=1.0): +def load_diffusers(name, network_on_disk, lora_scale=1.0) -> network.Network: t0 = time.time() cached = lora_cache.get(name, None) # if debug: @@ -96,11 +96,11 @@ def load_diffusers(name, network_on_disk, lora_scale=1.0): return net -def load_network(name, network_on_disk): +def load_network(name, network_on_disk) -> network.Network: t0 = time.time() cached = lora_cache.get(name, None) if debug: - shared.log.debug(f'LoRA load: name="{name}" file="{network_on_disk.filename}" {"cached" if cached else ""}') + shared.log.debug(f'LoRA load: name="{name}" file="{network_on_disk.filename}" type=lora {"cached" if cached else ""}') if cached is not None: return cached net = network.Network(name, network_on_disk) @@ -111,7 +111,16 @@ def load_network(name, network_on_disk): matched_networks = {} convert = lora_convert.KeyConvert() for key_network, weight in sd.items(): - key_network_without_network_parts, network_part = key_network.split(".", 1) + parts = key_network.split('.') + if len(parts) > 5: # messy handler for diffusers peft lora + key_network_without_network_parts = '_'.join(parts[:-2]) + if not key_network_without_network_parts.startswith('lora_'): + key_network_without_network_parts = 'lora_' + key_network_without_network_parts + network_part = '.'.join(parts[-2:]).replace('lora_A', 'lora_down').replace('lora_B', 'lora_up') + else: + key_network_without_network_parts, network_part = key_network.split(".", 1) + if debug: + shared.log.debug(f'LoRA load: name="{name}" full={key_network} network={network_part} key={key_network_without_network_parts}') key, sd_module = convert(key_network_without_network_parts) if sd_module is None: keys_failed_to_match[key_network] = key @@ -126,12 +135,15 @@ def load_network(name, network_on_disk): if net_module is not None: break if net_module is None: - raise AssertionError(f"Could not find a module type (out of {', '.join([x.__class__.__name__ for x in module_types])}) that would accept those keys: {', '.join(weights.w)}") - net.modules[key] = net_module - if keys_failed_to_match: + shared.log.error(f'LoRA unhandled: name={name} key={key} weights={weights.w.keys()}') + else: + net.modules[key] = net_module + if len(keys_failed_to_match) > 0: shared.log.warning(f"LoRA file={network_on_disk.filename} unmatched={len(keys_failed_to_match)} matched={len(matched_networks)}") if debug: shared.log.debug(f"LoRA file={network_on_disk.filename} unmatched={keys_failed_to_match}") + elif debug: + shared.log.debug(f"LoRA file={network_on_disk.filename} unmatched={len(keys_failed_to_match)} matched={len(matched_networks)}") lora_cache[name] = net t1 = time.time() timer['load'] += t1 - t0 @@ -169,6 +181,8 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No for i, (network_on_disk, name) in enumerate(zip(networks_on_disk, names)): net = None if network_on_disk is not None: + if debug: + shared.log.debug(f'LoRA load start: name="{name}" file="{network_on_disk.filename}"') try: if recompile_model: shared.compiled_model_state.lora_model.append(f"{name}:{te_multipliers[i] if te_multipliers else 1.0}") @@ -188,7 +202,7 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No network_on_disk.read_hash() if net is None: failed_to_load_networks.append(name) - shared.log.error(f"LoRA unknown: network={name}") + shared.log.error(f"LoRA unknown type: network={name}") continue net.te_multiplier = te_multipliers[i] if te_multipliers else 1.0 net.unet_multiplier = unet_multipliers[i] if unet_multipliers else 1.0 @@ -271,10 +285,11 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn if current_names != wanted_names: network_restore_weights_from_backup(self) for net in loaded_networks: + # default workflow where module is known and has weights module = net.modules.get(network_layer_name, None) if module is not None and hasattr(self, 'weight'): try: - with torch.no_grad(): + with devices.inference_context(): updown, ex_bias = module.calc_updown(self.weight) if len(self.weight.shape) == 4 and self.weight.shape[1] == 9: # inpainting model. zero pad updown to make channel[1] 4 to 9 @@ -286,17 +301,21 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn else: self.bias += ex_bias except RuntimeError as e: - if debug: - shared.log.debug(f"LoRA apply weight network={net.name} layer={network_layer_name} {e}") extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 + if debug: + module_name = net.modules.get(network_layer_name, None) + shared.log.error(f"LoRA apply weight name={net.name} module={module_name} layer={network_layer_name} {e}") + errors.display(e, 'LoRA apply weight') + raise RuntimeError('LoRA apply weight') from e continue + # alternative workflow looking at _*_proj layers module_q = net.modules.get(network_layer_name + "_q_proj", None) module_k = net.modules.get(network_layer_name + "_k_proj", None) module_v = net.modules.get(network_layer_name + "_v_proj", None) module_out = net.modules.get(network_layer_name + "_out_proj", None) if isinstance(self, torch.nn.MultiheadAttention) and module_q and module_k and module_v and module_out: try: - with torch.no_grad(): + with devices.inference_context(): updown_q, _ = module_q.calc_updown(self.in_proj_weight) updown_k, _ = module_k.calc_updown(self.in_proj_weight) updown_v, _ = module_v.calc_updown(self.in_proj_weight) diff --git a/extensions-builtin/Lora/ui_extra_networks_lora.py b/extensions-builtin/Lora/ui_extra_networks_lora.py index bd408b327..cee5ad29e 100644 --- a/extensions-builtin/Lora/ui_extra_networks_lora.py +++ b/extensions-builtin/Lora/ui_extra_networks_lora.py @@ -19,7 +19,6 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage): try: path, _ext = os.path.splitext(l.filename) name = os.path.splitext(os.path.relpath(l.filename, shared.cmd_opts.lora_dir))[0] - if shared.backend == shared.Backend.ORIGINAL: if l.sd_version == network.SdVersion.SDXL: return None diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index 01e4574d8..4ceb6e8a4 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit 01e4574d8e01fa00628fdae0c8215283a1c36a8d +Subproject commit 4ceb6e8a4b86605cdeb6cd7087d1d7ef34452d85 diff --git a/installer.py b/installer.py index 29d65bb5d..1db105dca 100644 --- a/installer.py +++ b/installer.py @@ -574,36 +574,6 @@ def install_packages(): print_profile(pr, 'Packages') -# clone required repositories -def install_repositories(): - """ - if args.profile: - pr = cProfile.Profile() - pr.enable() - def d(name): - return os.path.join(os.path.dirname(__file__), 'repositories', name) - log.info('Verifying repositories') - os.makedirs(os.path.join(os.path.dirname(__file__), 'repositories'), exist_ok=True) - stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git") - stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', None) - clone(stable_diffusion_repo, d('stable-diffusion-stability-ai'), stable_diffusion_commit) - taming_transformers_repo = os.environ.get('TAMING_TRANSFORMERS_REPO', "https://github.com/CompVis/taming-transformers.git") - taming_transformers_commit = os.environ.get('TAMING_TRANSFORMERS_COMMIT_HASH', None) - clone(taming_transformers_repo, d('taming-transformers'), taming_transformers_commit) - k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git') - k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', '0455157') - clone(k_diffusion_repo, d('k-diffusion'), k_diffusion_commit) - codeformer_repo = os.environ.get('CODEFORMER_REPO', 'https://github.com/sczhou/CodeFormer.git') - codeformer_commit = os.environ.get('CODEFORMER_COMMIT_HASH', "7a584fd") - clone(codeformer_repo, d('CodeFormer'), codeformer_commit) - blip_repo = os.environ.get('BLIP_REPO', 'https://github.com/salesforce/BLIP.git') - blip_commit = os.environ.get('BLIP_COMMIT_HASH', None) - clone(blip_repo, d('BLIP'), blip_commit) - if args.profile: - print_profile(pr, 'Repositories') - """ - - # run extension installer def run_extension_installer(folder): path_installer = os.path.realpath(os.path.join(folder, "install.py")) @@ -776,6 +746,7 @@ def set_environment(): os.environ.setdefault('USE_TORCH', '1') os.environ.setdefault('UVICORN_TIMEOUT_KEEP_ALIVE', '60') os.environ.setdefault('KINETO_LOG_LEVEL', '3') + os.environ.setdefault('DO_NOT_TRACK', '1') os.environ.setdefault('HF_HUB_CACHE', opts.get('hfcache_dir', os.path.join(os.path.expanduser('~'), '.cache', 'huggingface', 'hub'))) log.debug(f'Cache folder: {os.environ.get("HF_HUB_CACHE")}') if sys.platform == 'darwin': diff --git a/launch.py b/launch.py index 00adf4475..4617e5f60 100755 --- a/launch.py +++ b/launch.py @@ -215,7 +215,6 @@ if __name__ == "__main__": installer.log.info('Startup: standard') installer.install_requirements() installer.install_packages() - installer.install_repositories() installer.install_submodules() init_paths() installer.install_extensions() diff --git a/modules/lora b/modules/lora index 0a52b83c6..1a36f9dc6 160000 --- a/modules/lora +++ b/modules/lora @@ -1 +1 @@ -Subproject commit 0a52b83c6a4c87d6375dc6d4d55ad78c39f9f666 +Subproject commit 1a36f9dc65fb3be8baa9dcbde832048cc5644efa diff --git a/requirements.txt b/requirements.txt index be2442bb9..5e3c49937 100644 --- a/requirements.txt +++ b/requirements.txt @@ -55,7 +55,7 @@ opencv-python-headless==4.7.0.72 diffusers==0.24.0 einops==0.4.1 gradio==3.43.2 -huggingface_hub==0.19.4 +huggingface_hub==0.20.1 numexpr==2.8.4 numpy==1.24.4 numba==0.57.1