add support for lora oft, lyco oft, peft

CHANGELOG.md

@@ -1,17 +1,17 @@
 # Change Log for SD.Next
 
-## Update for 2023-12-23
+## Update for 2023-12-24
 
 *Note*: based on `diffusers==0.25.0.dev0`
 
-- **Control**
+- **Control**  
   - native implementation of **ControlNet**, **ControlNet XS**, **T2I Adapters** and **IP Adapters**  
-  - top-level **Control** next to **Text** and **Image** generate
-  - supports all variations of **SD15** and **SD-XL** models
+  - top-level **Control** next to **Text** and **Image** generate  
+  - supports all variations of **SD15** and **SD-XL** models  
   - supports *Text*, *Image*, *Batch* and *Video* processing  
   - for details and list of supported models and workflows, see Wiki documentation:  
-    <https://github.com/vladmandic/automatic/wiki/Control>
-- **Diffusers**
+    <https://github.com/vladmandic/automatic/wiki/Control>  
+- **Diffusers**  
   - **AnimateDiff**  
     - can now be used with *second pass* - enhance, upscale and hires your videos!  
   - **IP Adapter**  
@@ -58,6 +58,8 @@
     - add support for block weights, thanks @AI-Casanova  
       example `<lora:SDXL_LCM_LoRA:1.0:in=0:mid=1:out=0>`  
     - add support for LyCORIS GLora networks  
+    - add support for LoRA PEFT (*Diffusers*) networks  
+    - add support for Lora-OFT (*Kohya*) and Lyco-OFT (*Kohaku*) networks  
     - reintroduce alternative loading method in settings: `lora_force_diffusers`  
     - add support for `lora_fuse_diffusers` if using alternative method  
       use if you have multiple complex loras that may be causing performance degradation  

extensions-builtin/Lora/extra_networks_lora.py

@@ -62,7 +62,7 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork):
             if network_hashes:
                 p.extra_generation_params["Lora hashes"] = ", ".join(network_hashes)
         if len(names) > 0:
-            shared.log.info(f'Applying LoRA: {names} patch={t1-t0:.2f} load={t2-t1:.2f}')
+            shared.log.info(f'LoRA apply: {names} patch={t1-t0:.2f} load={t2-t1:.2f}')
         elif self.active:
             self.active = False
 

extensions-builtin/Lora/lora_convert.py

@@ -1,9 +1,11 @@
-from typing import Dict
+import os
 import re
 import bisect
+from typing import Dict
 from modules import shared
 
 
+debug = os.environ.get('SD_LORA_DEBUG', None) is not None
 suffix_conversion = {
     "attentions": {},
     "resnets": {
@@ -144,12 +146,13 @@ class KeyConvert:
             map_keys = list(self.UNET_CONVERSION_MAP.keys())  # prefix of U-Net modules
             map_keys.sort()
             search_key = key.replace(self.LORA_PREFIX_UNET, "").replace(self.OFT_PREFIX_UNET, "").replace(self.LORA_PREFIX_TEXT_ENCODER1, "").replace(self.LORA_PREFIX_TEXT_ENCODER2, "")
-
             position = bisect.bisect_right(map_keys, search_key)
             map_key = map_keys[position - 1]
             if search_key.startswith(map_key):
-                key = key.replace(map_key, self.UNET_CONVERSION_MAP[map_key]).replace("oft","lora") # pylint: disable=unsubscriptable-object
+                key = key.replace(map_key, self.UNET_CONVERSION_MAP[map_key]).replace("oft", "lora") # pylint: disable=unsubscriptable-object
         sd_module = shared.sd_model.network_layer_mapping.get(key, None)
+        if debug and sd_module is None:
+            raise RuntimeError(f"LoRA key not found in network_layer_mapping: key={key} mapping={shared.sd_model.network_layer_mapping.keys()}")
         return key, sd_module
 
     def __call__(self, key):

extensions-builtin/Lora/lyco_helpers.py

@@ -19,3 +19,50 @@ def rebuild_cp_decomposition(up, down, mid):
     up = up.reshape(up.size(0), -1)
     down = down.reshape(down.size(0), -1)
     return torch.einsum('n m k l, i n, m j -> i j k l', mid, up, down)
+
+
+# copied from https://github.com/KohakuBlueleaf/LyCORIS/blob/dev/lycoris/modules/lokr.py
+def factorization(dimension: int, factor:int=-1) -> tuple[int, int]:
+    '''
+    return a tuple of two value of input dimension decomposed by the number closest to factor
+    second value is higher or equal than first value.
+
+    In LoRA with Kroneckor Product, first value is a value for weight scale.
+    secon value is a value for weight.
+
+    Becuase of non-commutative property, A⊗B ≠ B⊗A. Meaning of two matrices is slightly different.
+
+    examples)
+    factor
+        -1               2                4               8               16               ...
+    127 -> 1, 127   127 -> 1, 127    127 -> 1, 127   127 -> 1, 127   127 -> 1, 127
+    128 -> 8, 16    128 -> 2, 64     128 -> 4, 32    128 -> 8, 16    128 -> 8, 16
+    250 -> 10, 25   250 -> 2, 125    250 -> 2, 125   250 -> 5, 50    250 -> 10, 25
+    360 -> 8, 45    360 -> 2, 180    360 -> 4, 90    360 -> 8, 45    360 -> 12, 30
+    512 -> 16, 32   512 -> 2, 256    512 -> 4, 128   512 -> 8, 64    512 -> 16, 32
+    1024 -> 32, 32  1024 -> 2, 512   1024 -> 4, 256  1024 -> 8, 128  1024 -> 16, 64
+    '''
+
+    if factor > 0 and (dimension % factor) == 0:
+        m = factor
+        n = dimension // factor
+        if m > n:
+            n, m = m, n
+        return m, n
+    if factor < 0:
+        factor = dimension
+    m, n = 1, dimension
+    length = m + n
+    while m<n:
+        new_m = m + 1
+        while dimension%new_m != 0:
+            new_m += 1
+        new_n = dimension // new_m
+        if new_m + new_n > length or new_m>factor:
+            break
+        else:
+            m, n = new_m, new_n
+    if m > n:
+        n, m = m, n
+    return m, n
+

extensions-builtin/Lora/network_full.py

@@ -16,12 +16,12 @@ class NetworkModuleFull(network.NetworkModule):
         self.weight = weights.w.get("diff")
         self.ex_bias = weights.w.get("diff_b")
 
-    def calc_updown(self, orig_weight):
+    def calc_updown(self, target):
         output_shape = self.weight.shape
-        updown = self.weight.to(orig_weight.device, dtype=orig_weight.dtype)
+        updown = self.weight.to(target.device, dtype=target.dtype)
         if self.ex_bias is not None:
-            ex_bias = self.ex_bias.to(orig_weight.device, dtype=orig_weight.dtype)
+            ex_bias = self.ex_bias.to(target.device, dtype=target.dtype)
         else:
             ex_bias = None
 
-        return self.finalize_updown(updown, orig_weight, output_shape, ex_bias)
+        return self.finalize_updown(updown, target, output_shape, ex_bias)

extensions-builtin/Lora/network_glora.py

@@ -20,11 +20,11 @@ class NetworkModuleGLora(network.NetworkModule): # pylint: disable=abstract-meth
         self.w2a = weights.w["a2.weight"]
         self.w2b = weights.w["b2.weight"]
 
-    def calc_updown(self, orig_weight): # pylint: disable=arguments-differ
-        w1a = self.w1a.to(orig_weight.device, dtype=orig_weight.dtype)
-        w1b = self.w1b.to(orig_weight.device, dtype=orig_weight.dtype)
-        w2a = self.w2a.to(orig_weight.device, dtype=orig_weight.dtype)
-        w2b = self.w2b.to(orig_weight.device, dtype=orig_weight.dtype)
+    def calc_updown(self, target): # pylint: disable=arguments-differ
+        w1a = self.w1a.to(target.device, dtype=target.dtype)
+        w1b = self.w1b.to(target.device, dtype=target.dtype)
+        w2a = self.w2a.to(target.device, dtype=target.dtype)
+        w2b = self.w2b.to(target.device, dtype=target.dtype)
         output_shape = [w1a.size(0), w1b.size(1)]
-        updown = (w2b @ w1b) + ((orig_weight @ w2a) @ w1a)
-        return self.finalize_updown(updown, orig_weight, output_shape)
+        updown = (w2b @ w1b) + ((target @ w2a) @ w1a)
+        return self.finalize_updown(updown, target, output_shape)

extensions-builtin/Lora/network_hada.py

@@ -22,15 +22,15 @@ class NetworkModuleHada(network.NetworkModule):
         self.t1 = weights.w.get("hada_t1")
         self.t2 = weights.w.get("hada_t2")
 
-    def calc_updown(self, orig_weight):
-        w1a = self.w1a.to(orig_weight.device, dtype=orig_weight.dtype)
-        w1b = self.w1b.to(orig_weight.device, dtype=orig_weight.dtype)
-        w2a = self.w2a.to(orig_weight.device, dtype=orig_weight.dtype)
-        w2b = self.w2b.to(orig_weight.device, dtype=orig_weight.dtype)
+    def calc_updown(self, target):
+        w1a = self.w1a.to(target.device, dtype=target.dtype)
+        w1b = self.w1b.to(target.device, dtype=target.dtype)
+        w2a = self.w2a.to(target.device, dtype=target.dtype)
+        w2b = self.w2b.to(target.device, dtype=target.dtype)
         output_shape = [w1a.size(0), w1b.size(1)]
         if self.t1 is not None:
             output_shape = [w1a.size(1), w1b.size(1)]
-            t1 = self.t1.to(orig_weight.device, dtype=orig_weight.dtype)
+            t1 = self.t1.to(target.device, dtype=target.dtype)
             updown1 = lyco_helpers.make_weight_cp(t1, w1a, w1b)
             output_shape += t1.shape[2:]
         else:
@@ -38,9 +38,9 @@ class NetworkModuleHada(network.NetworkModule):
                 output_shape += w1b.shape[2:]
             updown1 = lyco_helpers.rebuild_conventional(w1a, w1b, output_shape)
         if self.t2 is not None:
-            t2 = self.t2.to(orig_weight.device, dtype=orig_weight.dtype)
+            t2 = self.t2.to(target.device, dtype=target.dtype)
             updown2 = lyco_helpers.make_weight_cp(t2, w2a, w2b)
         else:
             updown2 = lyco_helpers.rebuild_conventional(w2a, w2b, output_shape)
         updown = updown1 * updown2
-        return self.finalize_updown(updown, orig_weight, output_shape)
+        return self.finalize_updown(updown, target, output_shape)

extensions-builtin/Lora/network_ia3.py

@@ -15,12 +15,12 @@ class NetworkModuleIa3(network.NetworkModule):
         self.w = weights.w["weight"]
         self.on_input = weights.w["on_input"].item()
 
-    def calc_updown(self, orig_weight):
-        w = self.w.to(orig_weight.device, dtype=orig_weight.dtype)
-        output_shape = [w.size(0), orig_weight.size(1)]
+    def calc_updown(self, target):
+        w = self.w.to(target.device, dtype=target.dtype)
+        output_shape = [w.size(0), target.size(1)]
         if self.on_input:
             output_shape.reverse()
         else:
             w = w.reshape(-1, 1)
-        updown = orig_weight * w
-        return self.finalize_updown(updown, orig_weight, output_shape)
+        updown = target * w
+        return self.finalize_updown(updown, target, output_shape)

extensions-builtin/Lora/network_lokr.py

@@ -32,26 +32,26 @@ class NetworkModuleLokr(network.NetworkModule):
         self.dim = self.w2b.shape[0] if self.w2b is not None else self.dim
         self.t2 = weights.w.get("lokr_t2")
 
-    def calc_updown(self, orig_weight):
+    def calc_updown(self, target):
         if self.w1 is not None:
-            w1 = self.w1.to(orig_weight.device, dtype=orig_weight.dtype)
+            w1 = self.w1.to(target.device, dtype=target.dtype)
         else:
-            w1a = self.w1a.to(orig_weight.device, dtype=orig_weight.dtype)
-            w1b = self.w1b.to(orig_weight.device, dtype=orig_weight.dtype)
+            w1a = self.w1a.to(target.device, dtype=target.dtype)
+            w1b = self.w1b.to(target.device, dtype=target.dtype)
             w1 = w1a @ w1b
         if self.w2 is not None:
-            w2 = self.w2.to(orig_weight.device, dtype=orig_weight.dtype)
+            w2 = self.w2.to(target.device, dtype=target.dtype)
         elif self.t2 is None:
-            w2a = self.w2a.to(orig_weight.device, dtype=orig_weight.dtype)
-            w2b = self.w2b.to(orig_weight.device, dtype=orig_weight.dtype)
+            w2a = self.w2a.to(target.device, dtype=target.dtype)
+            w2b = self.w2b.to(target.device, dtype=target.dtype)
             w2 = w2a @ w2b
         else:
-            t2 = self.t2.to(orig_weight.device, dtype=orig_weight.dtype)
-            w2a = self.w2a.to(orig_weight.device, dtype=orig_weight.dtype)
-            w2b = self.w2b.to(orig_weight.device, dtype=orig_weight.dtype)
+            t2 = self.t2.to(target.device, dtype=target.dtype)
+            w2a = self.w2a.to(target.device, dtype=target.dtype)
+            w2b = self.w2b.to(target.device, dtype=target.dtype)
             w2 = lyco_helpers.make_weight_cp(t2, w2a, w2b)
         output_shape = [w1.size(0) * w2.size(0), w1.size(1) * w2.size(1)]
-        if len(orig_weight.shape) == 4:
-            output_shape = orig_weight.shape
+        if len(target.shape) == 4:
+            output_shape = target.shape
         updown = make_kron(output_shape, w1, w2)
-        return self.finalize_updown(updown, orig_weight, output_shape)
+        return self.finalize_updown(updown, target, output_shape)

extensions-builtin/Lora/network_lora.py

@@ -51,20 +51,20 @@ class NetworkModuleLora(network.NetworkModule):
         module.weight.requires_grad_(False)
         return module
 
-    def calc_updown(self, orig_weight): # pylint: disable=W0237
-        up = self.up_model.weight.to(orig_weight.device, dtype=orig_weight.dtype)
-        down = self.down_model.weight.to(orig_weight.device, dtype=orig_weight.dtype)
+    def calc_updown(self, target): # pylint: disable=W0237
+        up = self.up_model.weight.to(target.device, dtype=target.dtype)
+        down = self.down_model.weight.to(target.device, dtype=target.dtype)
         output_shape = [up.size(0), down.size(1)]
         if self.mid_model is not None:
             # cp-decomposition
-            mid = self.mid_model.weight.to(orig_weight.device, dtype=orig_weight.dtype)
+            mid = self.mid_model.weight.to(target.device, dtype=target.dtype)
             updown = lyco_helpers.rebuild_cp_decomposition(up, down, mid)
             output_shape += mid.shape[2:]
         else:
             if len(down.shape) == 4:
                 output_shape += down.shape[2:]
             updown = lyco_helpers.rebuild_conventional(up, down, output_shape, self.network.dyn_dim)
-        return self.finalize_updown(updown, orig_weight, output_shape)
+        return self.finalize_updown(updown, target, output_shape)
 
     def forward(self, x, y):
         self.up_model.to(device=devices.device)

extensions-builtin/Lora/network_norm.py

@@ -14,11 +14,11 @@ class NetworkModuleNorm(network.NetworkModule):
         self.w_norm = weights.w.get("w_norm")
         self.b_norm = weights.w.get("b_norm")
 
-    def calc_updown(self, orig_weight):
+    def calc_updown(self, target):
         output_shape = self.w_norm.shape
-        updown = self.w_norm.to(orig_weight.device, dtype=orig_weight.dtype)
+        updown = self.w_norm.to(target.device, dtype=target.dtype)
         if self.b_norm is not None:
-            ex_bias = self.b_norm.to(orig_weight.device, dtype=orig_weight.dtype)
+            ex_bias = self.b_norm.to(target.device, dtype=target.dtype)
         else:
             ex_bias = None
-        return self.finalize_updown(updown, orig_weight, output_shape, ex_bias)
+        return self.finalize_updown(updown, target, output_shape, ex_bias)

extensions-builtin/Lora/network_oft.py

@@ -1,49 +1,85 @@
 import torch
-import diffusers.models.lora as diffusers_lora
 import network
-from modules import devices
+from lyco_helpers import factorization
+from einops import rearrange
+
 
 class ModuleTypeOFT(network.ModuleType):
     def create_module(self, net: network.Network, weights: network.NetworkWeights):
-        """
-        weights.w.items()
-
-        alpha  :  tensor(0.0010, dtype=torch.bfloat16)
-        oft_blocks  :  tensor([[[ 0.0000e+00,  1.4400e-04,  1.7319e-03,  ..., -8.8882e-04,
-           5.7373e-03, -4.4250e-03],
-         [-1.4400e-04,  0.0000e+00,  8.6594e-04,  ...,  1.5945e-03,
-          -8.5449e-04,  1.9684e-03], ...etc...
-         , dtype=torch.bfloat16)"""
-
-        if "oft_blocks" in weights.w.keys():
-            module = NetworkModuleOFT(net, weights)
-            return module
-        else:
-            return None
+        if all(x in weights.w for x in ["oft_blocks"]) or all(x in weights.w for x in ["oft_diag"]):
+            return NetworkModuleOFT(net, weights)
 
+        return None
 
+# Supports both kohya-ss' implementation of COFT  https://github.com/kohya-ss/sd-scripts/blob/main/networks/oft.py
+# and KohakuBlueleaf's implementation of OFT/COFT https://github.com/KohakuBlueleaf/LyCORIS/blob/dev/lycoris/modules/diag_oft.py
 class NetworkModuleOFT(network.NetworkModule):
-    def __init__(self, net: network.Network, weights: network.NetworkWeights):
+    def __init__(self,  net: network.Network, weights: network.NetworkWeights):
+
         super().__init__(net, weights)
 
-        self.weights = weights.w.get("oft_blocks").to(device=devices.device)
-        self.dim = self.weights.shape[0]  # num blocks
-        self.alpha = self.multiplier()
-        self.block_size = self.weights.shape[-1]
+        self.lin_module = None
+        self.org_module: list[torch.Module] = [self.sd_module]
 
-    def get_weight(self):
-        block_Q = self.weights - self.weights.transpose(1, 2)
-        I = torch.eye(self.block_size, device=devices.device).unsqueeze(0).repeat(self.dim, 1, 1)
-        block_R = torch.matmul(I + block_Q, (I - block_Q).inverse())
-        block_R_weighted = self.alpha * block_R + (1 - self.alpha) * I
-        R = torch.block_diag(*block_R_weighted)
-        return R
+        self.scale = 1.0
 
-    def calc_updown(self, orig_weight):
-        R = self.get_weight().to(device=devices.device, dtype=orig_weight.dtype)
-        if orig_weight.dim() == 4:
-            updown = torch.einsum("oihw, op -> pihw", orig_weight, R) * self.calc_scale()
+        # kohya-ss
+        if "oft_blocks" in weights.w.keys():
+            self.is_kohya = True
+            self.oft_blocks = weights.w["oft_blocks"] # (num_blocks, block_size, block_size)
+            self.alpha = weights.w["alpha"] # alpha is constraint
+            self.dim = self.oft_blocks.shape[0] # lora dim
+        # LyCORIS
+        elif "oft_diag" in weights.w.keys():
+            self.is_kohya = False
+            self.oft_blocks = weights.w["oft_diag"]
+            # self.alpha is unused
+            self.dim = self.oft_blocks.shape[1] # (num_blocks, block_size, block_size)
+
+        is_linear = type(self.sd_module) in [torch.nn.Linear, torch.nn.modules.linear.NonDynamicallyQuantizableLinear]
+        is_conv = type(self.sd_module) in [torch.nn.Conv2d]
+        is_other_linear = type(self.sd_module) in [torch.nn.MultiheadAttention] # unsupported
+
+        if is_linear:
+            self.out_dim = self.sd_module.out_features
+        elif is_conv:
+            self.out_dim = self.sd_module.out_channels
+        elif is_other_linear:
+            self.out_dim = self.sd_module.embed_dim
+
+        if self.is_kohya:
+            self.constraint = self.alpha * self.out_dim
+            self.num_blocks = self.dim
+            self.block_size = self.out_dim // self.dim
         else:
-            updown = torch.einsum("oi, op -> pi", orig_weight, R) * self.calc_scale()
+            self.constraint = None
+            self.block_size, self.num_blocks = factorization(self.out_dim, self.dim)
 
-        return self.finalize_updown(updown, orig_weight, orig_weight.shape)
+    def calc_updown(self, target):
+        oft_blocks = self.oft_blocks.to(target.device, dtype=target.dtype)
+        eye = torch.eye(self.block_size, device=target.device)
+        constraint = self.constraint.to(target.device)
+
+        if self.is_kohya:
+            block_Q = oft_blocks - oft_blocks.transpose(1, 2) # ensure skew-symmetric orthogonal matrix
+            norm_Q = torch.norm(block_Q.flatten()).to(target.device)
+            new_norm_Q = torch.clamp(norm_Q, max=constraint)
+            block_Q = block_Q * ((new_norm_Q + 1e-8) / (norm_Q + 1e-8))
+            mat1 = eye + block_Q
+            mat2 = (eye - block_Q).float().inverse()
+            oft_blocks = torch.matmul(mat1, mat2)
+
+        R = oft_blocks.to(target.device, dtype=target.dtype)
+
+        # This errors out for MultiheadAttention, might need to be handled up-stream
+        merged_weight = rearrange(target, '(k n) ... -> k n ...', k=self.num_blocks, n=self.block_size)
+        merged_weight = torch.einsum(
+            'k n m, k n ... -> k m ...',
+            R,
+            merged_weight
+        )
+        merged_weight = rearrange(merged_weight, 'k m ... -> (k m) ...')
+
+        updown = merged_weight.to(target.device, dtype=target.dtype) - target
+        output_shape = target.shape
+        return self.finalize_updown(updown, target, output_shape)

extensions-builtin/Lora/networks.py

@@ -1,4 +1,4 @@
-from typing import Union
+from typing import Union, List
 import os
 import re
 import time
@@ -18,12 +18,12 @@ import diffusers.models.lora
 from modules import shared, devices, sd_models, sd_models_compile, errors, scripts, sd_hijack
 
 
-debug = os.environ.get('SD_LORA_DEBUG', None)
+debug = os.environ.get('SD_LORA_DEBUG', None) is not None
 originals: lora_patches.LoraPatches = None
 extra_network_lora = None
 available_networks = {}
 available_network_aliases = {}
-loaded_networks = []
+loaded_networks: List[network.Network] = []
 timer = { 'load': 0, 'apply': 0, 'restore': 0 }
 # networks_in_memory = {}
 lora_cache = {}
@@ -76,7 +76,7 @@ def assign_network_names_to_compvis_modules(sd_model):
     sd_model.network_layer_mapping = network_layer_mapping
 
 
-def load_diffusers(name, network_on_disk, lora_scale=1.0):
+def load_diffusers(name, network_on_disk, lora_scale=1.0) -> network.Network:
     t0 = time.time()
     cached = lora_cache.get(name, None)
     # if debug:
@@ -96,11 +96,11 @@ def load_diffusers(name, network_on_disk, lora_scale=1.0):
     return net
 
 
-def load_network(name, network_on_disk):
+def load_network(name, network_on_disk) -> network.Network:
     t0 = time.time()
     cached = lora_cache.get(name, None)
     if debug:
-        shared.log.debug(f'LoRA load: name="{name}" file="{network_on_disk.filename}" {"cached" if cached else ""}')
+        shared.log.debug(f'LoRA load: name="{name}" file="{network_on_disk.filename}" type=lora {"cached" if cached else ""}')
     if cached is not None:
         return cached
     net = network.Network(name, network_on_disk)
@@ -111,7 +111,16 @@ def load_network(name, network_on_disk):
     matched_networks = {}
     convert = lora_convert.KeyConvert()
     for key_network, weight in sd.items():
-        key_network_without_network_parts, network_part = key_network.split(".", 1)
+        parts = key_network.split('.')
+        if len(parts) > 5: # messy handler for diffusers peft lora
+            key_network_without_network_parts = '_'.join(parts[:-2])
+            if not key_network_without_network_parts.startswith('lora_'):
+                key_network_without_network_parts = 'lora_' + key_network_without_network_parts
+            network_part = '.'.join(parts[-2:]).replace('lora_A', 'lora_down').replace('lora_B', 'lora_up')
+        else:
+            key_network_without_network_parts, network_part = key_network.split(".", 1)
+        if debug:
+            shared.log.debug(f'LoRA load: name="{name}" full={key_network} network={network_part} key={key_network_without_network_parts}')
         key, sd_module = convert(key_network_without_network_parts)
         if sd_module is None:
             keys_failed_to_match[key_network] = key
@@ -126,12 +135,15 @@ def load_network(name, network_on_disk):
             if net_module is not None:
                 break
         if net_module is None:
-            raise AssertionError(f"Could not find a module type (out of {', '.join([x.__class__.__name__ for x in module_types])}) that would accept those keys: {', '.join(weights.w)}")
-        net.modules[key] = net_module
-    if keys_failed_to_match:
+            shared.log.error(f'LoRA unhandled: name={name} key={key} weights={weights.w.keys()}')
+        else:
+            net.modules[key] = net_module
+    if len(keys_failed_to_match) > 0:
         shared.log.warning(f"LoRA file={network_on_disk.filename} unmatched={len(keys_failed_to_match)} matched={len(matched_networks)}")
         if debug:
             shared.log.debug(f"LoRA file={network_on_disk.filename} unmatched={keys_failed_to_match}")
+    elif debug:
+        shared.log.debug(f"LoRA file={network_on_disk.filename} unmatched={len(keys_failed_to_match)} matched={len(matched_networks)}")
     lora_cache[name] = net
     t1 = time.time()
     timer['load'] += t1 - t0
@@ -169,6 +181,8 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No
     for i, (network_on_disk, name) in enumerate(zip(networks_on_disk, names)):
         net = None
         if network_on_disk is not None:
+            if debug:
+                shared.log.debug(f'LoRA load start: name="{name}" file="{network_on_disk.filename}"')
             try:
                 if recompile_model:
                     shared.compiled_model_state.lora_model.append(f"{name}:{te_multipliers[i] if te_multipliers else 1.0}")
@@ -188,7 +202,7 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No
             network_on_disk.read_hash()
         if net is None:
             failed_to_load_networks.append(name)
-            shared.log.error(f"LoRA unknown: network={name}")
+            shared.log.error(f"LoRA unknown type: network={name}")
             continue
         net.te_multiplier = te_multipliers[i] if te_multipliers else 1.0
         net.unet_multiplier = unet_multipliers[i] if unet_multipliers else 1.0
@@ -271,10 +285,11 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn
     if current_names != wanted_names:
         network_restore_weights_from_backup(self)
         for net in loaded_networks:
+            # default workflow where module is known and has weights
             module = net.modules.get(network_layer_name, None)
             if module is not None and hasattr(self, 'weight'):
                 try:
-                    with torch.no_grad():
+                    with devices.inference_context():
                         updown, ex_bias = module.calc_updown(self.weight)
                         if len(self.weight.shape) == 4 and self.weight.shape[1] == 9:
                             # inpainting model. zero pad updown to make channel[1]  4 to 9
@@ -286,17 +301,21 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn
                             else:
                                 self.bias += ex_bias
                 except RuntimeError as e:
-                    if debug:
-                        shared.log.debug(f"LoRA apply weight network={net.name} layer={network_layer_name} {e}")
                     extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1
+                    if debug:
+                        module_name = net.modules.get(network_layer_name, None)
+                        shared.log.error(f"LoRA apply weight name={net.name} module={module_name} layer={network_layer_name} {e}")
+                        errors.display(e, 'LoRA apply weight')
+                        raise RuntimeError('LoRA apply weight') from e
                 continue
+            # alternative workflow looking at _*_proj layers
             module_q = net.modules.get(network_layer_name + "_q_proj", None)
             module_k = net.modules.get(network_layer_name + "_k_proj", None)
             module_v = net.modules.get(network_layer_name + "_v_proj", None)
             module_out = net.modules.get(network_layer_name + "_out_proj", None)
             if isinstance(self, torch.nn.MultiheadAttention) and module_q and module_k and module_v and module_out:
                 try:
-                    with torch.no_grad():
+                    with devices.inference_context():
                         updown_q, _ = module_q.calc_updown(self.in_proj_weight)
                         updown_k, _ = module_k.calc_updown(self.in_proj_weight)
                         updown_v, _ = module_v.calc_updown(self.in_proj_weight)

extensions-builtin/Lora/ui_extra_networks_lora.py

@@ -19,7 +19,6 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage):
         try:
             path, _ext = os.path.splitext(l.filename)
             name = os.path.splitext(os.path.relpath(l.filename, shared.cmd_opts.lora_dir))[0]
-
             if shared.backend == shared.Backend.ORIGINAL:
                 if l.sd_version == network.SdVersion.SDXL:
                     return None

installer.py

@@ -574,36 +574,6 @@ def install_packages():
         print_profile(pr, 'Packages')
 
 
-# clone required repositories
-def install_repositories():
-    """
-    if args.profile:
-        pr = cProfile.Profile()
-        pr.enable()
-    def d(name):
-        return os.path.join(os.path.dirname(__file__), 'repositories', name)
-    log.info('Verifying repositories')
-    os.makedirs(os.path.join(os.path.dirname(__file__), 'repositories'), exist_ok=True)
-    stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git")
-    stable_diffusion_commit = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', None)
-    clone(stable_diffusion_repo, d('stable-diffusion-stability-ai'), stable_diffusion_commit)
-    taming_transformers_repo = os.environ.get('TAMING_TRANSFORMERS_REPO', "https://github.com/CompVis/taming-transformers.git")
-    taming_transformers_commit = os.environ.get('TAMING_TRANSFORMERS_COMMIT_HASH', None)
-    clone(taming_transformers_repo, d('taming-transformers'), taming_transformers_commit)
-    k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git')
-    k_diffusion_commit = os.environ.get('K_DIFFUSION_COMMIT_HASH', '0455157')
-    clone(k_diffusion_repo, d('k-diffusion'), k_diffusion_commit)
-    codeformer_repo = os.environ.get('CODEFORMER_REPO', 'https://github.com/sczhou/CodeFormer.git')
-    codeformer_commit = os.environ.get('CODEFORMER_COMMIT_HASH', "7a584fd")
-    clone(codeformer_repo, d('CodeFormer'), codeformer_commit)
-    blip_repo = os.environ.get('BLIP_REPO', 'https://github.com/salesforce/BLIP.git')
-    blip_commit = os.environ.get('BLIP_COMMIT_HASH', None)
-    clone(blip_repo, d('BLIP'), blip_commit)
-    if args.profile:
-        print_profile(pr, 'Repositories')
-    """
-
-
 # run extension installer
 def run_extension_installer(folder):
     path_installer = os.path.realpath(os.path.join(folder, "install.py"))
@@ -776,6 +746,7 @@ def set_environment():
     os.environ.setdefault('USE_TORCH', '1')
     os.environ.setdefault('UVICORN_TIMEOUT_KEEP_ALIVE', '60')
     os.environ.setdefault('KINETO_LOG_LEVEL', '3')
+    os.environ.setdefault('DO_NOT_TRACK', '1')
     os.environ.setdefault('HF_HUB_CACHE', opts.get('hfcache_dir', os.path.join(os.path.expanduser('~'), '.cache', 'huggingface', 'hub')))
     log.debug(f'Cache folder: {os.environ.get("HF_HUB_CACHE")}')
     if sys.platform == 'darwin':

launch.py

@@ -215,7 +215,6 @@ if __name__ == "__main__":
         installer.log.info('Startup: standard')
         installer.install_requirements()
         installer.install_packages()
-        installer.install_repositories()
         installer.install_submodules()
         init_paths()
         installer.install_extensions()

requirements.txt

@@ -55,7 +55,7 @@ opencv-python-headless==4.7.0.72
 diffusers==0.24.0
 einops==0.4.1
 gradio==3.43.2
-huggingface_hub==0.19.4
+huggingface_hub==0.20.1
 numexpr==2.8.4
 numpy==1.24.4
 numba==0.57.1