From f2fc41cfc26688adae7e878ba234f93db859a3c8 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 8 Oct 2023 12:00:51 -0400 Subject: [PATCH] refactor lora load/unload --- .../Lora/extra_networks_lora.py | 11 + extensions-builtin/Lora/lora_convert.py | 208 +++++++++++ extensions-builtin/Lora/lora_patches.py | 3 + extensions-builtin/Lora/network.py | 5 +- extensions-builtin/Lora/networks.py | 325 ++++-------------- .../Lora/scripts/lora_script.py | 19 +- modules/processing.py | 23 +- modules/script_callbacks.py | 32 ++ modules/shared.py | 13 +- modules/styles.py | 2 +- 10 files changed, 344 insertions(+), 297 deletions(-) create mode 100644 extensions-builtin/Lora/lora_convert.py diff --git a/extensions-builtin/Lora/extra_networks_lora.py b/extensions-builtin/Lora/extra_networks_lora.py index 4882614bf..131dae9b4 100644 --- a/extensions-builtin/Lora/extra_networks_lora.py +++ b/extensions-builtin/Lora/extra_networks_lora.py @@ -1,4 +1,6 @@ +import time import networks +import lora_patches from modules import extra_networks, shared @@ -9,6 +11,8 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): """mapping of network names to the number of errors the network had during operation""" def activate(self, p, params_list): + t0 = time.time() + networks.originals = lora_patches.LoraPatches() additional = shared.opts.sd_lora self.errors.clear() if additional != "None" and additional in networks.available_networks and not any(x for x in params_list if x.items[0] == additional): @@ -30,7 +34,9 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): te_multipliers.append(te_multiplier) unet_multipliers.append(unet_multiplier) dyn_dims.append(dyn_dim) + t1 = time.time() networks.load_networks(names, te_multipliers, unet_multipliers, dyn_dims) + t2 = time.time() if shared.opts.lora_add_hashes_to_infotext: network_hashes = [] for item in networks.loaded_networks: @@ -44,8 +50,13 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): network_hashes.append(f"{alias}: {shorthash}") if network_hashes: p.extra_generation_params["Lora hashes"] = ", ".join(network_hashes) + shared.log.info(f'Applying LoRA: {names} patch={t1-t0:.2f}s load={t2-t1:.2f}s') + def deactivate(self, p): + networks.originals.undo() + if networks.debug: + shared.log.debug(f"LoRA timers: load={networks.timer['load']:.2f}s apply={networks.timer['apply']:.2f}s restore={networks.timer['restore']:.2f}s") if self.errors: p.comment("Networks with errors: " + ", ".join(f"{k} ({v})" for k, v in self.errors.items())) for k, v in self.errors.items(): diff --git a/extensions-builtin/Lora/lora_convert.py b/extensions-builtin/Lora/lora_convert.py new file mode 100644 index 000000000..5843c7ad8 --- /dev/null +++ b/extensions-builtin/Lora/lora_convert.py @@ -0,0 +1,208 @@ +from typing import Dict +import re +import bisect +from modules import shared + + +suffix_conversion = { + "attentions": {}, + "resnets": { + "conv1": "in_layers_2", + "conv2": "out_layers_3", + "norm1": "in_layers_0", + "norm2": "out_layers_0", + "time_emb_proj": "emb_layers_1", + "conv_shortcut": "skip_connection", + } +} +re_digits = re.compile(r"\d+") +re_x_proj = re.compile(r"(.*)_([qkv]_proj)$") +re_compiled = {} + + +def make_unet_conversion_map() -> Dict[str, str]: + unet_conversion_map_layer = [] + + for i in range(3): # num_blocks is 3 in sdxl + # loop over downblocks/upblocks + for j in range(2): + # loop over resnets/attentions for downblocks + hf_down_res_prefix = f"down_blocks.{i}.resnets.{j}." + sd_down_res_prefix = f"input_blocks.{3 * i + j + 1}.0." + unet_conversion_map_layer.append((sd_down_res_prefix, hf_down_res_prefix)) + if i < 3: + # no attention layers in down_blocks.3 + hf_down_atn_prefix = f"down_blocks.{i}.attentions.{j}." + sd_down_atn_prefix = f"input_blocks.{3 * i + j + 1}.1." + unet_conversion_map_layer.append((sd_down_atn_prefix, hf_down_atn_prefix)) + + for j in range(3): + # loop over resnets/attentions for upblocks + hf_up_res_prefix = f"up_blocks.{i}.resnets.{j}." + sd_up_res_prefix = f"output_blocks.{3 * i + j}.0." + unet_conversion_map_layer.append((sd_up_res_prefix, hf_up_res_prefix)) + # if i > 0: commentout for sdxl + # no attention layers in up_blocks.0 + hf_up_atn_prefix = f"up_blocks.{i}.attentions.{j}." + sd_up_atn_prefix = f"output_blocks.{3 * i + j}.1." + unet_conversion_map_layer.append((sd_up_atn_prefix, hf_up_atn_prefix)) + + if i < 3: + # no downsample in down_blocks.3 + hf_downsample_prefix = f"down_blocks.{i}.downsamplers.0.conv." + sd_downsample_prefix = f"input_blocks.{3 * (i + 1)}.0.op." + unet_conversion_map_layer.append((sd_downsample_prefix, hf_downsample_prefix)) + # no upsample in up_blocks.3 + hf_upsample_prefix = f"up_blocks.{i}.upsamplers.0." + sd_upsample_prefix = f"output_blocks.{3 * i + 2}.{2}." # change for sdxl + unet_conversion_map_layer.append((sd_upsample_prefix, hf_upsample_prefix)) + + hf_mid_atn_prefix = "mid_block.attentions.0." + sd_mid_atn_prefix = "middle_block.1." + unet_conversion_map_layer.append((sd_mid_atn_prefix, hf_mid_atn_prefix)) + + for j in range(2): + hf_mid_res_prefix = f"mid_block.resnets.{j}." + sd_mid_res_prefix = f"middle_block.{2 * j}." + unet_conversion_map_layer.append((sd_mid_res_prefix, hf_mid_res_prefix)) + + unet_conversion_map_resnet = [ + # (stable-diffusion, HF Diffusers) + ("in_layers.0.", "norm1."), + ("in_layers.2.", "conv1."), + ("out_layers.0.", "norm2."), + ("out_layers.3.", "conv2."), + ("emb_layers.1.", "time_emb_proj."), + ("skip_connection.", "conv_shortcut."), + ] + + unet_conversion_map = [] + for sd, hf in unet_conversion_map_layer: + if "resnets" in hf: + for sd_res, hf_res in unet_conversion_map_resnet: + unet_conversion_map.append((sd + sd_res, hf + hf_res)) + else: + unet_conversion_map.append((sd, hf)) + + for j in range(2): + hf_time_embed_prefix = f"time_embedding.linear_{j + 1}." + sd_time_embed_prefix = f"time_embed.{j * 2}." + unet_conversion_map.append((sd_time_embed_prefix, hf_time_embed_prefix)) + + for j in range(2): + hf_label_embed_prefix = f"add_embedding.linear_{j + 1}." + sd_label_embed_prefix = f"label_emb.0.{j * 2}." + unet_conversion_map.append((sd_label_embed_prefix, hf_label_embed_prefix)) + + unet_conversion_map.append(("input_blocks.0.0.", "conv_in.")) + unet_conversion_map.append(("out.0.", "conv_norm_out.")) + unet_conversion_map.append(("out.2.", "conv_out.")) + + sd_hf_conversion_map = {sd.replace(".", "_")[:-1]: hf.replace(".", "_")[:-1] for sd, hf in unet_conversion_map} + return sd_hf_conversion_map + + +class KeyConvert: + def __init__(self): + if shared.backend == shared.Backend.ORIGINAL: + self.converter = self.original + self.is_sd2 = 'model_transformer_resblocks' in shared.sd_model.network_layer_mapping + + else: + self.converter = self.diffusers + self.is_sdxl = True if shared.sd_model_type == "sdxl" else False + self.UNET_CONVERSION_MAP = make_unet_conversion_map() if self.is_sdxl else None + self.LORA_PREFIX_UNET = "lora_unet" + self.LORA_PREFIX_TEXT_ENCODER = "lora_te" + # SDXL: must starts with LORA_PREFIX_TEXT_ENCODER + self.LORA_PREFIX_TEXT_ENCODER1 = "lora_te1" + self.LORA_PREFIX_TEXT_ENCODER2 = "lora_te2" + + def original(self, key): + key = convert_diffusers_name_to_compvis(key, self.is_sd2) + sd_module = shared.sd_model.network_layer_mapping.get(key, None) + if sd_module is None: + m = re_x_proj.match(key) + if m: + sd_module = shared.sd_model.network_layer_mapping.get(m.group(1), None) + # SDXL loras seem to already have correct compvis keys, so only need to replace "lora_unet" with "diffusion_model" + if sd_module is None and "lora_unet" in key: + key = key.replace("lora_unet", "diffusion_model") + sd_module = shared.sd_model.network_layer_mapping.get(key, None) + elif sd_module is None and "lora_te1_text_model" in key: + key = key.replace("lora_te1_text_model", "0_transformer_text_model") + sd_module = shared.sd_model.network_layer_mapping.get(key, None) + # some SD1 Loras also have correct compvis keys + if sd_module is None: + key = key.replace("lora_te1_text_model", "transformer_text_model") + sd_module = shared.sd_model.network_layer_mapping.get(key, None) + return key, sd_module + + def diffusers(self, key): + if self.is_sdxl: + map_keys = list(self.UNET_CONVERSION_MAP.keys()) # prefix of U-Net modules + map_keys.sort() + search_key = key.replace(self.LORA_PREFIX_UNET + "_", "").replace(self.LORA_PREFIX_TEXT_ENCODER1 + "_", + "").replace( + self.LORA_PREFIX_TEXT_ENCODER2 + "_", "") + position = bisect.bisect_right(map_keys, search_key) + map_key = map_keys[position - 1] + if search_key.startswith(map_key): + key = key.replace(map_key, self.UNET_CONVERSION_MAP[map_key]) # pylint: disable=unsubscriptable-object + sd_module = shared.sd_model.network_layer_mapping.get(key, None) + return key, sd_module + + def __call__(self, key): + return self.converter(key) + + +def convert_diffusers_name_to_compvis(key, is_sd2): + def match(match_list, regex_text): + regex = re_compiled.get(regex_text) + if regex is None: + regex = re.compile(regex_text) + re_compiled[regex_text] = regex + r = re.match(regex, key) + if not r: + return False + match_list.clear() + match_list.extend([int(x) if re.match(re_digits, x) else x for x in r.groups()]) + return True + + m = [] + if match(m, r"lora_unet_conv_in(.*)"): + return f'diffusion_model_input_blocks_0_0{m[0]}' + if match(m, r"lora_unet_conv_out(.*)"): + return f'diffusion_model_out_2{m[0]}' + if match(m, r"lora_unet_time_embedding_linear_(\d+)(.*)"): + return f"diffusion_model_time_embed_{m[0] * 2 - 2}{m[1]}" + if match(m, r"lora_unet_down_blocks_(\d+)_(attentions|resnets)_(\d+)_(.+)"): + suffix = suffix_conversion.get(m[1], {}).get(m[3], m[3]) + return f"diffusion_model_input_blocks_{1 + m[0] * 3 + m[2]}_{1 if m[1] == 'attentions' else 0}_{suffix}" + if match(m, r"lora_unet_mid_block_(attentions|resnets)_(\d+)_(.+)"): + suffix = suffix_conversion.get(m[0], {}).get(m[2], m[2]) + return f"diffusion_model_middle_block_{1 if m[0] == 'attentions' else m[1] * 2}_{suffix}" + if match(m, r"lora_unet_up_blocks_(\d+)_(attentions|resnets)_(\d+)_(.+)"): + suffix = suffix_conversion.get(m[1], {}).get(m[3], m[3]) + return f"diffusion_model_output_blocks_{m[0] * 3 + m[2]}_{1 if m[1] == 'attentions' else 0}_{suffix}" + if match(m, r"lora_unet_down_blocks_(\d+)_downsamplers_0_conv"): + return f"diffusion_model_input_blocks_{3 + m[0] * 3}_0_op" + if match(m, r"lora_unet_up_blocks_(\d+)_upsamplers_0_conv"): + return f"diffusion_model_output_blocks_{2 + m[0] * 3}_{2 if m[0]>0 else 1}_conv" + if match(m, r"lora_te_text_model_encoder_layers_(\d+)_(.+)"): + if is_sd2: + if 'mlp_fc1' in m[1]: + return f"model_transformer_resblocks_{m[0]}_{m[1].replace('mlp_fc1', 'mlp_c_fc')}" + elif 'mlp_fc2' in m[1]: + return f"model_transformer_resblocks_{m[0]}_{m[1].replace('mlp_fc2', 'mlp_c_proj')}" + else: + return f"model_transformer_resblocks_{m[0]}_{m[1].replace('self_attn', 'attn')}" + return f"transformer_text_model_encoder_layers_{m[0]}_{m[1]}" + if match(m, r"lora_te2_text_model_encoder_layers_(\d+)_(.+)"): + if 'mlp_fc1' in m[1]: + return f"1_model_transformer_resblocks_{m[0]}_{m[1].replace('mlp_fc1', 'mlp_c_fc')}" + elif 'mlp_fc2' in m[1]: + return f"1_model_transformer_resblocks_{m[0]}_{m[1].replace('mlp_fc2', 'mlp_c_proj')}" + else: + return f"1_model_transformer_resblocks_{m[0]}_{m[1].replace('self_attn', 'attn')}" + return key diff --git a/extensions-builtin/Lora/lora_patches.py b/extensions-builtin/Lora/lora_patches.py index 5f6c97b71..993582b6b 100644 --- a/extensions-builtin/Lora/lora_patches.py +++ b/extensions-builtin/Lora/lora_patches.py @@ -15,6 +15,9 @@ class LoraPatches: self.LayerNorm_load_state_dict = patches.patch(__name__, torch.nn.LayerNorm, '_load_from_state_dict', networks.network_LayerNorm_load_state_dict) self.MultiheadAttention_forward = patches.patch(__name__, torch.nn.MultiheadAttention, 'forward', networks.network_MultiheadAttention_forward) self.MultiheadAttention_load_state_dict = patches.patch(__name__, torch.nn.MultiheadAttention, '_load_from_state_dict', networks.network_MultiheadAttention_load_state_dict) + networks.timer['load'] = 0 + networks.timer['apply'] = 0 + networks.timer['restore'] = 0 def undo(self): self.Linear_forward = patches.undo(__name__, torch.nn.Linear, 'forward') # pylint: disable=E1128 diff --git a/extensions-builtin/Lora/network.py b/extensions-builtin/Lora/network.py index 71deb6498..d7c158770 100644 --- a/extensions-builtin/Lora/network.py +++ b/extensions-builtin/Lora/network.py @@ -75,10 +75,7 @@ class NetworkOnDisk: def get_alias(self): import networks - if shared.opts.lora_preferred_name == "Filename" or self.alias.lower() in networks.forbidden_network_aliases: - return self.name - else: - return self.alias + return self.name if shared.opts.lora_preferred_name == "filename" or self.alias.lower() in networks.forbidden_network_aliases else self.alias class Network: # LoraModule diff --git a/extensions-builtin/Lora/networks.py b/extensions-builtin/Lora/networks.py index 174949f5c..8e4e01e52 100644 --- a/extensions-builtin/Lora/networks.py +++ b/extensions-builtin/Lora/networks.py @@ -1,8 +1,7 @@ -from typing import Dict, Union -import logging +from typing import Union import os import re -import bisect +import time import lora_patches import network import network_lora @@ -11,11 +10,24 @@ import network_ia3 import network_lokr import network_full import network_norm +import lora_convert import torch +import diffusers.models.lora from modules import shared, devices, sd_models, errors, scripts, sd_hijack -import diffusers.models.lora as diffusers_lora +debug = os.environ.get('SD_LORA_DEBUG', None) +originals: lora_patches.LoraPatches = None +extra_network_lora = None +available_networks = {} +available_network_aliases = {} +loaded_networks = [] +timer = { 'load': 0, 'apply': 0, 'restore': 0 } +# networks_in_memory = {} +lora_cache = {} +available_network_hash_lookup = {} +forbidden_network_aliases = {} +re_network_name = re.compile(r"(.*)\s*\([0-9a-fA-F]+\)") module_types = [ network_lora.ModuleTypeLora(), network_hada.ModuleTypeHada(), @@ -26,215 +38,6 @@ module_types = [ ] -re_digits = re.compile(r"\d+") -re_x_proj = re.compile(r"(.*)_([qkv]_proj)$") -re_compiled = {} - -suffix_conversion = { - "attentions": {}, - "resnets": { - "conv1": "in_layers_2", - "conv2": "out_layers_3", - "norm1": "in_layers_0", - "norm2": "out_layers_0", - "time_emb_proj": "emb_layers_1", - "conv_shortcut": "skip_connection", - } -} - - -def make_unet_conversion_map() -> Dict[str, str]: - unet_conversion_map_layer = [] - - for i in range(3): # num_blocks is 3 in sdxl - # loop over downblocks/upblocks - for j in range(2): - # loop over resnets/attentions for downblocks - hf_down_res_prefix = f"down_blocks.{i}.resnets.{j}." - sd_down_res_prefix = f"input_blocks.{3 * i + j + 1}.0." - unet_conversion_map_layer.append((sd_down_res_prefix, hf_down_res_prefix)) - - if i < 3: - # no attention layers in down_blocks.3 - hf_down_atn_prefix = f"down_blocks.{i}.attentions.{j}." - sd_down_atn_prefix = f"input_blocks.{3 * i + j + 1}.1." - unet_conversion_map_layer.append((sd_down_atn_prefix, hf_down_atn_prefix)) - - for j in range(3): - # loop over resnets/attentions for upblocks - hf_up_res_prefix = f"up_blocks.{i}.resnets.{j}." - sd_up_res_prefix = f"output_blocks.{3 * i + j}.0." - unet_conversion_map_layer.append((sd_up_res_prefix, hf_up_res_prefix)) - - # if i > 0: commentout for sdxl - # no attention layers in up_blocks.0 - hf_up_atn_prefix = f"up_blocks.{i}.attentions.{j}." - sd_up_atn_prefix = f"output_blocks.{3 * i + j}.1." - unet_conversion_map_layer.append((sd_up_atn_prefix, hf_up_atn_prefix)) - - if i < 3: - # no downsample in down_blocks.3 - hf_downsample_prefix = f"down_blocks.{i}.downsamplers.0.conv." - sd_downsample_prefix = f"input_blocks.{3 * (i + 1)}.0.op." - unet_conversion_map_layer.append((sd_downsample_prefix, hf_downsample_prefix)) - - # no upsample in up_blocks.3 - hf_upsample_prefix = f"up_blocks.{i}.upsamplers.0." - sd_upsample_prefix = f"output_blocks.{3 * i + 2}.{2}." # change for sdxl - unet_conversion_map_layer.append((sd_upsample_prefix, hf_upsample_prefix)) - - hf_mid_atn_prefix = "mid_block.attentions.0." - sd_mid_atn_prefix = "middle_block.1." - unet_conversion_map_layer.append((sd_mid_atn_prefix, hf_mid_atn_prefix)) - - for j in range(2): - hf_mid_res_prefix = f"mid_block.resnets.{j}." - sd_mid_res_prefix = f"middle_block.{2 * j}." - unet_conversion_map_layer.append((sd_mid_res_prefix, hf_mid_res_prefix)) - - unet_conversion_map_resnet = [ - # (stable-diffusion, HF Diffusers) - ("in_layers.0.", "norm1."), - ("in_layers.2.", "conv1."), - ("out_layers.0.", "norm2."), - ("out_layers.3.", "conv2."), - ("emb_layers.1.", "time_emb_proj."), - ("skip_connection.", "conv_shortcut."), - ] - - unet_conversion_map = [] - for sd, hf in unet_conversion_map_layer: - if "resnets" in hf: - for sd_res, hf_res in unet_conversion_map_resnet: - unet_conversion_map.append((sd + sd_res, hf + hf_res)) - else: - unet_conversion_map.append((sd, hf)) - - for j in range(2): - hf_time_embed_prefix = f"time_embedding.linear_{j + 1}." - sd_time_embed_prefix = f"time_embed.{j * 2}." - unet_conversion_map.append((sd_time_embed_prefix, hf_time_embed_prefix)) - - for j in range(2): - hf_label_embed_prefix = f"add_embedding.linear_{j + 1}." - sd_label_embed_prefix = f"label_emb.0.{j * 2}." - unet_conversion_map.append((sd_label_embed_prefix, hf_label_embed_prefix)) - - unet_conversion_map.append(("input_blocks.0.0.", "conv_in.")) - unet_conversion_map.append(("out.0.", "conv_norm_out.")) - unet_conversion_map.append(("out.2.", "conv_out.")) - - sd_hf_conversion_map = {sd.replace(".", "_")[:-1]: hf.replace(".", "_")[:-1] for sd, hf in unet_conversion_map} - return sd_hf_conversion_map - - -class KeyConvert: - def __init__(self): - if shared.backend == shared.Backend.ORIGINAL: - self.converter = self.original - self.is_sd2 = 'model_transformer_resblocks' in shared.sd_model.network_layer_mapping - - else: - self.converter = self.diffusers - self.is_sdxl = True if shared.sd_model_type == "sdxl" else False - self.UNET_CONVERSION_MAP = make_unet_conversion_map() if self.is_sdxl else None - self.LORA_PREFIX_UNET = "lora_unet" - self.LORA_PREFIX_TEXT_ENCODER = "lora_te" - - # SDXL: must starts with LORA_PREFIX_TEXT_ENCODER - self.LORA_PREFIX_TEXT_ENCODER1 = "lora_te1" - self.LORA_PREFIX_TEXT_ENCODER2 = "lora_te2" - - def original(self, key): - key = convert_diffusers_name_to_compvis(key, self.is_sd2) - sd_module = shared.sd_model.network_layer_mapping.get(key, None) - if sd_module is None: - m = re_x_proj.match(key) - if m: - sd_module = shared.sd_model.network_layer_mapping.get(m.group(1), None) - # SDXL loras seem to already have correct compvis keys, so only need to replace "lora_unet" with "diffusion_model" - if sd_module is None and "lora_unet" in key: - key = key.replace("lora_unet", "diffusion_model") - sd_module = shared.sd_model.network_layer_mapping.get(key, None) - elif sd_module is None and "lora_te1_text_model" in key: - key = key.replace("lora_te1_text_model", "0_transformer_text_model") - sd_module = shared.sd_model.network_layer_mapping.get(key, None) - # some SD1 Loras also have correct compvis keys - if sd_module is None: - key = key.replace("lora_te1_text_model", "transformer_text_model") - sd_module = shared.sd_model.network_layer_mapping.get(key, None) - return key, sd_module - - def diffusers(self, key): - if self.is_sdxl: - map_keys = list(self.UNET_CONVERSION_MAP.keys()) # prefix of U-Net modules - map_keys.sort() - search_key = key.replace(self.LORA_PREFIX_UNET + "_", "").replace(self.LORA_PREFIX_TEXT_ENCODER1 + "_", - "").replace( - self.LORA_PREFIX_TEXT_ENCODER2 + "_", "") - position = bisect.bisect_right(map_keys, search_key) - map_key = map_keys[position - 1] - if search_key.startswith(map_key): - key = key.replace(map_key, self.UNET_CONVERSION_MAP[map_key]) - sd_module = shared.sd_model.network_layer_mapping.get(key, None) - return key, sd_module - - def __call__(self, key): - return self.converter(key) - - -def convert_diffusers_name_to_compvis(key, is_sd2): - def match(match_list, regex_text): - regex = re_compiled.get(regex_text) - if regex is None: - regex = re.compile(regex_text) - re_compiled[regex_text] = regex - r = re.match(regex, key) - if not r: - return False - match_list.clear() - match_list.extend([int(x) if re.match(re_digits, x) else x for x in r.groups()]) - return True - - m = [] - if match(m, r"lora_unet_conv_in(.*)"): - return f'diffusion_model_input_blocks_0_0{m[0]}' - if match(m, r"lora_unet_conv_out(.*)"): - return f'diffusion_model_out_2{m[0]}' - if match(m, r"lora_unet_time_embedding_linear_(\d+)(.*)"): - return f"diffusion_model_time_embed_{m[0] * 2 - 2}{m[1]}" - if match(m, r"lora_unet_down_blocks_(\d+)_(attentions|resnets)_(\d+)_(.+)"): - suffix = suffix_conversion.get(m[1], {}).get(m[3], m[3]) - return f"diffusion_model_input_blocks_{1 + m[0] * 3 + m[2]}_{1 if m[1] == 'attentions' else 0}_{suffix}" - if match(m, r"lora_unet_mid_block_(attentions|resnets)_(\d+)_(.+)"): - suffix = suffix_conversion.get(m[0], {}).get(m[2], m[2]) - return f"diffusion_model_middle_block_{1 if m[0] == 'attentions' else m[1] * 2}_{suffix}" - if match(m, r"lora_unet_up_blocks_(\d+)_(attentions|resnets)_(\d+)_(.+)"): - suffix = suffix_conversion.get(m[1], {}).get(m[3], m[3]) - return f"diffusion_model_output_blocks_{m[0] * 3 + m[2]}_{1 if m[1] == 'attentions' else 0}_{suffix}" - if match(m, r"lora_unet_down_blocks_(\d+)_downsamplers_0_conv"): - return f"diffusion_model_input_blocks_{3 + m[0] * 3}_0_op" - if match(m, r"lora_unet_up_blocks_(\d+)_upsamplers_0_conv"): - return f"diffusion_model_output_blocks_{2 + m[0] * 3}_{2 if m[0]>0 else 1}_conv" - if match(m, r"lora_te_text_model_encoder_layers_(\d+)_(.+)"): - if is_sd2: - if 'mlp_fc1' in m[1]: - return f"model_transformer_resblocks_{m[0]}_{m[1].replace('mlp_fc1', 'mlp_c_fc')}" - elif 'mlp_fc2' in m[1]: - return f"model_transformer_resblocks_{m[0]}_{m[1].replace('mlp_fc2', 'mlp_c_proj')}" - else: - return f"model_transformer_resblocks_{m[0]}_{m[1].replace('self_attn', 'attn')}" - return f"transformer_text_model_encoder_layers_{m[0]}_{m[1]}" - if match(m, r"lora_te2_text_model_encoder_layers_(\d+)_(.+)"): - if 'mlp_fc1' in m[1]: - return f"1_model_transformer_resblocks_{m[0]}_{m[1].replace('mlp_fc1', 'mlp_c_fc')}" - elif 'mlp_fc2' in m[1]: - return f"1_model_transformer_resblocks_{m[0]}_{m[1].replace('mlp_fc2', 'mlp_c_proj')}" - else: - return f"1_model_transformer_resblocks_{m[0]}_{m[1].replace('self_attn', 'attn')}" - return key - - def assign_network_names_to_compvis_modules(sd_model): """ if shared.sd_model.is_sdxl: @@ -282,14 +85,19 @@ def assign_network_names_to_compvis_modules(sd_model): def load_network(name, network_on_disk): + t0 = time.time() + cached = lora_cache.get(name, None) + if debug: + shared.log.debug(f'LoRA load: name={name} file={network_on_disk.filename} {"cached" if cached else ""}') + if cached is not None: + return cached net = network.Network(name, network_on_disk) net.mtime = os.path.getmtime(network_on_disk.filename) sd = sd_models.read_state_dict(network_on_disk.filename) - # this should not be needed but is here as an emergency fix for an unknown error people are experiencing in 1.2.0 - assign_network_names_to_compvis_modules(shared.sd_model) + assign_network_names_to_compvis_modules(shared.sd_model) # this should not be needed but is here as an emergency fix for an unknown error people are experiencing in 1.2.0 keys_failed_to_match = {} matched_networks = {} - convert = KeyConvert() + convert = lora_convert.KeyConvert() for key_network, weight in sd.items(): key_network_without_network_parts, network_part = key_network.split(".", 1) key, sd_module = convert(key_network_without_network_parts) @@ -309,28 +117,21 @@ def load_network(name, network_on_disk): raise AssertionError(f"Could not find a module type (out of {', '.join([x.__class__.__name__ for x in module_types])}) that would accept those keys: {', '.join(weights.w)}") net.modules[key] = net_module if keys_failed_to_match: - logging.debug(f"Network {network_on_disk.filename} didn't match keys: {keys_failed_to_match}") + shared.log.warning(f"LoRA unmatched keys: file={network_on_disk.filename} keys={len(keys_failed_to_match)}") + if debug: + shared.log.debug(f"LoRA unmatched keys: file={network_on_disk.filename} keys={keys_failed_to_match}") + lora_cache[name] = net + t1 = time.time() + timer['load'] += t1 - t0 return net -def purge_networks_from_memory(): - while len(networks_in_memory) > shared.opts.lora_in_memory_limit and len(networks_in_memory) > 0: - name = next(iter(networks_in_memory)) - networks_in_memory.pop(name, None) - devices.torch_gc() - def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None): - already_loaded = {} - for net in loaded_networks: - if net.name in names: - already_loaded[net.name] = net - loaded_networks.clear() networks_on_disk = [available_network_aliases.get(name, None) for name in names] if any(x is None for x in networks_on_disk): list_available_networks() networks_on_disk = [available_network_aliases.get(name, None) for name in names] failed_to_load_networks = [] - recompile_model = False if shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx": if len(names) == len(shared.compiled_model_state.lora_model): @@ -346,25 +147,21 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No shared.opts.cuda_compile = False sd_models.reload_model_weights(op='model') shared.opts.cuda_compile = True - + loaded_networks.clear() for i, (network_on_disk, name) in enumerate(zip(networks_on_disk, names)): - net = already_loaded.get(name, None) if network_on_disk is not None: - if net is None: - net = networks_in_memory.get(name) - if net is None or os.path.getmtime(network_on_disk.filename) > net.mtime: - try: - net = load_network(name, network_on_disk) - networks_in_memory.pop(name, None) - networks_in_memory[name] = net - except Exception as e: - errors.display(e, f"loading network {network_on_disk.filename}") - continue + try: + net = load_network(name, network_on_disk) + except Exception as e: + shared.log.error(f"LoRA load failed: file={network_on_disk.filename}") + if debug: + errors.display(e, f"LoRA load failed file={network_on_disk.filename}") + continue net.mentioned_name = name network_on_disk.read_hash() if net is None: failed_to_load_networks.append(name) - logging.info(f"Couldn't find network with name {name}") + shared.log.error(f"LoRA unknown network: file={network_on_disk.filename} network={name}") continue net.te_multiplier = te_multipliers[i] if te_multipliers else 1.0 net.unet_multiplier = unet_multipliers[i] if unet_multipliers else 1.0 @@ -372,25 +169,33 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No loaded_networks.append(net) if failed_to_load_networks: sd_hijack.model_hijack.comments.append("Networks not found: " + ", ".join(failed_to_load_networks)) - purge_networks_from_memory() + + while len(lora_cache) > shared.opts.lora_in_memory_limit: + name = next(iter(lora_cache)) + lora_cache.pop(name, None) + if debug: + shared.log.debug(f'LoRA cache: {list(lora_cache)}') + devices.torch_gc() if recompile_model: - shared.log.info("Networks: Recompiling model") + shared.log.info("LoRA recompiling model") sd_models.compile_diffusers(shared.sd_model) -def network_restore_weights_from_backup(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, torch.nn.MultiheadAttention, diffusers_lora.LoRACompatibleLinear, diffusers_lora.LoRACompatibleConv]): +def network_restore_weights_from_backup(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, torch.nn.MultiheadAttention, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv]): + t0 = time.time() weights_backup = getattr(self, "network_weights_backup", None) bias_backup = getattr(self, "network_bias_backup", None) if weights_backup is None and bias_backup is None: return + # if debug: + # shared.log.debug('LoRA restore weights') if weights_backup is not None: if isinstance(self, torch.nn.MultiheadAttention): self.in_proj_weight.copy_(weights_backup[0]) self.out_proj.weight.copy_(weights_backup[1]) else: self.weight.copy_(weights_backup) - if bias_backup is not None: if isinstance(self, torch.nn.MultiheadAttention): self.out_proj.bias.copy_(bias_backup) @@ -401,9 +206,11 @@ def network_restore_weights_from_backup(self: Union[torch.nn.Conv2d, torch.nn.Li self.out_proj.bias = None else: self.bias = None + t1 = time.time() + timer['restore'] += t1 - t0 -def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, torch.nn.MultiheadAttention, diffusers_lora.LoRACompatibleLinear, diffusers_lora.LoRACompatibleConv]): +def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, torch.nn.MultiheadAttention, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv]): """ Applies the currently selected set of networks to the weights of torch layer self. If weights already have this particular set of networks applied, does nothing. @@ -412,6 +219,7 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn network_layer_name = getattr(self, 'network_layer_name', None) if network_layer_name is None: return + t0 = time.time() current_names = getattr(self, "network_current_names", ()) wanted_names = tuple((x.name, x.te_multiplier, x.unet_multiplier, x.dyn_dim) for x in loaded_networks) weights_backup = getattr(self, "network_weights_backup", None) @@ -432,6 +240,7 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn else: bias_backup = None self.network_bias_backup = bias_backup + if current_names != wanted_names: network_restore_weights_from_backup(self) for net in loaded_networks: @@ -442,7 +251,7 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn updown, ex_bias = module.calc_updown(self.weight) if len(self.weight.shape) == 4 and self.weight.shape[1] == 9: # inpainting model. zero pad updown to make channel[1] 4 to 9 - updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) + updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) # pylint: disable=not-callable self.weight += updown if ex_bias is not None and hasattr(self, 'bias'): if self.bias is None: @@ -450,7 +259,8 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn else: self.bias += ex_bias except RuntimeError as e: - logging.debug(f"Network {net.name} layer {network_layer_name}: {e}") + if debug: + shared.log.debug(f"LoRA apply weight network={net.name} layer={network_layer_name} {e}") extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 continue module_q = net.modules.get(network_layer_name + "_q_proj", None) @@ -473,14 +283,17 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn else: self.out_proj.bias += ex_bias except RuntimeError as e: - logging.debug(f"Network {net.name} layer {network_layer_name}: {e}") + if debug: + shared.log.debug(f"LoRA network={net.name} layer={network_layer_name} {e}") extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 continue if module is None: continue - logging.debug(f"Network {net.name} layer {network_layer_name}: couldn't find supported operation") + shared.log.warning(f"LoRA network={net.name} layer={network_layer_name} unsupported operation") extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 self.network_current_names = wanted_names + t1 = time.time() + timer['apply'] += t1 - t0 def network_forward(module, input, original_forward): # pylint: disable=W0622 @@ -590,8 +403,6 @@ def list_available_networks(): available_network_aliases[name] = entry available_network_aliases[entry.alias] = entry -re_network_name = re.compile(r"(.*)\s*\([0-9a-fA-F]+\)") - def infotext_pasted(infotext, params): # pylint: disable=W0613 if "AddNet Module 1" in [x[1] for x in scripts.scripts_txt2img.infotext_fields]: @@ -615,12 +426,4 @@ def infotext_pasted(infotext, params): # pylint: disable=W0613 params["Prompt"] += "\n" + "".join(added) -originals: lora_patches.LoraPatches = None -extra_network_lora = None -available_networks = {} -available_network_aliases = {} -loaded_networks = [] -networks_in_memory = {} -available_network_hash_lookup = {} -forbidden_network_aliases = {} list_available_networks() diff --git a/extensions-builtin/Lora/scripts/lora_script.py b/extensions-builtin/Lora/scripts/lora_script.py index b2803bf32..bf7e9e2be 100644 --- a/extensions-builtin/Lora/scripts/lora_script.py +++ b/extensions-builtin/Lora/scripts/lora_script.py @@ -4,14 +4,14 @@ from fastapi import FastAPI import network import networks import lora # noqa:F401 # pylint: disable=unused-import -import lora_patches +# import lora_patches import extra_networks_lora import ui_extra_networks_lora from modules import script_callbacks, ui_extra_networks, extra_networks, shared -def unload(): - networks.originals.undo() +# def unload(): +# networks.originals.undo() def before_ui(): @@ -21,25 +21,17 @@ def before_ui(): # extra_networks.register_extra_network_alias(networks.extra_network_lora, "lyco") -networks.originals = lora_patches.LoraPatches() +# networks.originals = lora_patches.LoraPatches() script_callbacks.on_model_loaded(networks.assign_network_names_to_compvis_modules) -script_callbacks.on_script_unloaded(unload) +# script_callbacks.on_script_unloaded(unload) script_callbacks.on_before_ui(before_ui) script_callbacks.on_infotext_pasted(networks.infotext_pasted) shared.options_templates.update(shared.options_section(('extra_networks', "Extra Networks"), { "sd_lora": shared.OptionInfo("None", "Add network to prompt", gr.Dropdown, lambda: {"choices": ["None", *networks.available_networks], "visible": False}, refresh=networks.list_available_networks), - "lora_preferred_name": shared.OptionInfo("Alias from file", "When adding to prompt, refer to Lora by", gr.Radio, {"choices": ["Alias from file", "Filename"]}), - "lora_add_hashes_to_infotext": shared.OptionInfo(True, "Add Lora hashes to infotext"), # "lora_show_all": shared.OptionInfo(False, "Always show all networks on the Lora page").info("otherwise, those detected as for incompatible version of Stable Diffusion will be hidden"), # "lora_hide_unknown_for_versions": shared.OptionInfo([], "Hide networks of unknown versions for model versions", gr.CheckboxGroup, {"choices": ["SD1", "SD2", "SDXL"]}), - "lora_in_memory_limit": shared.OptionInfo(0, "Lora in-memory cache", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1}), -})) - - -shared.options_templates.update(shared.options_section(('compatibility', "Compatibility"), { - "lora_functional": shared.OptionInfo(False, "Lora/Networks: use old method that takes longer when you have multiple Loras active and produces same results as kohya-ss/sd-webui-additional-networks extension"), })) @@ -85,4 +77,3 @@ def infotext_pasted(infotext, d): # pylint: disable=unused-argument script_callbacks.on_infotext_pasted(infotext_pasted) -shared.opts.onchange("lora_in_memory_limit", networks.purge_networks_from_memory) diff --git a/modules/processing.py b/modules/processing.py index 467cae2e2..453d2b033 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -22,6 +22,7 @@ import modules.lowvram import modules.masking import modules.paths import modules.scripts +import modules.script_callbacks import modules.prompt_parser import modules.extra_networks import modules.face_restoration @@ -364,8 +365,7 @@ class Processed: return self.token_merging_ratio_hr if for_hr else self.token_merging_ratio -# from https://discuss.pytorch.org/t/help-regarding-slerp-function-for-generative-model-sampling/32475/3 -def slerp(val, low, high): +def slerp(val, low, high): # from https://discuss.pytorch.org/t/help-regarding-slerp-function-for-generative-model-sampling/32475/3 low_norm = low/torch.norm(low, dim=1, keepdim=True) high_norm = high/torch.norm(high, dim=1, keepdim=True) dot = (low_norm*high_norm).sum(1) @@ -382,7 +382,6 @@ def slerp(val, low, high): def create_random_tensors(shape, seeds, subseeds=None, subseed_strength=0.0, seed_resize_from_h=0, seed_resize_from_w=0, p=None): eta_noise_seed_delta = shared.opts.eta_noise_seed_delta or 0 xs = [] - # if we have multiple seeds, this means we are working with batch size>1; this then # enables the generation of additional tensors with noise that the sampler will use during its processing. # Using those pre-generated tensors instead of simple torch.randn allows a batch with seeds [100, 101] to @@ -391,24 +390,19 @@ def create_random_tensors(shape, seeds, subseeds=None, subseed_strength=0.0, see sampler_noises = [[] for _ in range(p.sampler.number_of_needed_noises(p))] else: sampler_noises = None - for i, seed in enumerate(seeds): noise_shape = shape if seed_resize_from_h <= 0 or seed_resize_from_w <= 0 else (shape[0], seed_resize_from_h//8, seed_resize_from_w//8) - subnoise = None if subseeds is not None: subseed = 0 if i >= len(subseeds) else subseeds[i] subnoise = devices.randn(subseed, noise_shape) - # randn results depend on device; gpu and cpu get different results for same seed; # the way I see it, it's better to do this on CPU, so that everyone gets same result; # but the original script had it like this, so I do not dare change it for now because # it will break everyone's seeds. noise = devices.randn(seed, noise_shape) - if subnoise is not None: noise = slerp(subseed_strength, noise, subnoise) - if noise_shape != shape: x = devices.randn(seed, shape) dx = (shape[2] - noise_shape[2]) // 2 @@ -421,19 +415,15 @@ def create_random_tensors(shape, seeds, subseeds=None, subseed_strength=0.0, see dy = max(-dy, 0) x[:, ty:ty+h, tx:tx+w] = noise[:, dy:dy+h, dx:dx+w] noise = x - if sampler_noises is not None: cnt = p.sampler.number_of_needed_noises(p) if eta_noise_seed_delta > 0: torch.manual_seed(seed + eta_noise_seed_delta) for j in range(cnt): sampler_noises[j].append(devices.randn_without_seed(tuple(noise_shape))) - xs.append(noise) - if sampler_noises is not None: p.sampler.sampler_noises = [torch.stack(n).to(shared.device) for n in sampler_noises] - x = torch.stack(xs).to(shared.device) return x @@ -532,10 +522,9 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts, all_seeds, all_su args["Face restoration"] = shared.opts.face_restoration_model if 'color' in p.ops: args["Color correction"] = True - + # embeddings if hasattr(modules.sd_hijack.model_hijack, 'embedding_db') and len(modules.sd_hijack.model_hijack.embedding_db.embeddings_used) > 0: # this is for original hijaacked models only, diffusers are handled separately args["Embeddings"] = ', '.join(modules.sd_hijack.model_hijack.embedding_db.embeddings_used) - # samplers args["Sampler ENSD"] = shared.opts.eta_noise_seed_delta if shared.opts.eta_noise_seed_delta != 0 and modules.sd_samplers_common.is_sampler_using_eta_noise_seed_delta(p) else None args["Sampler ENSM"] = p.initial_noise_multiplier if getattr(p, 'initial_noise_multiplier', 1.0) != 1.0 else None @@ -645,6 +634,8 @@ def process_images(p: StableDiffusionProcessing) -> Processed: modules.sd_models.apply_token_merging(p.sd_model, p.get_token_merging_ratio()) modules.sd_hijack_freeu.apply_freeu(p, shared.backend == shared.Backend.ORIGINAL) + modules.script_callbacks.before_process_callback(p) + if shared.cmd_opts.profile: """ import torch.profiler # pylint: disable=redefined-outer-name @@ -656,7 +647,8 @@ def process_images(p: StableDiffusionProcessing) -> Processed: import cProfile pr = cProfile.Profile() pr.enable() - res = process_images_inner(p) + with context_hypertile_vae(p), context_hypertile_unet(p): + res = process_images_inner(p) print_profile(pr, 'Torch') else: with context_hypertile_vae(p), context_hypertile_unet(p): @@ -664,6 +656,7 @@ def process_images(p: StableDiffusionProcessing) -> Processed: finally: if not shared.opts.cuda_compile: modules.sd_models.apply_token_merging(p.sd_model, 0) + modules.script_callbacks.after_process_callback(p) if p.override_settings_restore_afterwards: # restore opts to original state for k, v in stored_opts.items(): setattr(shared.opts, k, v) diff --git a/modules/script_callbacks.py b/modules/script_callbacks.py index 28b906aaf..299107115 100644 --- a/modules/script_callbacks.py +++ b/modules/script_callbacks.py @@ -92,6 +92,8 @@ class ImageGridLoopParams: ScriptCallback = namedtuple("ScriptCallback", ["script", "callback"]) callback_map = dict( callbacks_app_started=[], + callbacks_before_process=[], + callbacks_after_process=[], callbacks_model_loaded=[], callbacks_ui_tabs=[], callbacks_ui_train_tabs=[], @@ -134,6 +136,26 @@ def app_started_callback(demo: Optional[Blocks], app: FastAPI): report_exception(e, c, 'app_started_callback') +def before_process_callback(p): + for c in callback_map['callbacks_before_process']: + try: + t0 = time.time() + c.callback(p) + timer(t0, c.script, 'before_process') + except Exception as e: + report_exception(e, c, 'before_process_callback') + + +def after_process_callback(p): + for c in callback_map['callbacks_after_process']: + try: + t0 = time.time() + c.callback(p) + timer(t0, c.script, 'after_process') + except Exception as e: + report_exception(e, c, 'after_process_callback') + + def app_reload_callback(): for c in callback_map['callbacks_on_reload']: try: @@ -334,6 +356,16 @@ def on_app_started(callback): add_callback(callback_map['callbacks_app_started'], callback) +def on_before_process(callback): + """register a function to be called just before processing starts""" + add_callback(callback_map['callbacks_before_process'], callback) + + +def on_after_process(callback): + """register a function to be called just after processing ends""" + add_callback(callback_map['callbacks_after_process'], callback) + + def on_before_reload(callback): """register a function to be called just before the server reloads.""" add_callback(callback_map['callbacks_on_reload'], callback) diff --git a/modules/shared.py b/modules/shared.py index 40677344a..656dc9671 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -697,18 +697,27 @@ options_templates.update(options_section(('interrogate', "Interrogate"), { })) options_templates.update(options_section(('extra_networks', "Extra Networks"), { + "extra_networks_sep1": OptionInfo("

Extra networks UI

", "", gr.HTML), "extra_networks": OptionInfo(["All"], "Extra networks", ui_components.DropdownMulti, lambda: {"choices": ['All'] + [en.title for en in extra_networks]}), - "extra_networks_styles": OptionInfo(True, "Show built-in styles"), "extra_networks_card_cover": OptionInfo("sidebar", "UI position", gr.Radio, lambda: {"choices": ["cover", "inline", "sidebar"]}), "extra_networks_height": OptionInfo(53, "UI height (%)", gr.Slider, {"minimum": 10, "maximum": 100, "step": 1}), "extra_networks_sidebar_width": OptionInfo(35, "UI sidebar width (%)", gr.Slider, {"minimum": 10, "maximum": 80, "step": 1}), "extra_networks_card_size": OptionInfo(160, "UI card size (px)", gr.Slider, {"minimum": 20, "maximum": 2000, "step": 1}), "extra_networks_card_square": OptionInfo(True, "UI disable variable aspect ratio"), "extra_networks_card_fit": OptionInfo("cover", "UI image contain method", gr.Radio, lambda: {"choices": ["contain", "cover", "fill"], "visible": False}), + + "extra_networks_sep2": OptionInfo("

Extra networks general

", "", gr.HTML), "extra_network_skip_indexing": OptionInfo(False, "Do not automatically build extra network pages", gr.Checkbox), + "extra_networks_default_multiplier": OptionInfo(1.0, "Default multiplier for extra networks", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), + + "extra_networks_sep3": OptionInfo("

Extra networks settings

", "", gr.HTML), + "extra_networks_styles": OptionInfo(True, "Show built-in styles"), + "lora_preferred_name": OptionInfo("filename", "LoRA preffered name", gr.Radio, {"choices": ["filename", "alias"]}), + "lora_add_hashes_to_infotext": OptionInfo(True, "LoRA add hash info"), + "lora_in_memory_limit": OptionInfo(0, "LoRA memory cache", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1}), "lyco_patch_lora": OptionInfo(False, "Use LyCoris handler for all LoRA types", gr.Checkbox, { "visible": False }), "lora_functional": OptionInfo(False, "Use Kohya method for handling multiple LoRA", gr.Checkbox, { "visible": False }), - "extra_networks_default_multiplier": OptionInfo(1.0, "Default multiplier for extra networks", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), + "sd_hypernetwork": OptionInfo("None", "Add hypernetwork to prompt", gr.Dropdown, lambda: { "choices": ["None"] + list(hypernetworks.keys()), "visible": False }, refresh=reload_hypernetworks), })) diff --git a/modules/styles.py b/modules/styles.py index c5682a4dc..2b637800a 100644 --- a/modules/styles.py +++ b/modules/styles.py @@ -60,7 +60,7 @@ def apply_styles_to_extra(p, style: Style): v = type(orig)(v) setattr(p, k, v) fields.append(f'{k}={v}') - log.debug(f'Applied style: {style.name} extra={fields}') + log.info(f'Applying style: name={style.name} extra={fields}') class StyleDatabase: