diff --git a/modules/detailer/detailer.py b/modules/detailer/detailer.py index c7a6ef7ac..3e02ea991 100644 --- a/modules/detailer/detailer.py +++ b/modules/detailer/detailer.py @@ -247,7 +247,7 @@ class Detailer(): pc.disable_extra_networks = True # disable processing_diffusers from handling network activation since its handled here network_same = len(p.network_data.values()) == len(pc.network_data.values()) and all(x == y for x, y in zip(p.network_data.values(), pc.network_data.values())) if not network_same: - extra_networks.activate(pc, pc.network_data) + extra_networks.activate_filtered(pc, pc.network_data) log.debug(f'Detail: model="{i+1}:{name}" item={j+1}/{len(items)} box={item.box} label="{item.label}" score={item.score:.2f} seg={detailer_opt(p, "detailer_segmentation")} network={network_same} prompt="{pc.prompt}"') pc.init_images = [image] pc.image_mask = [item.mask] diff --git a/modules/extra_networks.py b/modules/extra_networks.py index 89acf5a4e..0737f68f8 100644 --- a/modules/extra_networks.py +++ b/modules/extra_networks.py @@ -121,6 +121,15 @@ def activate(p: StableDiffusionProcessing, extra_network_data: defaultdict[str, p.network_data = extra_network_data +def activate_filtered(p: StableDiffusionProcessing, extra_network_data: defaultdict[str, list[ExtraNetworkParams]] | None = None, step=0): + """activate with text encoder components gated on lora_apply_te; must run before prompt encode so te networks affect embeds""" + apply_te = getattr(p, 'lora_apply_te', None) + if apply_te is None: + apply_te = shared.opts.lora_apply_te + exclude = [] if apply_te else ['text_encoder', 'text_encoder_2', 'text_encoder_3'] + activate(p, extra_network_data, step=step, exclude=exclude) + + def deactivate(p: StableDiffusionProcessing, extra_network_data: defaultdict[str, list[ExtraNetworkParams]] | None = None, force: bool | None = None): """call deactivate for extra networks in extra_network_data in specified order, then call deactivate for all remaining registered networks""" if p.disable_extra_networks: diff --git a/modules/face/faceid.py b/modules/face/faceid.py index b2263d840..4c63bcd66 100644 --- a/modules/face/faceid.py +++ b/modules/face/faceid.py @@ -216,7 +216,7 @@ def face_id( p.subseeds = p.all_subseeds[n * p.batch_size:(n+1) * p.batch_size] p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts, p.network_data) - extra_networks.activate(p, p.network_data) + extra_networks.activate_filtered(p, p.network_data) ip_model_dict.update({ "prompt": p.prompts[0], "negative_prompt": p.negative_prompts[0], diff --git a/modules/lora/extra_networks_lora.py b/modules/lora/extra_networks_lora.py index e0d5dd984..f17d357aa 100644 --- a/modules/lora/extra_networks_lora.py +++ b/modules/lora/extra_networks_lora.py @@ -191,11 +191,13 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): key = f'include={",".join(include)}:exclude={",".join(exclude)}' loaded = sd_model.loaded_loras.get(key, []) if len(requested) != len(loaded): + sd_model.loaded_loras.clear() # single-entry cache: any activation invalidates state recorded under other filter keys sd_model.loaded_loras[key] = requested debug_log(f'Network check: type=LoRA key="{key}" requested={requested} loaded={loaded} status="num changed"') return True, "num changed" for req, load in zip(requested, loaded, strict=False): if req != load: + sd_model.loaded_loras.clear() sd_model.loaded_loras[key] = requested debug_log(f'Network check: type=LoRA key="{key}" requested={requested} loaded={loaded} status="content changed"') return True, "content changed" @@ -237,7 +239,7 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): has_changed = lora_nunchaku.load_nunchaku(names, unet_multipliers) else: # native - lora_load.network_load(names, te_multipliers, unet_multipliers, dyn_dims) # load + lora_load.network_load(names, te_multipliers, unet_multipliers, dyn_dims, activate=False) # load only, activation below honors include/exclude has_changed, reason = self.changed(requested, include, exclude) if has_changed: jobid = shared.state.begin('LoRA') diff --git a/modules/lora/lora_load.py b/modules/lora/lora_load.py index 604b24342..9f719e9e1 100644 --- a/modules/lora/lora_load.py +++ b/modules/lora/lora_load.py @@ -263,7 +263,7 @@ def gather_networks(names): return networks_on_disk -def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None, lora_modules=None): +def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None, lora_modules=None, activate=True): networks_on_disk = gather_networks(names) failed_to_load_networks = [] recompile_model, skip_lora_load = maybe_recompile_model(names, te_multipliers) @@ -342,9 +342,10 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non # Activate native modules loaded via diffusers path (e.g., LoKR on Flux2) # Also restore backed-up weights when previously active native modules are removed + # Callers that run their own deactivate/activate sequence pass activate=False from modules.lora import networks native_nets = [net for net in l.loaded_networks if len(net.modules) > 0] - if native_nets or networks.native_active: + if activate and (native_nets or networks.native_active): networks.network_activate() if len(l.loaded_networks) > 0 and l.debug: diff --git a/modules/lora/networks.py b/modules/lora/networks.py index fe84f3b8a..53bfd1606 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -28,11 +28,13 @@ def network_activate(include=None, exclude=None): modules = {} components = include if len(include) > 0 else default_components components = [x for x in components if x not in exclude] + filtered_components = [x for x in default_components if x not in components] # filtered components restore to backup so a filter means detached, not frozen with stale weights active_components = [] - for name in components: + for name in components + filtered_components: component = getattr(sd_model, name, None) if component is not None and hasattr(component, 'named_modules'): - active_components.append(name) + if name in components: + active_components.append(name) modules[name] = list(component.named_modules()) total = sum(len(x) for x in modules.values()) if len(l.loaded_networks) > 0: @@ -48,16 +50,25 @@ def network_activate(include=None, exclude=None): applied_layers.clear() backup_size = 0 for component in modules.keys(): + component_wanted = wanted_names if component in components else () device = getattr(sd_model, component, None).device for _, module in modules[component]: network_layer_name = getattr(module, 'network_layer_name', None) current_names = getattr(module, "network_current_names", ()) - if getattr(module, 'weight', None) is None or shared.state.interrupted or (network_layer_name is None) or (current_names == wanted_names): + if getattr(module, 'weight', None) is None or shared.state.interrupted or (network_layer_name is None) or (current_names == component_wanted): if task is not None: pbar.update(task, advance=1) continue - backup_size += network_backup_weights(module, network_layer_name, wanted_names) - batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name, elimit=elimit) + backup_size += network_backup_weights(module, network_layer_name, component_wanted) + if component_wanted == (): + weights_backup = getattr(module, "network_weights_backup", None) + if weights_backup is None or isinstance(weights_backup, bool): # fuse mode has no tensor backup, restore stays with network_deactivate + if task is not None: + pbar.update(task, advance=1) + continue + batch_updown, batch_ex_bias = None, None # restore-only pass, apply with no weights reverts to backup + else: + batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name, elimit=elimit) if shared.opts.lora_fuse_native: network_apply_direct(module, batch_updown, batch_ex_bias, device=device) else: @@ -68,7 +79,7 @@ def network_activate(include=None, exclude=None): applied_bias += 1 if batch_ex_bias is not None else 0 batch_updown, batch_ex_bias = None, None del batch_updown, batch_ex_bias - module.network_current_names = wanted_names + module.network_current_names = component_wanted if task is not None: bs = round(backup_size/1024/1024/1024, 2) if backup_size > 0 else None pbar.update(task, advance=1, description=f'networks={len(l.loaded_networks)} modules={active_components} layers={total} weights={applied_weight} bias={applied_bias} backup={bs} device={device}') diff --git a/modules/processing_args.py b/modules/processing_args.py index 36872e2e5..eca6b4ce5 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -6,7 +6,7 @@ import inspect import torch import numpy as np from PIL import Image -from modules import shared, sd_models, processing, processing_vae, processing_helpers, sd_hijack_hypertile, extra_networks, sd_vae +from modules import shared, sd_models, processing, processing_vae, processing_helpers, sd_hijack_hypertile, sd_vae from modules.logger import log from modules.processing_callbacks import diffusers_callback_legacy, diffusers_callback, set_callbacks_p from modules.processing_helpers import get_generator, apply_circular # pylint: disable=unused-import @@ -241,9 +241,6 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:l else: args['clip_skip'] = clip_skip - 1 - if shared.opts.lora_apply_te: - extra_networks.activate(p, include=['text_encoder', 'text_encoder_2', 'text_encoder_3']) - if 'complex_human_instruction' in possible: chi = shared.opts.te_complex_human_instruction p.extra_generation_params["CHI"] = chi diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index e22ae2d79..079868382 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -147,6 +147,8 @@ def process_base(p: processing.StableDiffusionProcessing): desc = 'Base' if 'detailer' in p.ops: desc = 'Detail' + p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts, p.network_data) + extra_networks.activate_filtered(p) # networks must patch weights before prompt encode so te loras affect embeds base_args = set_pipeline_args( p=p, model=shared.sd_model, @@ -176,9 +178,6 @@ def process_base(p: processing.StableDiffusionProcessing): modelstats.analyze() try: t0 = time.time() - p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts, p.network_data) - extra_networks.activate(p, exclude=['text_encoder', 'text_encoder_2', 'text_encoder_3']) - if hasattr(shared.sd_model, 'tgate') and getattr(p, 'gate_step', -1) > 0: base_args['gate_step'] = p.gate_step output = shared.sd_model.tgate(**base_args) # pylint: disable=not-callable @@ -311,7 +310,7 @@ def process_hires(p: processing.StableDiffusionProcessing, output): prompts, p.network_data = extra_networks.parse_prompts(prompts) reset_prompts = True if reset_prompts or ('base' in p.skip): - extra_networks.activate(p) + extra_networks.activate_filtered(p) hires_args = set_pipeline_args( p=p, diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 2025a4ca6..0c69c34d7 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -130,8 +130,11 @@ class PromptEmbedder: # unpack EN data in case of TE LoRA en_data = p.network_data en_data = [idx.items for item in en_data.values() for idx in item] + apply_te = getattr(p, 'lora_apply_te', None) + if apply_te is None: + apply_te = shared.opts.lora_apply_te effective_batch = 1 if self.allsame else self.batchsize - key = str([self.prompts, self.negative_prompts, effective_batch, self.clip_skip, self.steps, en_data]) + key = str([self.prompts, self.negative_prompts, effective_batch, self.clip_skip, self.steps, en_data, apply_te]) item = cache.get(key) if not item: if not any(flatten(emb) for emb in [self.prompt_embeds,