From 4554b9a2779604ef8507c74c7be7d6b235dc8f2f Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Wed, 8 Jul 2026 03:06:04 +0100 Subject: [PATCH] fix(lora): apply te networks before encode and honor lora_apply_te Network activation ran after prompt encoding, so text encoder lora weights never affected embeds on the first generation and the stale result was then served from the embed cache. The trailing unfiltered activate in network_load also overrode the te exclude filter, so the lora_apply_te setting was never honored. - parse and activate networks in process_base before pipeline args are built - activate_filtered gates text encoder components on per-request or global lora_apply_te; used by base, hires, detailer and faceid call sites - network_load accepts activate=False for callers that run their own deactivate/activate sequence with include/exclude - network_activate walks excluded components in restore-only mode so a filtered text encoder reverts to backup instead of keeping stale deltas - loaded_loras cache is single-entry since per-filter entries go stale when the setting toggles - prompt embed cache key includes the effective lora_apply_te value --- modules/detailer/detailer.py | 2 +- modules/extra_networks.py | 9 +++++++++ modules/face/faceid.py | 2 +- modules/lora/extra_networks_lora.py | 4 +++- modules/lora/lora_load.py | 5 +++-- modules/lora/networks.py | 23 +++++++++++++++++------ modules/processing_args.py | 5 +---- modules/processing_diffusers.py | 7 +++---- modules/prompt_parser_diffusers.py | 5 ++++- 9 files changed, 42 insertions(+), 20 deletions(-) diff --git a/modules/detailer/detailer.py b/modules/detailer/detailer.py index c7a6ef7ac..3e02ea991 100644 --- a/modules/detailer/detailer.py +++ b/modules/detailer/detailer.py @@ -247,7 +247,7 @@ class Detailer(): pc.disable_extra_networks = True # disable processing_diffusers from handling network activation since its handled here network_same = len(p.network_data.values()) == len(pc.network_data.values()) and all(x == y for x, y in zip(p.network_data.values(), pc.network_data.values())) if not network_same: - extra_networks.activate(pc, pc.network_data) + extra_networks.activate_filtered(pc, pc.network_data) log.debug(f'Detail: model="{i+1}:{name}" item={j+1}/{len(items)} box={item.box} label="{item.label}" score={item.score:.2f} seg={detailer_opt(p, "detailer_segmentation")} network={network_same} prompt="{pc.prompt}"') pc.init_images = [image] pc.image_mask = [item.mask] diff --git a/modules/extra_networks.py b/modules/extra_networks.py index 89acf5a4e..0737f68f8 100644 --- a/modules/extra_networks.py +++ b/modules/extra_networks.py @@ -121,6 +121,15 @@ def activate(p: StableDiffusionProcessing, extra_network_data: defaultdict[str, p.network_data = extra_network_data +def activate_filtered(p: StableDiffusionProcessing, extra_network_data: defaultdict[str, list[ExtraNetworkParams]] | None = None, step=0): + """activate with text encoder components gated on lora_apply_te; must run before prompt encode so te networks affect embeds""" + apply_te = getattr(p, 'lora_apply_te', None) + if apply_te is None: + apply_te = shared.opts.lora_apply_te + exclude = [] if apply_te else ['text_encoder', 'text_encoder_2', 'text_encoder_3'] + activate(p, extra_network_data, step=step, exclude=exclude) + + def deactivate(p: StableDiffusionProcessing, extra_network_data: defaultdict[str, list[ExtraNetworkParams]] | None = None, force: bool | None = None): """call deactivate for extra networks in extra_network_data in specified order, then call deactivate for all remaining registered networks""" if p.disable_extra_networks: diff --git a/modules/face/faceid.py b/modules/face/faceid.py index b2263d840..4c63bcd66 100644 --- a/modules/face/faceid.py +++ b/modules/face/faceid.py @@ -216,7 +216,7 @@ def face_id( p.subseeds = p.all_subseeds[n * p.batch_size:(n+1) * p.batch_size] p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts, p.network_data) - extra_networks.activate(p, p.network_data) + extra_networks.activate_filtered(p, p.network_data) ip_model_dict.update({ "prompt": p.prompts[0], "negative_prompt": p.negative_prompts[0], diff --git a/modules/lora/extra_networks_lora.py b/modules/lora/extra_networks_lora.py index e0d5dd984..f17d357aa 100644 --- a/modules/lora/extra_networks_lora.py +++ b/modules/lora/extra_networks_lora.py @@ -191,11 +191,13 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): key = f'include={",".join(include)}:exclude={",".join(exclude)}' loaded = sd_model.loaded_loras.get(key, []) if len(requested) != len(loaded): + sd_model.loaded_loras.clear() # single-entry cache: any activation invalidates state recorded under other filter keys sd_model.loaded_loras[key] = requested debug_log(f'Network check: type=LoRA key="{key}" requested={requested} loaded={loaded} status="num changed"') return True, "num changed" for req, load in zip(requested, loaded, strict=False): if req != load: + sd_model.loaded_loras.clear() sd_model.loaded_loras[key] = requested debug_log(f'Network check: type=LoRA key="{key}" requested={requested} loaded={loaded} status="content changed"') return True, "content changed" @@ -237,7 +239,7 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): has_changed = lora_nunchaku.load_nunchaku(names, unet_multipliers) else: # native - lora_load.network_load(names, te_multipliers, unet_multipliers, dyn_dims) # load + lora_load.network_load(names, te_multipliers, unet_multipliers, dyn_dims, activate=False) # load only, activation below honors include/exclude has_changed, reason = self.changed(requested, include, exclude) if has_changed: jobid = shared.state.begin('LoRA') diff --git a/modules/lora/lora_load.py b/modules/lora/lora_load.py index 604b24342..9f719e9e1 100644 --- a/modules/lora/lora_load.py +++ b/modules/lora/lora_load.py @@ -263,7 +263,7 @@ def gather_networks(names): return networks_on_disk -def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None, lora_modules=None): +def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None, lora_modules=None, activate=True): networks_on_disk = gather_networks(names) failed_to_load_networks = [] recompile_model, skip_lora_load = maybe_recompile_model(names, te_multipliers) @@ -342,9 +342,10 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non # Activate native modules loaded via diffusers path (e.g., LoKR on Flux2) # Also restore backed-up weights when previously active native modules are removed + # Callers that run their own deactivate/activate sequence pass activate=False from modules.lora import networks native_nets = [net for net in l.loaded_networks if len(net.modules) > 0] - if native_nets or networks.native_active: + if activate and (native_nets or networks.native_active): networks.network_activate() if len(l.loaded_networks) > 0 and l.debug: diff --git a/modules/lora/networks.py b/modules/lora/networks.py index fe84f3b8a..53bfd1606 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -28,11 +28,13 @@ def network_activate(include=None, exclude=None): modules = {} components = include if len(include) > 0 else default_components components = [x for x in components if x not in exclude] + filtered_components = [x for x in default_components if x not in components] # filtered components restore to backup so a filter means detached, not frozen with stale weights active_components = [] - for name in components: + for name in components + filtered_components: component = getattr(sd_model, name, None) if component is not None and hasattr(component, 'named_modules'): - active_components.append(name) + if name in components: + active_components.append(name) modules[name] = list(component.named_modules()) total = sum(len(x) for x in modules.values()) if len(l.loaded_networks) > 0: @@ -48,16 +50,25 @@ def network_activate(include=None, exclude=None): applied_layers.clear() backup_size = 0 for component in modules.keys(): + component_wanted = wanted_names if component in components else () device = getattr(sd_model, component, None).device for _, module in modules[component]: network_layer_name = getattr(module, 'network_layer_name', None) current_names = getattr(module, "network_current_names", ()) - if getattr(module, 'weight', None) is None or shared.state.interrupted or (network_layer_name is None) or (current_names == wanted_names): + if getattr(module, 'weight', None) is None or shared.state.interrupted or (network_layer_name is None) or (current_names == component_wanted): if task is not None: pbar.update(task, advance=1) continue - backup_size += network_backup_weights(module, network_layer_name, wanted_names) - batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name, elimit=elimit) + backup_size += network_backup_weights(module, network_layer_name, component_wanted) + if component_wanted == (): + weights_backup = getattr(module, "network_weights_backup", None) + if weights_backup is None or isinstance(weights_backup, bool): # fuse mode has no tensor backup, restore stays with network_deactivate + if task is not None: + pbar.update(task, advance=1) + continue + batch_updown, batch_ex_bias = None, None # restore-only pass, apply with no weights reverts to backup + else: + batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name, elimit=elimit) if shared.opts.lora_fuse_native: network_apply_direct(module, batch_updown, batch_ex_bias, device=device) else: @@ -68,7 +79,7 @@ def network_activate(include=None, exclude=None): applied_bias += 1 if batch_ex_bias is not None else 0 batch_updown, batch_ex_bias = None, None del batch_updown, batch_ex_bias - module.network_current_names = wanted_names + module.network_current_names = component_wanted if task is not None: bs = round(backup_size/1024/1024/1024, 2) if backup_size > 0 else None pbar.update(task, advance=1, description=f'networks={len(l.loaded_networks)} modules={active_components} layers={total} weights={applied_weight} bias={applied_bias} backup={bs} device={device}') diff --git a/modules/processing_args.py b/modules/processing_args.py index 36872e2e5..eca6b4ce5 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -6,7 +6,7 @@ import inspect import torch import numpy as np from PIL import Image -from modules import shared, sd_models, processing, processing_vae, processing_helpers, sd_hijack_hypertile, extra_networks, sd_vae +from modules import shared, sd_models, processing, processing_vae, processing_helpers, sd_hijack_hypertile, sd_vae from modules.logger import log from modules.processing_callbacks import diffusers_callback_legacy, diffusers_callback, set_callbacks_p from modules.processing_helpers import get_generator, apply_circular # pylint: disable=unused-import @@ -241,9 +241,6 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:l else: args['clip_skip'] = clip_skip - 1 - if shared.opts.lora_apply_te: - extra_networks.activate(p, include=['text_encoder', 'text_encoder_2', 'text_encoder_3']) - if 'complex_human_instruction' in possible: chi = shared.opts.te_complex_human_instruction p.extra_generation_params["CHI"] = chi diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index e22ae2d79..079868382 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -147,6 +147,8 @@ def process_base(p: processing.StableDiffusionProcessing): desc = 'Base' if 'detailer' in p.ops: desc = 'Detail' + p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts, p.network_data) + extra_networks.activate_filtered(p) # networks must patch weights before prompt encode so te loras affect embeds base_args = set_pipeline_args( p=p, model=shared.sd_model, @@ -176,9 +178,6 @@ def process_base(p: processing.StableDiffusionProcessing): modelstats.analyze() try: t0 = time.time() - p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts, p.network_data) - extra_networks.activate(p, exclude=['text_encoder', 'text_encoder_2', 'text_encoder_3']) - if hasattr(shared.sd_model, 'tgate') and getattr(p, 'gate_step', -1) > 0: base_args['gate_step'] = p.gate_step output = shared.sd_model.tgate(**base_args) # pylint: disable=not-callable @@ -311,7 +310,7 @@ def process_hires(p: processing.StableDiffusionProcessing, output): prompts, p.network_data = extra_networks.parse_prompts(prompts) reset_prompts = True if reset_prompts or ('base' in p.skip): - extra_networks.activate(p) + extra_networks.activate_filtered(p) hires_args = set_pipeline_args( p=p, diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 2025a4ca6..0c69c34d7 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -130,8 +130,11 @@ class PromptEmbedder: # unpack EN data in case of TE LoRA en_data = p.network_data en_data = [idx.items for item in en_data.values() for idx in item] + apply_te = getattr(p, 'lora_apply_te', None) + if apply_te is None: + apply_te = shared.opts.lora_apply_te effective_batch = 1 if self.allsame else self.batchsize - key = str([self.prompts, self.negative_prompts, effective_batch, self.clip_skip, self.steps, en_data]) + key = str([self.prompts, self.negative_prompts, effective_batch, self.clip_skip, self.steps, en_data, apply_te]) item = cache.get(key) if not item: if not any(flatten(emb) for emb in [self.prompt_embeds,