From 917dd3a10970e960d9bb238446cee08e6057dc39 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Wed, 8 Jul 2026 03:05:30 +0100 Subject: [PATCH 1/4] fix(lora): handle flattened clip text model in kohya te keys transformers >=5.6 removed the text_model wrapper from CLIPTextModel, so kohya te keys no longer matched the network layer mapping and text encoder weights were silently skipped. KeyConvert retries te keys with the text_model segment dropped; lora extraction keeps writing canonical kohya naming for flattened encoders. --- modules/lora/lora_convert.py | 6 ++++++ modules/lora/lora_extract.py | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/modules/lora/lora_convert.py b/modules/lora/lora_convert.py index 8af280f98..48c1b09b3 100644 --- a/modules/lora/lora_convert.py +++ b/modules/lora/lora_convert.py @@ -130,6 +130,12 @@ class KeyConvert: sd_module = shared.sd_model.network_layer_mapping.get(key, None) if sd_module is None: sd_module = shared.sd_model.network_layer_mapping.get(key.replace("guidance", "timestep"), None) # FLUX1 fix + if sd_module is None and key.startswith("lora_te"): + # transformers >=5.6 flattened CLIPTextModel; kohya te keys still carry the text_model wrapper + flat_key = key.replace("_text_model_", "_", 1) + sd_module = shared.sd_model.network_layer_mapping.get(flat_key, None) + if sd_module is not None: + key = flat_key if debug and sd_module is None: raise RuntimeError(f"LoRA key not found in network_layer_mapping: key={key} mapping={shared.sd_model.network_layer_mapping.keys()}") return key, sd_module diff --git a/modules/lora/lora_extract.py b/modules/lora/lora_extract.py index 22c6019fc..3f25824e0 100644 --- a/modules/lora/lora_extract.py +++ b/modules/lora/lora_extract.py @@ -142,14 +142,17 @@ def make_lora(fn, maxrank, auto_rank, rank_ratio, modules, overwrite): if 'te' in modules and getattr(shared.sd_model, 'text_encoder', None) is not None: task = progress.add_task(description="te1 decompose", total=len(list(shared.sd_model.text_encoder.named_modules()))) + # transformers >=5.6 flattened CLIPTextModel; kohya naming keeps the text_model wrapper + flattened_clip = 'CLIPTextModel' in shared.sd_model.text_encoder.__class__.__name__ and not hasattr(shared.sd_model.text_encoder, 'text_model') for name, module in shared.sd_model.text_encoder.named_modules(): progress.update(task, advance=1) weights_backup = getattr(module, "network_weights_backup", None) if weights_backup is None or getattr(module, "network_current_names", None) is None: continue prefix = "lora_te1_" if hasattr(shared.sd_model, 'text_encoder_2') else "lora_te_" + key_name = f'text_model.{name}' if flattened_clip else name module.svdhandler = SVDHandler(maxrank, rank_ratio) - module.svdhandler.network_name = prefix + name.replace(".", "_") + module.svdhandler.network_name = prefix + key_name.replace(".", "_") with devices.inference_context(): module.svdhandler.decompose(module.weight, weights_backup) progress.remove_task(task) From 4554b9a2779604ef8507c74c7be7d6b235dc8f2f Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Wed, 8 Jul 2026 03:06:04 +0100 Subject: [PATCH 2/4] fix(lora): apply te networks before encode and honor lora_apply_te Network activation ran after prompt encoding, so text encoder lora weights never affected embeds on the first generation and the stale result was then served from the embed cache. The trailing unfiltered activate in network_load also overrode the te exclude filter, so the lora_apply_te setting was never honored. - parse and activate networks in process_base before pipeline args are built - activate_filtered gates text encoder components on per-request or global lora_apply_te; used by base, hires, detailer and faceid call sites - network_load accepts activate=False for callers that run their own deactivate/activate sequence with include/exclude - network_activate walks excluded components in restore-only mode so a filtered text encoder reverts to backup instead of keeping stale deltas - loaded_loras cache is single-entry since per-filter entries go stale when the setting toggles - prompt embed cache key includes the effective lora_apply_te value --- modules/detailer/detailer.py | 2 +- modules/extra_networks.py | 9 +++++++++ modules/face/faceid.py | 2 +- modules/lora/extra_networks_lora.py | 4 +++- modules/lora/lora_load.py | 5 +++-- modules/lora/networks.py | 23 +++++++++++++++++------ modules/processing_args.py | 5 +---- modules/processing_diffusers.py | 7 +++---- modules/prompt_parser_diffusers.py | 5 ++++- 9 files changed, 42 insertions(+), 20 deletions(-) diff --git a/modules/detailer/detailer.py b/modules/detailer/detailer.py index c7a6ef7ac..3e02ea991 100644 --- a/modules/detailer/detailer.py +++ b/modules/detailer/detailer.py @@ -247,7 +247,7 @@ class Detailer(): pc.disable_extra_networks = True # disable processing_diffusers from handling network activation since its handled here network_same = len(p.network_data.values()) == len(pc.network_data.values()) and all(x == y for x, y in zip(p.network_data.values(), pc.network_data.values())) if not network_same: - extra_networks.activate(pc, pc.network_data) + extra_networks.activate_filtered(pc, pc.network_data) log.debug(f'Detail: model="{i+1}:{name}" item={j+1}/{len(items)} box={item.box} label="{item.label}" score={item.score:.2f} seg={detailer_opt(p, "detailer_segmentation")} network={network_same} prompt="{pc.prompt}"') pc.init_images = [image] pc.image_mask = [item.mask] diff --git a/modules/extra_networks.py b/modules/extra_networks.py index 89acf5a4e..0737f68f8 100644 --- a/modules/extra_networks.py +++ b/modules/extra_networks.py @@ -121,6 +121,15 @@ def activate(p: StableDiffusionProcessing, extra_network_data: defaultdict[str, p.network_data = extra_network_data +def activate_filtered(p: StableDiffusionProcessing, extra_network_data: defaultdict[str, list[ExtraNetworkParams]] | None = None, step=0): + """activate with text encoder components gated on lora_apply_te; must run before prompt encode so te networks affect embeds""" + apply_te = getattr(p, 'lora_apply_te', None) + if apply_te is None: + apply_te = shared.opts.lora_apply_te + exclude = [] if apply_te else ['text_encoder', 'text_encoder_2', 'text_encoder_3'] + activate(p, extra_network_data, step=step, exclude=exclude) + + def deactivate(p: StableDiffusionProcessing, extra_network_data: defaultdict[str, list[ExtraNetworkParams]] | None = None, force: bool | None = None): """call deactivate for extra networks in extra_network_data in specified order, then call deactivate for all remaining registered networks""" if p.disable_extra_networks: diff --git a/modules/face/faceid.py b/modules/face/faceid.py index b2263d840..4c63bcd66 100644 --- a/modules/face/faceid.py +++ b/modules/face/faceid.py @@ -216,7 +216,7 @@ def face_id( p.subseeds = p.all_subseeds[n * p.batch_size:(n+1) * p.batch_size] p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts, p.network_data) - extra_networks.activate(p, p.network_data) + extra_networks.activate_filtered(p, p.network_data) ip_model_dict.update({ "prompt": p.prompts[0], "negative_prompt": p.negative_prompts[0], diff --git a/modules/lora/extra_networks_lora.py b/modules/lora/extra_networks_lora.py index e0d5dd984..f17d357aa 100644 --- a/modules/lora/extra_networks_lora.py +++ b/modules/lora/extra_networks_lora.py @@ -191,11 +191,13 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): key = f'include={",".join(include)}:exclude={",".join(exclude)}' loaded = sd_model.loaded_loras.get(key, []) if len(requested) != len(loaded): + sd_model.loaded_loras.clear() # single-entry cache: any activation invalidates state recorded under other filter keys sd_model.loaded_loras[key] = requested debug_log(f'Network check: type=LoRA key="{key}" requested={requested} loaded={loaded} status="num changed"') return True, "num changed" for req, load in zip(requested, loaded, strict=False): if req != load: + sd_model.loaded_loras.clear() sd_model.loaded_loras[key] = requested debug_log(f'Network check: type=LoRA key="{key}" requested={requested} loaded={loaded} status="content changed"') return True, "content changed" @@ -237,7 +239,7 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): has_changed = lora_nunchaku.load_nunchaku(names, unet_multipliers) else: # native - lora_load.network_load(names, te_multipliers, unet_multipliers, dyn_dims) # load + lora_load.network_load(names, te_multipliers, unet_multipliers, dyn_dims, activate=False) # load only, activation below honors include/exclude has_changed, reason = self.changed(requested, include, exclude) if has_changed: jobid = shared.state.begin('LoRA') diff --git a/modules/lora/lora_load.py b/modules/lora/lora_load.py index 604b24342..9f719e9e1 100644 --- a/modules/lora/lora_load.py +++ b/modules/lora/lora_load.py @@ -263,7 +263,7 @@ def gather_networks(names): return networks_on_disk -def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None, lora_modules=None): +def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None, lora_modules=None, activate=True): networks_on_disk = gather_networks(names) failed_to_load_networks = [] recompile_model, skip_lora_load = maybe_recompile_model(names, te_multipliers) @@ -342,9 +342,10 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non # Activate native modules loaded via diffusers path (e.g., LoKR on Flux2) # Also restore backed-up weights when previously active native modules are removed + # Callers that run their own deactivate/activate sequence pass activate=False from modules.lora import networks native_nets = [net for net in l.loaded_networks if len(net.modules) > 0] - if native_nets or networks.native_active: + if activate and (native_nets or networks.native_active): networks.network_activate() if len(l.loaded_networks) > 0 and l.debug: diff --git a/modules/lora/networks.py b/modules/lora/networks.py index fe84f3b8a..53bfd1606 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -28,11 +28,13 @@ def network_activate(include=None, exclude=None): modules = {} components = include if len(include) > 0 else default_components components = [x for x in components if x not in exclude] + filtered_components = [x for x in default_components if x not in components] # filtered components restore to backup so a filter means detached, not frozen with stale weights active_components = [] - for name in components: + for name in components + filtered_components: component = getattr(sd_model, name, None) if component is not None and hasattr(component, 'named_modules'): - active_components.append(name) + if name in components: + active_components.append(name) modules[name] = list(component.named_modules()) total = sum(len(x) for x in modules.values()) if len(l.loaded_networks) > 0: @@ -48,16 +50,25 @@ def network_activate(include=None, exclude=None): applied_layers.clear() backup_size = 0 for component in modules.keys(): + component_wanted = wanted_names if component in components else () device = getattr(sd_model, component, None).device for _, module in modules[component]: network_layer_name = getattr(module, 'network_layer_name', None) current_names = getattr(module, "network_current_names", ()) - if getattr(module, 'weight', None) is None or shared.state.interrupted or (network_layer_name is None) or (current_names == wanted_names): + if getattr(module, 'weight', None) is None or shared.state.interrupted or (network_layer_name is None) or (current_names == component_wanted): if task is not None: pbar.update(task, advance=1) continue - backup_size += network_backup_weights(module, network_layer_name, wanted_names) - batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name, elimit=elimit) + backup_size += network_backup_weights(module, network_layer_name, component_wanted) + if component_wanted == (): + weights_backup = getattr(module, "network_weights_backup", None) + if weights_backup is None or isinstance(weights_backup, bool): # fuse mode has no tensor backup, restore stays with network_deactivate + if task is not None: + pbar.update(task, advance=1) + continue + batch_updown, batch_ex_bias = None, None # restore-only pass, apply with no weights reverts to backup + else: + batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name, elimit=elimit) if shared.opts.lora_fuse_native: network_apply_direct(module, batch_updown, batch_ex_bias, device=device) else: @@ -68,7 +79,7 @@ def network_activate(include=None, exclude=None): applied_bias += 1 if batch_ex_bias is not None else 0 batch_updown, batch_ex_bias = None, None del batch_updown, batch_ex_bias - module.network_current_names = wanted_names + module.network_current_names = component_wanted if task is not None: bs = round(backup_size/1024/1024/1024, 2) if backup_size > 0 else None pbar.update(task, advance=1, description=f'networks={len(l.loaded_networks)} modules={active_components} layers={total} weights={applied_weight} bias={applied_bias} backup={bs} device={device}') diff --git a/modules/processing_args.py b/modules/processing_args.py index 36872e2e5..eca6b4ce5 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -6,7 +6,7 @@ import inspect import torch import numpy as np from PIL import Image -from modules import shared, sd_models, processing, processing_vae, processing_helpers, sd_hijack_hypertile, extra_networks, sd_vae +from modules import shared, sd_models, processing, processing_vae, processing_helpers, sd_hijack_hypertile, sd_vae from modules.logger import log from modules.processing_callbacks import diffusers_callback_legacy, diffusers_callback, set_callbacks_p from modules.processing_helpers import get_generator, apply_circular # pylint: disable=unused-import @@ -241,9 +241,6 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:l else: args['clip_skip'] = clip_skip - 1 - if shared.opts.lora_apply_te: - extra_networks.activate(p, include=['text_encoder', 'text_encoder_2', 'text_encoder_3']) - if 'complex_human_instruction' in possible: chi = shared.opts.te_complex_human_instruction p.extra_generation_params["CHI"] = chi diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index e22ae2d79..079868382 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -147,6 +147,8 @@ def process_base(p: processing.StableDiffusionProcessing): desc = 'Base' if 'detailer' in p.ops: desc = 'Detail' + p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts, p.network_data) + extra_networks.activate_filtered(p) # networks must patch weights before prompt encode so te loras affect embeds base_args = set_pipeline_args( p=p, model=shared.sd_model, @@ -176,9 +178,6 @@ def process_base(p: processing.StableDiffusionProcessing): modelstats.analyze() try: t0 = time.time() - p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts, p.network_data) - extra_networks.activate(p, exclude=['text_encoder', 'text_encoder_2', 'text_encoder_3']) - if hasattr(shared.sd_model, 'tgate') and getattr(p, 'gate_step', -1) > 0: base_args['gate_step'] = p.gate_step output = shared.sd_model.tgate(**base_args) # pylint: disable=not-callable @@ -311,7 +310,7 @@ def process_hires(p: processing.StableDiffusionProcessing, output): prompts, p.network_data = extra_networks.parse_prompts(prompts) reset_prompts = True if reset_prompts or ('base' in p.skip): - extra_networks.activate(p) + extra_networks.activate_filtered(p) hires_args = set_pipeline_args( p=p, diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 2025a4ca6..0c69c34d7 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -130,8 +130,11 @@ class PromptEmbedder: # unpack EN data in case of TE LoRA en_data = p.network_data en_data = [idx.items for item in en_data.values() for idx in item] + apply_te = getattr(p, 'lora_apply_te', None) + if apply_te is None: + apply_te = shared.opts.lora_apply_te effective_batch = 1 if self.allsame else self.batchsize - key = str([self.prompts, self.negative_prompts, effective_batch, self.clip_skip, self.steps, en_data]) + key = str([self.prompts, self.negative_prompts, effective_batch, self.clip_skip, self.steps, en_data, apply_te]) item = cache.get(key) if not item: if not any(flatten(emb) for emb in [self.prompt_embeds, From 2605764f43a793a4868d6405506a0d5e5c7fc265 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Wed, 8 Jul 2026 03:06:14 +0100 Subject: [PATCH 3/4] fix(lora): honor te multiplier for text encoder keys NetworkModule.multiplier matched text encoders via 'transformer' in the key prefix, which fits dit keys but never lora_te keys, so text encoder modules followed unet_multiplier[0] and the te= tag strength was ignored. --- modules/lora/network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/lora/network.py b/modules/lora/network.py index 5926cd4fa..99516b6e4 100644 --- a/modules/lora/network.py +++ b/modules/lora/network.py @@ -181,7 +181,7 @@ class NetworkModule: def multiplier(self): unet_multiplier = 3 * [self.network.unet_multiplier] if not isinstance(self.network.unet_multiplier, list) else self.network.unet_multiplier - if 'transformer' in self.sd_key[:20]: + if self.sd_key.startswith('lora_te') or 'transformer' in self.sd_key[:20]: return self.network.te_multiplier if "down_blocks" in self.sd_key: return unet_multiplier[0] From 1f549839dd242b7573a3645ad0091d80164e4e1c Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Wed, 8 Jul 2026 03:06:47 +0100 Subject: [PATCH 4/4] fix(lora): disable diffusers-method loras on removal Removing all loras never called set_adapters, so peft adapters stayed active until model reload. Removal now uses disable_lora, which keeps modules intact; unload_lora_weights would detach balanced offload hooks. Load calls enable_lora after set_adapters since peft set_adapter does not clear the disabled flag. Removal of fused diffusers loras remains unhandled. --- modules/lora/extra_networks_lora.py | 8 ++++++++ modules/lora/lora_load.py | 1 + 2 files changed, 9 insertions(+) diff --git a/modules/lora/extra_networks_lora.py b/modules/lora/extra_networks_lora.py index f17d357aa..9dd88560d 100644 --- a/modules/lora/extra_networks_lora.py +++ b/modules/lora/extra_networks_lora.py @@ -231,6 +231,14 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork): if has_changed: jobid = shared.state.begin('LoRA') lora_load.network_load(names, te_multipliers, unet_multipliers, dyn_dims, lora_modules) # load only on first call + if len(names) == 0: # removal disables adapters in place, unload_lora_weights would unwrap modules and detach offload hooks + sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) + if hasattr(sd_model, 'disable_lora'): + try: + sd_model.disable_lora() + log.info('Network unload: type=LoRA mode=diffusers') + except Exception as e: + log.error(f'Network unload: type=LoRA {e}') sd_models.set_diffuser_offload(shared.sd_model, op="model") shared.state.end(jobid) diff --git a/modules/lora/lora_load.py b/modules/lora/lora_load.py index 9f719e9e1..960c248fa 100644 --- a/modules/lora/lora_load.py +++ b/modules/lora/lora_load.py @@ -324,6 +324,7 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non log.trace(f'Network load: type=LoRA list={sd_model.get_list_adapters()}') log.trace(f'Network load: type=LoRA active={sd_model.get_active_adapters()}') sd_model.set_adapters(adapter_names=lora_diffusers.diffuser_loaded, adapter_weights=lora_diffusers.diffuser_scales) + sd_model.enable_lora() # set_adapters does not clear the disabled flag left by a prior removal except Exception as e: if str(e) not in exclude_errors: log.error(f'Network load: type=LoRA action=strength {str(e)}')