Merge pull request #5069 from vladmandic/feat/lora-sdnq-stack

feat(lora): multi-network stack modes and per-block strength

modules/control/run.py

@@ -403,7 +403,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
                 sequential_seed: bool | None = None,
                 # prompt/attention overrides
                 prompt_attention: str | None = None, prompt_mean_norm: bool | None = None, diffusers_zeros_prompt_pad: bool | None = None,
-                te_pooled_embeds: bool | None = None, lora_apply_te: bool | None = None, te_complex_human_instruction: str | None = None, te_use_mask: bool | None = None,
+                te_pooled_embeds: bool | None = None, te_complex_human_instruction: str | None = None, te_use_mask: bool | None = None,
                 # generation modifier overrides (hijack)
                 freeu_enabled: bool | None = None, freeu_b1: float | None = None, freeu_b2: float | None = None, freeu_s1: float | None = None, freeu_s2: float | None = None,
                 hypertile_unet_enabled: bool | None = None, hypertile_hires_only: bool | None = None, hypertile_unet_tile: int | None = None, hypertile_unet_min_tile: int | None = None,
@@ -586,7 +586,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
         # prompt/attention overrides
         prompt_attention=prompt_attention, prompt_mean_norm=prompt_mean_norm,
         diffusers_zeros_prompt_pad=diffusers_zeros_prompt_pad, te_pooled_embeds=te_pooled_embeds,
-        lora_apply_te=lora_apply_te, te_complex_human_instruction=te_complex_human_instruction, te_use_mask=te_use_mask,
+        te_complex_human_instruction=te_complex_human_instruction, te_use_mask=te_use_mask,
         # generation modifier overrides (hijack)
         freeu_enabled=freeu_enabled, freeu_b1=freeu_b1, freeu_b2=freeu_b2, freeu_s1=freeu_s1, freeu_s2=freeu_s2,
         hypertile_unet_enabled=hypertile_unet_enabled, hypertile_hires_only=hypertile_hires_only,

modules/detailer/detailer.py

@@ -321,7 +321,7 @@ class Detailer():
                 pc.disable_extra_networks = True # disable processing_diffusers from handling network activation since its handled here
                 network_same = len(p.network_data.values()) == len(pc.network_data.values()) and all(x == y for x, y in zip(p.network_data.values(), pc.network_data.values()))
                 if not network_same:
-                    extra_networks.activate_filtered(pc, pc.network_data)
+                    extra_networks.activate(pc, pc.network_data)
                 log.debug(f'Detail: model="{i+1}:{name}" item={j+1}/{len(items)} box={item.box} label="{item.label}" score={item.score:.2f} seg={detailer_opt(p, "detailer_segmentation")} network={network_same} prompt="{pc.prompt}"')
                 pc.init_images = [image]
                 pc.image_mask = [item.mask]

modules/extra_networks.py

@@ -121,15 +121,6 @@ def activate(p: StableDiffusionProcessing, extra_network_data: defaultdict[str,
     p.network_data = extra_network_data
 
 
-def activate_filtered(p: StableDiffusionProcessing, extra_network_data: defaultdict[str, list[ExtraNetworkParams]] | None = None, step=0):
-    """activate with text encoder components gated on lora_apply_te; must run before prompt encode so te networks affect embeds"""
-    apply_te = getattr(p, 'lora_apply_te', None)
-    if apply_te is None:
-        apply_te = shared.opts.lora_apply_te
-    exclude = [] if apply_te else ['text_encoder', 'text_encoder_2', 'text_encoder_3']
-    activate(p, extra_network_data, step=step, exclude=exclude)
-
-
 def deactivate(p: StableDiffusionProcessing, extra_network_data: defaultdict[str, list[ExtraNetworkParams]] | None = None, force: bool | None = None):
     """call deactivate for extra networks in extra_network_data in specified order, then call deactivate for all remaining registered networks"""
     if p.disable_extra_networks:

modules/face/faceid.py

@@ -216,7 +216,7 @@ def face_id(
                 p.subseeds = p.all_subseeds[n * p.batch_size:(n+1) * p.batch_size]
                 p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts, p.network_data)
 
-                extra_networks.activate_filtered(p, p.network_data)
+                extra_networks.activate(p, p.network_data)
                 ip_model_dict.update({
                         "prompt": p.prompts[0],
                         "negative_prompt": p.negative_prompts[0],

modules/lora/extra_networks_lora.py

@@ -98,6 +98,7 @@ def parse(p, params_list, step=0):
     unet_multipliers = []
     dyn_dims = []
     lora_modules = []
+    block_specs = []
     for params in params_list:
         name = params.positional[0]
 
@@ -131,6 +132,7 @@ def parse(p, params_list, step=0):
         te_multipliers.append(te_multiplier)
         unet_multipliers.append(unet_multiplier)
         dyn_dims.append(dyn_dim)
+        block_specs.append(params.named.get('lbw', None)) # per-block strength; resolved per layer by lora_blocks
 
         lora_module = []
         name_lower = params.positional[0].lower()
@@ -150,7 +152,7 @@ def parse(p, params_list, step=0):
 
         lora_modules.append(lora_module)
 
-    return names, te_multipliers, unet_multipliers, dyn_dims, lora_modules
+    return names, te_multipliers, unet_multipliers, dyn_dims, lora_modules, block_specs
 
 
 def unload_diffusers():
@@ -174,12 +176,13 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork):
         self.model = None
         self.errors = {}
 
-    def signature(self, names: list[str], te_multipliers: list, unet_multipliers: list):
-        return [f'{name}:{te}:{unet}' for name, te, unet in zip(names, te_multipliers, unet_multipliers, strict=False)]
+    def signature(self, names: list[str], te_multipliers: list, unet_multipliers: list, block_specs: list | None = None):
+        specs = block_specs if block_specs else [None] * len(names)
+        return [f'{name}:{te}:{unet}' + (f':lbw={str(spec).strip().lower()}' if spec else '') for name, te, unet, spec in zip(names, te_multipliers, unet_multipliers, specs, strict=False)]
 
     def changed(self, requested: list[str], include: list[str] | None = None, exclude: list[str] | None = None) -> tuple[bool, str]:
-        from modules.lora import lora_sdnq
-        requested = requested + [f'stack={lora_sdnq.signature()}'] # settings-only mechanism changes must re-trigger activation
+        from modules.lora import lora_sdnq, lora_stack
+        requested = requested + [f'stack={lora_stack.signature()}{lora_sdnq.signature()}'] # settings-only stack or mechanism changes must re-trigger activation
         if shared.opts.lora_force_reload:
             debug_log(f'Network check: type=LoRA requested={requested} status="forced"')
             return True, "forced"
@@ -220,11 +223,16 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork):
         if len(params_list) > 0 and not self.active: # activate patches once
             self.active = True
             self.model = shared.opts.sd_model_checkpoint
-        names, te_multipliers, unet_multipliers, dyn_dims, lora_modules = parse(p, params_list, step)
-        requested = self.signature(names, te_multipliers, unet_multipliers)
+        names, te_multipliers, unet_multipliers, dyn_dims, lora_modules, block_specs = parse(p, params_list, step)
+        requested = self.signature(names, te_multipliers, unet_multipliers, block_specs)
         reason = ''
 
         load_method, load_reason = lora_overrides.get_method()
+        from modules.lora import lora_stack
+        if load_method != 'native' and lora_stack.mode() != 'sum':
+            log.warning(f'Network stack: mode={lora_stack.mode()} method={load_method} fallback=sum')
+        if load_method != 'native' and any(block_specs):
+            log.warning(f'Network blocks: method={load_method} fallback=none')
         if debug:
             import sys
             fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access
@@ -251,12 +259,12 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork):
             has_changed = lora_nunchaku.load_nunchaku(names, unet_multipliers)
 
         else: # native
-            lora_load.network_load(names, te_multipliers, unet_multipliers, dyn_dims, activate=False) # load only, activation below honors include/exclude
+            lora_load.network_load(names, te_multipliers, unet_multipliers, dyn_dims, block_specs=block_specs, activate=False) # load only, activation below honors include/exclude
             has_changed, reason = self.changed(requested, include, exclude)
             if has_changed:
                 jobid = shared.state.begin('LoRA')
                 if len(l.previously_loaded_networks) > 0:
-                    log.info(f'Network unload: type=LoRA networks={[n.name for n in l.previously_loaded_networks]} mode={"fuse" if lora_overrides.fuse_native() else "backup"}')
+                    log.info(f'Network unload: type=LoRA networks={[n.name for n in l.previously_loaded_networks]} mode={networks.effective_mode()}')
                     networks.network_deactivate(include, exclude)
                 networks.network_activate(include, exclude)
                 debug_log(f'Network change: type=LoRA previous={[n.name for n in l.previously_loaded_networks]} current={[n.name for n in l.loaded_networks]}')
@@ -269,7 +277,8 @@ class ExtraNetworkLora(extra_networks.ExtraNetwork):
             prompt(p)
             if has_changed and len(include) == 0: # print only once
                 actual_method = 'native' if any(len(n.modules) > 0 for n in l.loaded_networks) else load_method
-                log.info(f'Network load: type=LoRA networks={[n.name for n in l.loaded_networks]} load={load_method}({load_reason}) method={actual_method} mode={"fuse" if lora_overrides.fuse_native() else "backup"} te={te_multipliers} unet={unet_multipliers} time={l.timer.summary} reason="{reason}"')
+                stack = lora_stack.signature() if actual_method == 'native' else 'sum' # non-native paths always combine as sum
+                log.info(f'Network load: type=LoRA networks={[n.name for n in l.loaded_networks]} load={load_method}({load_reason}) method={actual_method} mode={networks.effective_mode()} stack={stack} te={te_multipliers} unet={unet_multipliers} time={l.timer.summary} reason="{reason}"')
 
     def deactivate(self, p, force=False):
         if len(lora_diffusers.diffuser_loaded) > 0 and (shared.opts.lora_force_reload or force):

modules/lora/lora_apply.py

@@ -5,6 +5,7 @@ import time
 from typing import TYPE_CHECKING
 import torch
 from modules.lora import lora_common as l
+from modules.lora import lora_stack
 from modules import shared, devices, errors
 from modules.logger import log
 
@@ -67,7 +68,7 @@ def network_backup_weights(self: torch.nn.Conv2d | torch.nn.Linear | torch.nn.Gr
     return backup_size
 
 
-def network_calc_weights(self: torch.nn.Conv2d | torch.nn.Linear | torch.nn.GroupNorm | torch.nn.LayerNorm | diffusers.models.lora.LoRACompatibleLinear | diffusers.models.lora.LoRACompatibleConv, network_layer_name: str, use_previous: bool = False, *, elimit: Callable[[], None] | None = None):
+def network_calc_weights(self: torch.nn.Conv2d | torch.nn.Linear | torch.nn.GroupNorm | torch.nn.LayerNorm | diffusers.models.lora.LoRACompatibleLinear | diffusers.models.lora.LoRACompatibleConv, network_layer_name: str, use_previous: bool = False, *, elimit: Callable[[], None] | None = None, per_net: bool = False):
     if shared.opts.diffusers_offload_mode == "none":
         try:
             self.to(devices.device)
@@ -75,6 +76,9 @@ def network_calc_weights(self: torch.nn.Conv2d | torch.nn.Linear | torch.nn.Grou
             pass
     batch_updown = None
     batch_ex_bias = None
+    stack_deltas = None
+    if per_net or (lora_stack.mode() in lora_stack.DENSE_MODES and network_layer_name is not None and not network_layer_name.startswith('lora_te')):
+        stack_deltas = [] # collect per-net deltas; combined after the loop unless the caller wants them separate (bias deltas stay summed)
     loaded = l.loaded_networks if not use_previous else l.previously_loaded_networks
     for net in loaded:
         module = net.modules.get(network_layer_name, None)
@@ -107,7 +111,9 @@ def network_calc_weights(self: torch.nn.Conv2d | torch.nn.Linear | torch.nn.Grou
             del weight
 
             if updown is not None:
-                if batch_updown is not None:
+                if stack_deltas is not None:
+                    stack_deltas.append((net.name, updown.to(devices.device)))
+                elif batch_updown is not None:
                     batch_updown += updown.to(batch_updown.device)
                 else:
                     batch_updown = updown.to(devices.device)
@@ -136,6 +142,17 @@ def network_calc_weights(self: torch.nn.Conv2d | torch.nn.Linear | torch.nn.Grou
             if elimit is not None:
                 elimit()
         continue
+    if per_net:
+        return stack_deltas, batch_ex_bias
+    if stack_deltas is not None and stack_deltas:
+        if len(stack_deltas) >= 2:
+            t0 = time.time()
+            batch_updown = lora_stack.combine(stack_deltas, network_layer_name)
+            l.timer.calc += time.time() - t0
+        else:
+            batch_updown = stack_deltas[0][1]
+        if shared.opts.diffusers_offload_mode == "sequential":
+            batch_updown = batch_updown.to(devices.cpu)
     return batch_updown, batch_ex_bias
 
 

modules/lora/lora_blocks.py

@@ -0,0 +1,348 @@
+"""Per-block LoRA strength: <lora:name:1.0:lbw=VALUE>.
+
+Each targeted layer maps to one slot of a per-architecture weight vector and
+the network's multiplier is scaled by that slot. Slot 0 is BASE: on unet
+architectures it covers the text encoder and the unet layers outside the
+block chain, on transformer architectures the layers outside the block
+chain(s). The remaining slots follow the merge block-weight layout on unet
+architectures (26 on sd, 20 on sdxl: input blocks, mid, output blocks) and
+the transformer chain(s) in depth order elsewhere, with chain lengths
+scanned from the live network_layer_mapping rather than hardcoded.
+
+VALUE is a preset name (case-insensitive), a single number broadcast to
+every slot, or a comma list with one number per slot. Named presets force
+BASE to 1.0, since the merge tables carry 0 there with merge semantics, and
+stretch onto the block count of the current model; classic segment names
+(INS, OUTALL, ...) generate from ranges, so they also work on transformer
+chains via thirds, and DOUBLE/SINGLE mute one chain on two-chain
+architectures. Explicit vectors are taken verbatim at the slot count, with
+the a1111 17-slot (sd) and 12-slot (sdxl) layouts accepted and expanded,
+omitted slots neutral. A value that fits nothing is ignored with a warning
+and the network applies at its plain strength.
+"""
+
+import re
+
+from modules import shared
+from modules.logger import log
+from modules.lora import lora_common as l
+
+
+UNET_ARCHES = ('sd', 'sdxl')
+CHAINS = { # arch -> anchored tail prefixes, one per chain, in depth order
+    'sd3': ('transformer_blocks_',),
+    'anima': ('transformer_blocks_',),
+    'f1': ('transformer_blocks_', 'single_transformer_blocks_'),
+    'f2': ('transformer_blocks_', 'single_transformer_blocks_'),
+    'chroma': ('transformer_blocks_', 'single_transformer_blocks_'),
+    'zimage': ('layers_',),
+    'ernieimage': ('layers_',),
+    'krea2': ('blocks_',),
+}
+CLASSIC = ('ALL', 'NONE', 'INALL', 'INS', 'IND', 'MIDD', 'OUTALL', 'OUTD', 'OUTS')
+CHAIN_NAMES = ('DOUBLE', 'SINGLE')
+SD1_17 = (0, 2, 3, 5, 6, 8, 9, 13, 17, 18, 19, 20, 21, 22, 23, 24, 25) # BASE, IN01, IN02, IN04, IN05, IN07, IN08, MID, OUT03..OUT11
+SDXL_12 = (0, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16) # BASE, IN04, IN05, IN07, IN08, MID, OUT00..OUT05
+VECTOR_MEMO_CAP = 64
+MISS = object()
+
+re_down = re.compile(r'^down_blocks_(\d+)_(resnets|attentions|downsamplers)_(\d+)')
+re_up = re.compile(r'^up_blocks_(\d+)_(resnets|attentions|upsamplers)_(\d+)')
+re_chain_index = re.compile(r'^(\d+)')
+
+state: dict = {'stamp': None, 'layout': None, 'index': {}, 'vectors': {}}
+warned: set = set()
+
+
+def warn_once(key, message):
+    if key not in warned:
+        warned.add(key)
+        log.warning(message)
+
+
+def build_unet_layout(arch, mapping):
+    down, up = -1, -1
+    for key in mapping:
+        if not key.startswith('lora_unet_'):
+            continue
+        tail = key[len('lora_unet_'):]
+        m = re_down.match(tail)
+        if m is not None:
+            down = max(down, int(m.group(1)))
+            continue
+        m = re_up.match(tail)
+        if m is not None:
+            up = max(up, int(m.group(1)))
+    if down < 0 or up < 0:
+        return None
+    n_in = 3 * (down + 1) # conv_in plus two pairs and a sampler slot per group: the compvis input_blocks count
+    n_out = 3 * (up + 1)
+    n = 2 + n_in + n_out
+    return {
+        'arch': arch, 'kind': 'unet', 'n': n, 'n_in': n_in,
+        'ins': list(range(1, 1 + n_in)),
+        'mids': [1 + n_in],
+        'outs': list(range(2 + n_in, n)),
+    }
+
+
+def build_dit_layout(arch, mapping):
+    prefixes = CHAINS.get(arch)
+    if prefixes is None:
+        return None
+    counts = [0 for _ in prefixes]
+    for key in mapping:
+        if not key.startswith('lora_transformer_'):
+            continue
+        tail = key[len('lora_transformer_'):]
+        for i, prefix in enumerate(prefixes):
+            if tail.startswith(prefix):
+                m = re_chain_index.match(tail[len(prefix):])
+                if m is not None:
+                    counts[i] = max(counts[i], int(m.group(1)) + 1)
+                break
+    total = sum(counts)
+    if total == 0:
+        return None
+    chains = []
+    offset = 0
+    for prefix, count in zip(prefixes, counts, strict=False):
+        chains.append((prefix, count, offset))
+        offset += count
+    n = 1 + total
+    blocks = list(range(1, n))
+    return {
+        'arch': arch, 'kind': 'dit', 'n': n, 'chains': chains,
+        'ins': [s for i, s in enumerate(blocks) if i * 3 // total == 0],
+        'mids': [s for i, s in enumerate(blocks) if i * 3 // total == 1],
+        'outs': [s for i, s in enumerate(blocks) if i * 3 // total == 2],
+    }
+
+
+def layout():
+    sd_model = getattr(shared, 'sd_model', None)
+    mapping = getattr(sd_model, 'network_layer_mapping', None) if sd_model is not None else None
+    if not mapping:
+        return None
+    arch = shared.sd_model_type
+    stamp = (arch, id(mapping))
+    if state['stamp'] == stamp:
+        return state['layout']
+    state['stamp'] = stamp
+    state['layout'] = build_unet_layout(arch, mapping) if arch in UNET_ARCHES else build_dit_layout(arch, mapping)
+    state['index'].clear()
+    state['vectors'].clear()
+    return state['layout']
+
+
+def classify(sd_key, lay):
+    if sd_key.startswith('lora_te'):
+        return 0 if lay['kind'] == 'unet' else None # BASE covers the TE on unet arches; transformer vectors do not model the TE
+    if sd_key.startswith('lora_llm_adapter_'):
+        return None
+    if lay['kind'] == 'unet':
+        if not sd_key.startswith('lora_unet_'):
+            return None
+        tail = sd_key[len('lora_unet_'):]
+        m = re_down.match(tail)
+        if m is not None:
+            slot = 1 + 3 * int(m.group(1)) + (2 if m.group(2) == 'downsamplers' else int(m.group(3)))
+            return 1 + slot
+        m = re_up.match(tail)
+        if m is not None:
+            slot = 3 * int(m.group(1)) + (2 if m.group(2) == 'upsamplers' else int(m.group(3)))
+            return 2 + lay['n_in'] + slot
+        if tail.startswith('mid_block'):
+            return 1 + lay['n_in']
+        if tail.startswith('conv_in'):
+            return 1 # IN00
+        if tail.startswith('conv_out') or tail.startswith('conv_norm_out'):
+            return lay['n'] - 1 # the compvis out group belongs to the last output block
+        return 0 # time_embedding, add_embedding and other non-block leaves
+    if not sd_key.startswith('lora_transformer_'):
+        return None
+    tail = sd_key[len('lora_transformer_'):]
+    for prefix, count, offset in lay['chains']:
+        if tail.startswith(prefix):
+            m = re_chain_index.match(tail[len(prefix):])
+            if m is not None and int(m.group(1)) < count:
+                return 1 + offset + int(m.group(1))
+            return 0
+    return 0 # embedders, projections, refiners and other non-chain layers
+
+
+def block_index(sd_key):
+    lay = layout()
+    if lay is None:
+        return None
+    cached = state['index'].get(sd_key, MISS)
+    if cached is not MISS:
+        return cached
+    idx = classify(sd_key, lay)
+    state['index'][sd_key] = idx
+    return idx
+
+
+def fill_band(vec, slots, lo, hi):
+    k = len(slots)
+    for i, s in enumerate(slots):
+        if lo * k <= i < hi * k:
+            vec[s] = 1.0
+
+
+def classic_vector(name, lay):
+    if name == 'ALL':
+        return [1.0] * lay['n']
+    vec = [0.0] * lay['n']
+    if name == 'NONE':
+        return vec
+    vec[0] = 1.0
+    if name == 'INALL':
+        fill_band(vec, lay['ins'], 0.0, 1.0)
+    elif name == 'INS': # shallow half of the input side
+        fill_band(vec, lay['ins'], 0.0, 0.5)
+    elif name == 'IND': # deep half of the input side
+        fill_band(vec, lay['ins'], 0.5, 1.0)
+    elif name == 'MIDD': # the middle of the network: deep input half, mid, deep output half
+        fill_band(vec, lay['ins'], 0.5, 1.0)
+        fill_band(vec, lay['mids'], 0.0, 1.0)
+        fill_band(vec, lay['outs'], 0.0, 0.5)
+    elif name == 'OUTALL':
+        fill_band(vec, lay['outs'], 0.0, 1.0)
+    elif name == 'OUTD': # deep half of the output side, nearest the mid
+        fill_band(vec, lay['outs'], 0.0, 0.5)
+    elif name == 'OUTS': # shallow half of the output side, nearest the image
+        fill_band(vec, lay['outs'], 0.5, 1.0)
+    return vec
+
+
+def chain_vector(name, lay):
+    chains = lay.get('chains') or []
+    if len(chains) != 2:
+        return None
+    vec = [1.0] * lay['n']
+    keep = 0 if name == 'DOUBLE' else 1
+    for i, (_prefix, count, offset) in enumerate(chains):
+        val = 1.0 if i == keep else 0.0
+        for s in range(1 + offset, 1 + offset + count):
+            vec[s] = val
+    return vec
+
+
+def stretch(src, k):
+    if k == len(src):
+        return [float(v) for v in src]
+    out = []
+    for i in range(k):
+        x = i * (len(src) - 1) / (k - 1) if k > 1 else 0.0
+        lo = int(x)
+        hi = min(lo + 1, len(src) - 1)
+        f = x - lo
+        out.append(float(src[lo]) * (1.0 - f) + float(src[hi]) * f)
+    return out
+
+
+def preset_vector(name, lay):
+    from modules.merging.merge_presets import BLOCK_WEIGHTS_PRESETS, SDXL_BLOCK_WEIGHTS_PRESETS
+    if name in CHAIN_NAMES:
+        return chain_vector(name, lay)
+    if name in CLASSIC:
+        return classic_vector(name, lay)
+    if lay['arch'] == 'sdxl':
+        src = SDXL_BLOCK_WEIGHTS_PRESETS.get(name) or SDXL_BLOCK_WEIGHTS_PRESETS.get('SDXL_' + name)
+        if src is not None:
+            return [1.0] + [float(v) for v in src[1:]] # merge tables carry 0 in the BASE slot; a preset must leave the TE alone
+    if name.startswith('SDXL_'):
+        return None # explicitly arch-tagged, not reinterpreted elsewhere
+    src = BLOCK_WEIGHTS_PRESETS.get(name)
+    if src is None:
+        return None
+    if lay['arch'] == 'sd':
+        return [1.0] + [float(v) for v in src[1:]]
+    return [1.0] + stretch(src[1:], lay['n'] - 1)
+
+
+def parse_vector(parts, lay):
+    try:
+        vals = [float(x) for x in parts]
+    except ValueError:
+        return None
+    n = lay['n']
+    if len(vals) == n:
+        return vals
+    if len(vals) == n - 1:
+        return [1.0] + vals
+    legacy = SD1_17 if lay['arch'] == 'sd' else (SDXL_12 if lay['arch'] == 'sdxl' else None)
+    if legacy is not None and len(vals) == len(legacy):
+        vec = [1.0] * n # slots the a1111 layouts omit stay neutral
+        for slot, v in zip(legacy, vals, strict=False):
+            vec[slot] = v
+        return vec
+    return None
+
+
+def resolve(spec):
+    """Resolve a raw lbw value into a slot vector for the current model, or None when it fits nothing."""
+    lay = layout()
+    if lay is None:
+        return None
+    raw = str(spec).strip()
+    key = raw.lower()
+    if key in state['vectors']:
+        return state['vectors'][key]
+    if len(state['vectors']) > VECTOR_MEMO_CAP:
+        state['vectors'].clear()
+    vec = None
+    if ',' in raw:
+        vec = parse_vector([x.strip() for x in raw.split(',')], lay)
+        if vec is None:
+            warn_once(f'lbw-vector:{key}:{lay["arch"]}', f'Network blocks: value="{raw}" arch={lay["arch"]} expected={lay["n"]} fallback=none')
+    else:
+        try:
+            vec = [float(raw)] * lay['n']
+        except ValueError:
+            vec = preset_vector(raw.upper(), lay)
+            if vec is None:
+                warn_once(f'lbw-name:{key}:{lay["arch"]}', f'Network blocks: preset="{raw}" arch={lay["arch"]} fallback=none')
+    if vec is not None:
+        log.info(f'Network blocks: value="{raw}" arch={lay["arch"]} slots={lay["n"]} range={min(vec):.2f}-{max(vec):.2f}')
+    state['vectors'][key] = vec
+    return vec
+
+
+def factor(sd_key, net):
+    """Per-layer scale from a network's block vector; 1.0 whenever the vector does not apply."""
+    try:
+        spec = getattr(net, 'block_spec', None)
+        if not spec:
+            return 1.0
+        vec = resolve(spec)
+        if vec is None:
+            return 1.0
+        idx = block_index(sd_key)
+        if idx is None:
+            return 1.0
+        return float(vec[idx])
+    except Exception as e:
+        warn_once('lbw-error', f'Network blocks: {e} fallback=none')
+        return 1.0
+
+
+def net_signature(net):
+    """Normalized spec of one network, or None; joins content identities such as the factor cache signature."""
+    spec = getattr(net, 'block_spec', None)
+    if not spec:
+        return None
+    return str(spec).strip().lower()
+
+
+def active():
+    return any(getattr(net, 'block_spec', None) for net in l.loaded_networks)
+
+
+def signature():
+    """Identity suffix for the per-module apply stamp; empty while no loaded network carries block weights."""
+    specs = [f'{net.name}:{net_signature(net)}' for net in l.loaded_networks if getattr(net, 'block_spec', None)]
+    if len(specs) == 0:
+        return ''
+    return '|lbw=' + ','.join(specs)

modules/lora/lora_factor_cache.py

@@ -51,12 +51,15 @@ def signature(wanted_names):
     if model_name is None:
         return None
     calib_path = lora_calib.calib_file(model_name)
+    from modules.lora import lora_stack
     parts = {
         'model': model_name,
         'rank': int(getattr(shared.opts, 'lora_sdnq_host_rank', 0) or 0),
         'calib': int(os.path.getmtime(calib_path)) if lora_calib.enabled() and os.path.isfile(calib_path) else None, # the toggle is part of the identity: factors computed under the other setting must not replay
+        'stack': lora_stack.signature(),
         'nets': [],
     }
+    from modules.lora import lora_blocks
     for name, te, unet, dyn in wanted_names:
         net = next((n for n in l.loaded_networks if n.name == name), None)
         filename = getattr(getattr(net, 'network_on_disk', None), 'filename', None)
@@ -64,7 +67,11 @@ def signature(wanted_names):
             st = os.stat(filename)
         except Exception:
             return None
-        parts['nets'].append([name, repr(te), repr(unet), repr(dyn), filename, int(st.st_mtime), st.st_size])
+        entry = [name, repr(te), repr(unet), repr(dyn), filename, int(st.st_mtime), st.st_size]
+        spec = lora_blocks.net_signature(net)
+        if spec is not None: # appended only when set so existing cache files stay valid without block weights
+            entry.append(spec)
+        parts['nets'].append(entry)
     return parts
 
 
@@ -155,6 +162,30 @@ def fetch(network_layer_name):
     return entry
 
 
+def lookup_scores(network_layer_name):
+    """Cached select scores for a layer as ((s0, s1), (a0, a1)), or None.
+
+    Score records ride the same signature-keyed entry as factors, and the
+    signature already pins everything the scores depend on (pair, multipliers,
+    stack mode and params). No hit/miss accounting: a record saves scoring and
+    delta assembly, not a sketch.
+    """
+    if state['sig'] is None:
+        return None
+    t = state['store'].get(f'{network_layer_name}.sel')
+    if t is None:
+        return None
+    return (float(t[0]), float(t[1])), (float(t[2]), float(t[3]))
+
+
+def store_scores(network_layer_name, scores, abs_sums):
+    """Persist a select-mode score record; additive to the entry, older files upgrade on their next pass."""
+    if state['sig'] is None:
+        return
+    state['store'][f'{network_layer_name}.sel'] = torch.tensor([scores[0], scores[1], abs_sums[0], abs_sums[1]], dtype=torch.float64)
+    state['dirty'] = True
+
+
 def store(network_layer_name, up, down, energy, calibrated, rms):
     """Quantize-before-use: returns the dequantized round-trip the caller must apply.
 

modules/lora/lora_load.py

@@ -261,7 +261,7 @@ def gather_networks(names):
     return networks_on_disk
 
 
-def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None, lora_modules=None, activate=True):
+def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None, lora_modules=None, block_specs=None, activate=True):
     networks_on_disk = gather_networks(names)
     failed_to_load_networks = []
     recompile_model, skip_lora_load = maybe_recompile_model(names, te_multipliers)
@@ -309,6 +309,7 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non
             'te': te_multipliers[i] if te_multipliers else shared.opts.extra_networks_default_multiplier,
             'unet': unet_multipliers[i] if unet_multipliers else shared.opts.extra_networks_default_multiplier,
             'dyn': dyn_dims[i] if dyn_dims else None, # a multiplier is not a rank; float dyn_dim crashes every consumer that slices with it
+            'blocks': block_specs[i] if block_specs and len(block_specs) > i else None,
         }
         l.loaded_networks.append(net)
 

modules/lora/lora_overrides.py

@@ -112,6 +112,10 @@ def disable_fuse():
     round-trips it through its storage format. On quantized weights that is a
     dequantize-add-requantize cycle per network swap whose error compounds.
     """
+    from modules.lora import lora_common as l
+    from modules.lora import lora_stack
+    if lora_stack.select_possible(len(l.loaded_networks)) or lora_stack.select_engaged():
+        return True # select flips per-layer winners against the pristine backup; a dormant select mode leaves fuse alone
     sd_model = getattr(shared.sd_model, 'pipe', shared.sd_model)
     if is_quantized(sd_model):
         return True

modules/lora/lora_sdnq.py

@@ -47,7 +47,7 @@ a low-rank delta hosts exactly however fat it is.
 import torch
 
 from modules import devices, shared
-from modules.lora import lora_calib, lora_factor_cache
+from modules.lora import lora_calib, lora_factor_cache, lora_stack
 from modules.lora import lora_common as l
 from modules.logger import log
 
@@ -55,6 +55,8 @@ from modules.logger import log
 fallback_layers: list[str] = []
 hosted_layers: list[tuple[str, float, bool]] = []
 hosted_ranks: list[int] = []
+factor_layers: list[str] = []
+select_layers: list[str] = []
 routed_layers: list[str] = []
 
 REQUANT_RATIO = 0.30 # delta rms over mean grid step above which requantize can retain the delta
@@ -137,6 +139,9 @@ def factor_candidate(self, network_layer_name, wanted_names):
         return False # declined layers with factors still attached are stripped by the activate fallthrough
     if getattr(self, 'sdnq_dequantizer', None) is None or self.__class__.__name__ != 'SDNQLinear':
         return False
+    if wanted_names != () and lora_stack.mode() in lora_stack.DENSE_MODES and not network_layer_name.startswith('lora_te'):
+        if sum(1 for net in l.loaded_networks if net.modules.get(network_layer_name, None) is not None) >= 2:
+            return False # dense stack modes combine dense deltas; the factor concat would sum
     if hasattr(self, 'sdnq_lora_svd_stash'):
         return True
     if wanted_names == ():  # nothing attached, nothing to remove
@@ -171,6 +176,7 @@ def remove_factors(self):
     self.svd_up = svd_up
     self.svd_down = svd_down
     del self.sdnq_lora_svd_stash
+    lora_stack.drop(getattr(self, 'network_layer_name', None)) # a selection schedule must not outlive the segments it points into
     return True
 
 
@@ -207,15 +213,28 @@ def apply_factors(self, network_layer_name, wanted_names):
     if not ups:
         return changed
     append_factors(self, ups, downs)
+    factor_layers.append(network_layer_name)
     return True
 
 
 def append_factors(self, ups, downs):
-    """Concatenate ``[out, r]`` / ``[r, in]`` factor pairs onto the layer's svd channel and stash the originals."""
+    """Concatenate ``[out, r]`` / ``[r, in]`` factor pairs onto the layer's svd channel and stash the originals.
+
+    Returns the appended parts' rank ranges plus the transposed-layout flag; the
+    checkpoint's own factors occupy the range before the first entry and bucket
+    padding lands after the last, so the ranges stay valid on the live buffers.
+    """
     deq = self.sdnq_dequantizer
     device = self.scale.device
     dtype = deq.result_dtype
     orig_up, orig_down = self.svd_up, self.svd_down
+    orig_rank = 0
+    if orig_up is not None:
+        orig_rank = orig_up.shape[0] if deq.use_quantized_matmul else orig_up.shape[1]
+    segments, offset = [], orig_rank
+    for u in ups:
+        segments.append((offset, offset + u.shape[1]))
+        offset += u.shape[1]
     if deq.use_quantized_matmul:
         # matmul layout stores factors transposed: svd_up [r, out], svd_down [in, r]
         parts_up = ([orig_up.to(device=devices.device, dtype=dtype)] if orig_up is not None else []) + [u.t() for u in ups]
@@ -237,10 +256,11 @@ def append_factors(self, ups, downs):
     self.sdnq_lora_svd_stash = (orig_up, orig_down)
     self.svd_up = torch.nn.Parameter(new_up.to(device=device), requires_grad=False)
     self.svd_down = torch.nn.Parameter(new_down.to(device=device), requires_grad=False)
+    return segments, deq.use_quantized_matmul
 
 
-def host_candidate(self, network_layer_name, wanted_names):
-    """True when a non-factorable set on this layer should be hosted as a truncated svd."""
+def select_candidate(self, network_layer_name, wanted_names):
+    """True when this layer can carry a set on the svd channel; select pairs ride it at any bit width."""
     if not enabled():
         return False
     if int(getattr(shared.opts, 'lora_sdnq_host_rank', 0) or 0) <= 0:
@@ -249,11 +269,20 @@ def host_candidate(self, network_layer_name, wanted_names):
         return False
     if wanted_names == ():
         return False
+    return any(net.modules.get(network_layer_name, None) is not None for net in l.loaded_networks)
+
+
+def host_candidate(self, network_layer_name, wanted_names):
+    """True when this layer's set should ride the svd channel as a truncated svd: non-factorable sets below 8 bits, dense-combined sets at any width."""
+    if not select_candidate(self, network_layer_name, wanted_names):
+        return False
+    if lora_stack.mode() in lora_stack.DENSE_MODES and not network_layer_name.startswith('lora_te'):
+        if sum(1 for net in l.loaded_networks if net.modules.get(network_layer_name, None) is not None) >= 2:
+            return True # combined deltas host at any width: requantizing them is checkpoint-fragile, while single-adapter requantize is well retained
     from sdnq.common import dtype_dict
     if dtype_dict[self.sdnq_dequantizer.weights_dtype]['num_bits'] >= 8:
-        return False # requantize retains most of the delta at 8 bits and above; truncation would lose more than it saves
-
-    return any(net.modules.get(network_layer_name, None) is not None for net in l.loaded_networks)
+        return False # requantize retains most of a single set's delta at 8 bits and above; truncation would lose more than it saves
+    return True
 
 
 def apply_cached(self, network_layer_name, wanted_names):
@@ -277,15 +306,17 @@ def apply_cached(self, network_layer_name, wanted_names):
     deq = self.sdnq_dequantizer
     dtype = deq.result_dtype
     remove_factors(self) # before the rule: the svd-channel check must see the checkpoint's own state, and a declined layer must fall through pristine
+    stack_dense = lora_stack.mode() in lora_stack.DENSE_MODES and not network_layer_name.startswith('lora_te')
     members = []
-    for net in l.loaded_networks:
-        module = net.modules.get(network_layer_name, None)
-        if module is None:
-            continue
-        factors = get_module_factors(module, devices.device, dtype, original_shape=deq.original_shape)
-        if factors is not None:
-            members.append(factors)
-    if len(members) == 0 and self.svd_up is None:
+    if not stack_dense:
+        for net in l.loaded_networks:
+            module = net.modules.get(network_layer_name, None)
+            if module is None:
+                continue
+            factors = get_module_factors(module, devices.device, dtype, original_shape=deq.original_shape)
+            if factors is not None:
+                members.append(factors)
+    if not stack_dense and len(members) == 0 and self.svd_up is None:
         step = float(self.scale.detach().float().mean())
         if step > 0 and rms / step > REQUANT_RATIO and energy < REQUANT_ENERGY:
             return None # routed to the grid: the caller assembles the delta and requantizes
@@ -328,21 +359,23 @@ def apply_hosted(self, network_layer_name, updown, wanted_names):
     dtype = deq.result_dtype
 
     members = []
-    for net in l.loaded_networks:
-        module = net.modules.get(network_layer_name, None)
-        if module is None:
-            continue
-        factors = get_module_factors(module, devices.device, dtype, original_shape=deq.original_shape)
-        if factors is not None:
-            members.append(factors)
+    stack_dense = lora_stack.mode() in lora_stack.DENSE_MODES and not network_layer_name.startswith('lora_te')
+    if not stack_dense: # dense stack modes host the combined delta wholesale; the members' content is already inside it
+        for net in l.loaded_networks:
+            module = net.modules.get(network_layer_name, None)
+            if module is None:
+                continue
+            factors = get_module_factors(module, devices.device, dtype, original_shape=deq.original_shape)
+            if factors is not None:
+                members.append(factors)
 
     # requantize keeps a delta the grid can resolve and that truncation would genuinely
     # cut: both terms must agree, since a thin delta rounds away on the grid however
     # low its capture, and a low-rank delta hosts exactly however fat it is. Scoped to
     # sets the side-channel would otherwise carry whole: factorable members ride
-    # exactly.
+    # exactly and dense-combined deltas stay hosted at any magnitude.
     delta_rms = float(updown.detach().float().square().mean().sqrt())
-    maybe_requant = len(members) == 0 and self.svd_up is None
+    maybe_requant = not stack_dense and len(members) == 0 and self.svd_up is None
     if maybe_requant:
         step = float(self.scale.detach().float().mean())
         maybe_requant = step > 0 and delta_rms / step > REQUANT_RATIO
@@ -427,6 +460,104 @@ def truncate_delta(self, D, dtype):
     return up_h, down_h, energy, rms is not None
 
 
+def apply_select_cached(self, network_layer_name, wanted_names):
+    """Serve a select pair from cache and live factors before the walk assembles deltas.
+
+    A cached score record plus a factor pair per network (exact factors for
+    factorable members, cached truncations otherwise) rebuild the segments and
+    the selection registration without any ``calc_updown``. Returns None when
+    any piece is missing; the caller assembles and ``apply_select`` recomputes
+    and stores.
+    """
+    from sdnq.quant_utils import rotate_hadamard
+    deq = self.sdnq_dequantizer
+    changed = remove_factors(self)
+    if wanted_names == ():
+        return changed
+    if len(l.loaded_networks) != 2:
+        return None
+    dtype = deq.result_dtype
+    lora_factor_cache.begin_pass(wanted_names)
+    rec = lora_factor_cache.lookup_scores(network_layer_name)
+    if rec is None:
+        return None
+    pairs, notes = [], []
+    for i, net in enumerate(l.loaded_networks):
+        module = net.modules.get(network_layer_name, None)
+        if module is None:
+            return None
+        factors = get_module_factors(module, devices.device, dtype, original_shape=deq.original_shape)
+        if factors is not None:
+            up_i, down_i = factors
+            if deq.use_hadamard:
+                down_i = rotate_hadamard(down_i.to(dtype=torch.float32), group_size=deq.hadamard_group_size).to(dtype=dtype)
+        else:
+            cached = lora_factor_cache.lookup(f'{network_layer_name}#{i}')
+            if cached is None:
+                return None
+            up_i, down_i = cached[0].to(device=devices.device, dtype=dtype), cached[1].to(device=devices.device, dtype=dtype)
+            notes.append((f'{network_layer_name}#{i}', cached[2], cached[3]))
+        pairs.append((up_i, down_i))
+    scores, abs_sums = rec
+    segments, transposed = append_factors(self, [pairs[0][0], pairs[1][0]], [pairs[0][1], pairs[1][1]])
+    lora_stack.register(network_layer_name, self, 'factor', scores, segments=(segments[0], segments[1], transposed), abs_sums=abs_sums)
+    for note in notes:
+        lora_factor_cache.note_hit()
+        hosted_layers.append(note)
+    select_layers.append(network_layer_name)
+    return True
+
+
+def apply_select(self, network_layer_name, per_net, wanted_names):
+    """Attach two networks' contributions as separate side-channel segments for per-layer selection.
+
+    Factorable members ride exactly; the rest host as their own truncated svd
+    with per-net cache entries. Segment ranges and selection scores register
+    with ``lora_stack``; the flip schedule executes from the step callback.
+    Returns None when the pair cannot ride the channel; the caller falls back.
+    """
+    from sdnq.quant_utils import rotate_hadamard
+    deq = self.sdnq_dequantizer
+    changed = remove_factors(self)
+    if wanted_names == ():
+        return changed
+    if per_net is None or len(per_net) != 2:
+        return None
+    dtype = deq.result_dtype
+    lora_factor_cache.begin_pass(wanted_names)
+    pairs, ranks = [], []
+    for i, (net_name, D) in enumerate(per_net):
+        if D is None or D.ndim != 2 or tuple(D.shape) != tuple(deq.original_shape):
+            return None
+        net = next((n for n in l.loaded_networks if n.name == net_name), None)
+        module = net.modules.get(network_layer_name, None) if net is not None else None
+        if module is None:
+            return None
+        ranks.append(int(getattr(module, 'dim', 0) or 0) or min(int(shared.opts.lora_sdnq_host_rank), *deq.original_shape))
+        factors = get_module_factors(module, devices.device, dtype, original_shape=deq.original_shape)
+        if factors is not None:
+            up_i, down_i = factors
+            if deq.use_hadamard:
+                down_i = rotate_hadamard(down_i.to(dtype=torch.float32), group_size=deq.hadamard_group_size).to(dtype=dtype)
+        else:
+            key = f'{network_layer_name}#{i}'
+            cached = lora_factor_cache.fetch(key)
+            if cached is not None:
+                up_i, down_i = cached[0].to(device=devices.device, dtype=dtype), cached[1].to(device=devices.device, dtype=dtype)
+                hosted_layers.append((key, cached[2], cached[3]))
+            else:
+                up_i, down_i, energy, calibrated = truncate_delta(self, D.detach().to(devices.device, torch.float32), dtype)
+                up_i, down_i = lora_factor_cache.store(key, up_i, down_i, energy, calibrated, float(D.detach().float().square().mean().sqrt()))
+                hosted_layers.append((key, energy, calibrated))
+        pairs.append((up_i, down_i))
+    scores, abs_sums = lora_stack.score_pair(per_net[0][1].detach(), per_net[1][1].detach(), ranks[0], ranks[1])
+    lora_factor_cache.store_scores(network_layer_name, scores, abs_sums)
+    segments, transposed = append_factors(self, [pairs[0][0], pairs[1][0]], [pairs[0][1], pairs[1][1]])
+    lora_stack.register(network_layer_name, self, 'factor', scores, segments=(segments[0], segments[1], transposed), abs_sums=abs_sums)
+    select_layers.append(network_layer_name) # counted apart from the plain concat: both ride the svd channel but only one is a summed set
+    return True
+
+
 def note_fallback(self, network_layer_name):
     """Record a quantized layer taking the requantize path (summary-logged per pass); layers the routing rule sent there are counted apart."""
     if getattr(self, 'sdnq_dequantizer', None) is not None and network_layer_name not in routed_layers:
@@ -437,6 +568,12 @@ def report_fallbacks():
     hits, misses = lora_factor_cache.flush()
     if hits > 0 or misses > 0:
         log.info(f'Network load: type=LoRA quant=sdnq cache hits={hits} misses={misses}')
+    if len(factor_layers) > 0:
+        log.info(f'Network load: type=LoRA quant=sdnq apply=exact layers={len(factor_layers)}')
+    factor_layers.clear()
+    if len(select_layers) > 0:
+        log.info(f'Network load: type=LoRA quant=sdnq apply=select layers={len(select_layers)} mode={lora_stack.mode()}')
+    select_layers.clear()
     if len(hosted_layers) > 0:
         energies = sorted(e for _name, e, _c in hosted_layers)
         median = energies[len(energies) // 2]
@@ -445,7 +582,7 @@ def report_fallbacks():
         if len(hosted_ranks) > 0 and min(hosted_ranks) < int(shared.opts.lora_sdnq_host_rank):
             rs = sorted(hosted_ranks)
             ranks = f' k={rs[0]}-{rs[len(rs) // 2]}-{rs[-1]}' # realized rank spread; shown only when a spectrum collapsed below the cap
-        log.info(f'Network load: type=LoRA quant=sdnq hosted={len(hosted_layers)} rank={int(shared.opts.lora_sdnq_host_rank)}{ranks}{f" calib={calibrated}" if calibrated else ""} energy={median:.2f} min={energies[0]:.2f} non-factorable networks hosted on the svd side-channel')
+        log.info(f'Network load: type=LoRA quant=sdnq apply=hosted layers={len(hosted_layers)} rank={int(shared.opts.lora_sdnq_host_rank)}{ranks}{f" calib={calibrated}" if calibrated else ""} energy={median:.2f} min={energies[0]:.2f}')
         if l.debug:
             log.debug(f'Network load: type=LoRA quant=sdnq hosted={[(n, round(e, 3)) for n, e, _c in hosted_layers[:8]]}{"..." if len(hosted_layers) > 8 else ""}')
     hosted_layers.clear()
@@ -457,7 +594,7 @@ def report_fallbacks():
     routed_layers.clear()
     if len(fallback_layers) > 0:
         if enabled():
-            log.warning(f'Network load: type=LoRA quant=sdnq layers={len(fallback_layers)} non-factorable networks requantized in place (reduced fidelity on quantized weights)')
+            log.warning(f'Network load: type=LoRA quant=sdnq apply=requantize layers={len(fallback_layers)} fidelity=reduced')
         else:
             log.info(f'Network load: type=LoRA quant=sdnq apply=requantize layers={len(fallback_layers)} reason=setting')
         if l.debug:

modules/lora/lora_stack.py

@@ -0,0 +1,405 @@
+"""Stack modes for combining multiple LoRA networks beyond plain summation.
+
+Dense modes (ties, dare_ties, dare_linear, magnitude_prune) combine the
+networks' dense deltas elementwise; the result rides the normal apply tail
+(side-channel hosting on sub-8-bit SDNQ, requantize at int8 and above,
+direct add on unquantized layers). Select modes (klora, estlora) keep both
+networks' contributions separate and choose a per-layer winner, shifting
+from the first loaded network (subject) toward the second (style) across
+the sampling steps. Selection scores depend only on the weights, so the
+shift reduces to at most one flip per layer per generation, executed from
+the step callback against a schedule finalized at apply time.
+
+TIES arXiv:2306.01708, DARE arXiv:2311.03099, K-LoRA arXiv:2502.18461,
+EST-LoRA arXiv:2508.02165 (its measured style-discrepancy estimate is
+exposed as an option instead of being derived from probe generations).
+"""
+
+import time
+import weakref
+import hashlib
+
+import torch
+
+from modules import shared
+from modules.logger import log
+
+
+DENSE_MODES = ('ties', 'dare_ties', 'dare_linear', 'magnitude_prune')
+SELECT_MODES = ('klora', 'estlora')
+KLORA_BETA = 0.5 # the paper's fixed ramp offset; only the slope is user-tunable
+ROW_CHUNK = 512 # fp32 interiors run in first-dim slices; also fixes the DARE draw sequence
+SAMPLE_CAP = 1 << 22 # strided subsample bound for magnitude quantiles (full-size quantile exceeds torch limits)
+
+state: dict = {'entries': {}, 'flips': {}, 'gamma': 1.0, 'gamma_e': 1.0, 'total_steps': 0, 'finalized': False, 'reported': None}
+warned: set = set()
+
+
+def mode():
+    return getattr(shared.opts, 'lora_stack_mode', 'sum') or 'sum'
+
+
+def density():
+    return float(getattr(shared.opts, 'lora_stack_density', 0.5))
+
+
+def ramp_alpha():
+    return float(getattr(shared.opts, 'lora_stack_alpha', 1.5))
+
+
+def manual_discrepancy():
+    return float(getattr(shared.opts, 'lora_stack_discrepancy', 0.5))
+
+
+def signature():
+    m = mode()
+    if m in DENSE_MODES:
+        return f'{m}:{density():.2f}'
+    if m in SELECT_MODES:
+        return f'{m}:{ramp_alpha():.2f}:{manual_discrepancy():.2f}'
+    return 'sum'
+
+
+def warn_once(key, message):
+    if key not in warned:
+        warned.add(key)
+        log.warning(message)
+
+
+def select_blocked():
+    return 'Model' in (getattr(shared.opts, 'cuda_compile', None) or [])
+
+
+def active_dense(n_contrib):
+    return mode() in DENSE_MODES and n_contrib >= 2
+
+
+def select_possible(n_loaded):
+    """True when the loaded set could engage a select mode; silent, for the fuse gate."""
+    return mode() in SELECT_MODES and n_loaded == 2 and not select_blocked()
+
+
+def select_engaged():
+    """True while selection schedules are live on model layers."""
+    return bool(state['entries'])
+
+
+def active_select(n_loaded):
+    m = mode()
+    if m not in SELECT_MODES:
+        return False
+    if n_loaded != 2:
+        log.warning(f'Network stack: mode={m} networks={n_loaded} required=2 fallback=sum')
+        return False
+    if select_blocked():
+        log.warning(f'Network stack: mode={m} compile=model fallback=sum')
+        return False
+    return True
+
+
+def seed_for(layer_name, net_name):
+    payload = f'{layer_name}|{net_name}|{mode()}|{round(density(), 6)}'
+    return int.from_bytes(hashlib.sha256(payload.encode()).digest()[:8], 'little')
+
+
+def magnitude_threshold(delta, dens):
+    flat = delta.abs().flatten()
+    step = max(1, flat.numel() // SAMPLE_CAP)
+    return torch.quantile(flat[::step].float(), 1.0 - dens)
+
+
+def dare_generator(device, layer_name, net_name):
+    gen = torch.Generator(device=device)
+    gen.manual_seed(seed_for(layer_name, net_name))
+    return gen
+
+
+def combine(named_deltas, layer_name):
+    """Combine per-network dense deltas under the active dense mode; returns a tensor in the first delta's dtype."""
+    m = mode()
+    dens = density()
+    deltas = [d for _, d in named_deltas]
+    out_dtype = deltas[0].dtype
+    result = torch.zeros_like(deltas[0], dtype=torch.float32)
+    thresholds = [magnitude_threshold(d, dens) for d in deltas] if m in ('ties', 'magnitude_prune') else [None] * len(deltas)
+    gens = [dare_generator(deltas[0].device, layer_name, name) for name, _ in named_deltas] if m in ('dare_ties', 'dare_linear') else [None] * len(deltas)
+    for start in range(0, deltas[0].shape[0], ROW_CHUNK):
+        stop = min(start + ROW_CHUNK, deltas[0].shape[0])
+        chunks = []
+        for i, d in enumerate(deltas):
+            c = d[start:stop].to(torch.float32)
+            if thresholds[i] is not None:
+                c = c * (c.abs() >= thresholds[i])
+            if gens[i] is not None:
+                keep = torch.rand(c.shape, generator=gens[i], device=c.device, dtype=torch.float32) < dens
+                c = c * keep / dens
+            chunks.append(c)
+        if m in ('ties', 'dare_ties'):
+            total = torch.stack(chunks).sum(dim=0)
+            elected = torch.sign(total)
+            agree = [c * ((torch.sign(c) == elected) & (c != 0)) for c in chunks]
+            count = torch.stack([(a != 0).to(torch.float32) for a in agree]).sum(dim=0).clamp(min=1.0)
+            result[start:stop] = torch.stack(agree).sum(dim=0) / count
+        else: # dare_linear, magnitude_prune: independent per-delta edits, plain sum
+            result[start:stop] = torch.stack(chunks).sum(dim=0)
+    return result.to(out_dtype)
+
+
+def score_pair(d0, d1, rank0, rank1):
+    """Selection scores for a dense delta pair: klora top-K sums (K = rank product) or est energies; plus abs-sums for the global balance.
+
+    Row-chunked fp32 interiors with fp64 accumulators and one device sync for
+    all four reductions. Full-tensor staging (fp32 copy, abs copy, top-k
+    workspace) peaks hundreds of MB per large layer, which collides with block
+    swapping on offloaded denoisers; chunking bounds the transient to the
+    chunk. The global top-K over per-chunk top-K candidates selects the same
+    element set as a whole-tensor top-K.
+    """
+    k = max(1, int(rank0) * int(rank1)) if mode() == 'klora' else 0
+    accs = []
+    for d in (d0, d1):
+        score = torch.zeros((), device=d.device, dtype=torch.float64)
+        abs_sum = torch.zeros((), device=d.device, dtype=torch.float64)
+        cands = []
+        for start in range(0, d.shape[0], ROW_CHUNK):
+            c = d[start:start + ROW_CHUNK].to(torch.float32).abs() # out-of-place abs: to() may alias a caller-owned fp32 tensor
+            abs_sum += c.sum(dtype=torch.float64)
+            if k:
+                flat = c.flatten()
+                cands.append(torch.topk(flat, min(k, flat.numel()), sorted=False).values)
+            else:
+                score += c.square().sum(dtype=torch.float64)
+        if k and cands:
+            allc = torch.cat(cands) if len(cands) > 1 else cands[0]
+            score = torch.topk(allc, min(k, allc.numel()), sorted=False).values.sum(dtype=torch.float64)
+        accs.append((score, abs_sum))
+    packed = torch.stack([accs[0][0], accs[0][1], accs[1][0], accs[1][1]]).cpu()
+    return (float(packed[0]), float(packed[2])), (float(packed[1]), float(packed[3]))
+
+
+def register_weight_pair(layer_name, module, per_net, wanted_names=None):
+    """Score and register a weight-kind selection pair; True when the layer is scheduled.
+
+    The scores persist in the factor cache when a pass identity is given, so a
+    later apply of the same configuration registers from the record alone.
+    """
+    from modules.lora import lora_common as l
+    if per_net is None or len(per_net) != 2:
+        return False
+    ranks, names = [], []
+    for net_name, d in per_net:
+        if d is None:
+            return False
+        net = next((n for n in l.loaded_networks if n.name == net_name), None)
+        net_module = net.modules.get(layer_name, None) if net is not None else None
+        if net_module is None:
+            return False
+        names.append(net_name)
+        ranks.append(int(getattr(net_module, 'dim', 0) or 0) or 64)
+    scores, abs_sums = score_pair(per_net[0][1], per_net[1][1], ranks[0], ranks[1])
+    if wanted_names is not None:
+        from modules.lora import lora_factor_cache
+        lora_factor_cache.begin_pass(wanted_names)
+        lora_factor_cache.store_scores(layer_name, scores, abs_sums)
+    register(layer_name, module, 'weight', scores, nets=tuple(names), abs_sums=abs_sums)
+    return True
+
+
+def register_weight_pair_cached(layer_name, module, wanted_names):
+    """Register a weight-kind pair from its cached score record; True when served.
+
+    The record was stored under the same configuration signature, which pins
+    the loaded pair, multipliers and stack settings, so both networks are known
+    to target the layer and the prompt-order roles are unchanged.
+    """
+    from modules.lora import lora_common as l
+    from modules.lora import lora_factor_cache
+    if len(l.loaded_networks) != 2:
+        return False
+    lora_factor_cache.begin_pass(wanted_names)
+    rec = lora_factor_cache.lookup_scores(layer_name)
+    if rec is None:
+        return False
+    scores, abs_sums = rec
+    register(layer_name, module, 'weight', scores, nets=tuple(n.name for n in l.loaded_networks), abs_sums=abs_sums)
+    return True
+
+
+def drop(layer_name):
+    """Forget a layer's selection entry (its factors were removed or restored)."""
+    if layer_name is not None and state['entries'].pop(layer_name, None) is not None:
+        state['finalized'] = False
+
+
+def score_energy(up, down):
+    """EST layer score: squared Frobenius norm of up@down via the Gram identity, no materialization."""
+    u = up.to(torch.float32)
+    dn = down.to(torch.float32)
+    return float(((u.t() @ u) * (dn @ dn.t())).sum())
+
+
+def clear():
+    state['entries'] = {}
+    state['flips'] = {}
+    state['gamma'] = 1.0
+    state['gamma_e'] = 1.0
+    state['total_steps'] = 0
+    state['finalized'] = False
+    state['reported'] = None
+
+
+def register(layer_name, module, kind, scores, segments=None, nets=None, abs_sums=None):
+    """Record a select-mode layer for schedule finalization.
+
+    kind 'factor': segments = ((s0, s1), (t0, t1), transposed) column ranges on the svd
+    channel; both segments' pristine values are stashed for flips. kind 'weight': nets =
+    the two network names; the winner delta is recomputed from the layer backup at
+    selection time. abs_sums feeds the global magnitude balance (klora gamma).
+    """
+    entry = {'layer': layer_name, 'module': weakref.ref(module), 'kind': kind, 'segments': segments, 'scores': scores, 'nets': nets, 'abs_sums': abs_sums, 'stash': None}
+    if kind == 'factor':
+        (s0, s1), (t0, t1), transposed = segments
+        up = module.svd_up.data
+        entry['stash'] = (segment_view(up, s0, s1, transposed).clone(), segment_view(up, t0, t1, transposed).clone())
+    state['entries'][layer_name] = entry
+    state['finalized'] = False
+
+
+def segment_view(up, start, stop, transposed):
+    return up[start:stop] if transposed else up[:, start:stop]
+
+
+def layer_flip_step(scores, total_steps):
+    """First step index at which the style side wins; total_steps when it never does, 0 when style wins from the start."""
+    m = mode()
+    sc, ss = scores
+    for step in range(total_steps):
+        t = step / max(1, total_steps - 1)
+        if m == 'klora':
+            ramp = state['gamma'] * (ramp_alpha() * t + KLORA_BETA)
+            if ss * ramp > sc:
+                return step
+        else: # estlora: content keeps the layer while sc >= gamma_t * ss
+            # est energies are ||dW||^2, so a magnitude gap enters squared; balance the style side by
+            # the total-energy ratio (mirrors klora's gamma) so the louder adapter cannot win on scale alone
+            ramp = ramp_alpha() * t + (1.0 - manual_discrepancy())
+            if sc < ramp * ss * state['gamma_e']:
+                return step
+    return total_steps
+
+
+def materialize_model():
+    """Weight-kind selection rewrites module weights outside the activation walk; rebuild balanced-offload modules real first (mirrors network_activate)."""
+    from modules import sd_models
+    if getattr(shared.opts, 'diffusers_offload_mode', None) == 'balanced' and getattr(shared, 'sd_model', None) is not None:
+        sd_models.apply_balanced_offload(shared.sd_model, force=True, silent=True)
+
+
+def finalize(total_steps):
+    """Build the inverted flip map for the pass; select-mode layers start at their step-0 winner."""
+    state['total_steps'] = int(total_steps)
+    # both balances derive from the live entries every time, so drops and re-registrations stay consistent by construction
+    num = sum(e['abs_sums'][0] for e in state['entries'].values() if e['abs_sums'] is not None)
+    den = sum(e['abs_sums'][1] for e in state['entries'].values() if e['abs_sums'] is not None)
+    state['gamma'] = (num / den) if den > 0 else 1.0
+    e_num = sum(e['scores'][0] for e in state['entries'].values()) # est scores ARE the per-layer energies; their totals give the scale-invariant balance
+    e_den = sum(e['scores'][1] for e in state['entries'].values())
+    state['gamma_e'] = (e_num / e_den) if e_den > 0 else 1.0
+    state['flips'] = {}
+    stats = {'weight_n': 0, 'factor_n': 0, 'materialize': 0.0, 'select': 0.0, 'w_move': 0.0, 'w_calc': 0.0, 'w_apply': 0.0}
+    state['stats'] = stats
+    stats['weight_n'] = sum(1 for e in state['entries'].values() if e['kind'] == 'weight')
+    stats['factor_n'] = len(state['entries']) - stats['weight_n']
+    if stats['weight_n'] > 0:
+        t0 = time.time()
+        materialize_model()
+        stats['materialize'] = time.time() - t0
+    style_first = 0
+    t0 = time.time()
+    for layer_name, entry in list(state['entries'].items()): # snapshot: apply_selection drops entries whose module died
+        flip_at = layer_flip_step(entry['scores'], state['total_steps'])
+        initial = 1 if flip_at == 0 else 0
+        style_first += initial
+        apply_selection(layer_name, entry, initial)
+        if 0 < flip_at < state['total_steps']:
+            state['flips'].setdefault(flip_at - 1, []).append(layer_name) # step callbacks fire after the denoise, so the flip runs one step early to be live during the crossover step's forward
+    stats['select'] = time.time() - t0
+    state['finalized'] = True
+    if len(state['entries']) > 0: # only a built schedule can carry a flip count, so this is the line that shows selection is live rather than requested
+        gamma = state['gamma_e'] if mode() == 'estlora' else state['gamma']
+        report = (mode(), len(state['entries']), style_first, sum(len(v) for v in state['flips'].values()), state['total_steps'], round(gamma, 3))
+        if report != state['reported']: # rebuilt every pass, so a batch would otherwise repeat one line per image
+            state['reported'] = report
+            log.info(f'Network load: type=LoRA stack={report[0]} layers={report[1]} style={report[2]} flips={report[3]} steps={report[4]} gamma={report[5]:.3f}')
+        # logged every pass: the reset runs outside the activate walk, so its cost is invisible to the load timers
+        log.debug(f'Network select: type=LoRA reset weight={stats["weight_n"]} factor={stats["factor_n"]} time={{materialize: {stats["materialize"]:.2f}, select: {stats["select"]:.2f}, move: {stats["w_move"]:.2f}, calc: {stats["w_calc"]:.2f}, apply: {stats["w_apply"]:.2f}}}')
+
+
+def reset(total_steps):
+    """Per-pass reset from set_callbacks_p: restore initial selections and reschedule for this pass's step count."""
+    if mode() not in SELECT_MODES or not state['entries'] or int(total_steps) <= 0:
+        return
+    finalize(total_steps)
+
+
+def on_step(step):
+    """Flip the layers whose crossover is this step; non-flip steps are a dict miss."""
+    if not state['finalized']:
+        return
+    layers = state['flips'].get(int(step), ())
+    if not layers:
+        return
+    t0 = time.time()
+    for layer_name in layers:
+        entry = state['entries'].get(layer_name)
+        if entry is not None:
+            apply_selection(layer_name, entry, 1)
+    log.debug(f'Network select: type=LoRA flip step={int(step)} layers={len(layers)} time={time.time() - t0:.2f}')
+
+
+def apply_selection(layer_name, entry, winner):
+    module = entry['module']()
+    if module is None:
+        state['entries'].pop(layer_name, None)
+        return
+    if entry['kind'] == 'factor':
+        (s0, s1), (t0, t1), transposed = entry['segments']
+        up = module.svd_up.data
+        keep_seg, drop_seg = ((t0, t1), (s0, s1)) if winner == 1 else ((s0, s1), (t0, t1))
+        stash = entry['stash'][winner]
+        segment_view(up, keep_seg[0], keep_seg[1], transposed).copy_(stash.to(device=up.device, dtype=up.dtype))
+        segment_view(up, drop_seg[0], drop_seg[1], transposed).zero_()
+    else:
+        weight_selection(module, entry, winner)
+
+
+def weight_selection(module, entry, winner):
+    from modules.lora import lora_common as l
+    from modules.lora.lora_apply import network_apply_weights
+    if getattr(module, 'sdnq_dequantizer', None) is not None:
+        warn_once('select-sdnq-weight', 'Network stack: flip=skipped layer=quantized') # quantized backups are packed tensors; only the segment path can flip them
+        return
+    backup = getattr(module, 'network_weights_backup', None)
+    if not isinstance(backup, torch.Tensor): # fuse mode keeps a bool sentinel, not a pristine copy
+        warn_once('select-nobackup', 'Network stack: flip=skipped backup=none')
+        return
+    net = next((n for n in l.loaded_networks if n.name == entry['nets'][winner]), None)
+    net_module = net.modules.get(entry['layer'], None) if net is not None else None
+    if net_module is None:
+        return
+    weight = getattr(module, 'weight', None)
+    if weight is None or weight.is_meta:
+        warn_once('select-offloaded', 'Network stack: flip=skipped weight=offloaded')
+        return
+    from modules import devices
+    stats = state.get('stats') or {}
+    device = weight.device
+    t0 = time.time()
+    base = backup.to(devices.device) # a swapped-out layer keeps its weight on cpu; the delta matmul belongs on the accelerator regardless
+    t1 = time.time()
+    updown = net_module.calc_updown(base)[0].to(device)
+    t2 = time.time()
+    network_apply_weights(module, updown, None, device=device) # recomputes from the pristine backup, requantizing where the layer needs it
+    stats['w_move'] = stats.get('w_move', 0.0) + (t1 - t0)
+    stats['w_calc'] = stats.get('w_calc', 0.0) + (t2 - t1)
+    stats['w_apply'] = stats.get('w_apply', 0.0) + (time.time() - t2)

modules/lora/network.py

@@ -148,6 +148,7 @@ class Network:  # LoraModule
         self.te_multiplier = 1.0
         self.unet_multiplier = [1.0] * 3
         self.dyn_dim = None
+        self.block_spec = None # raw lbw= value; per-layer factors resolve through lora_blocks
         self.pending_config = None # staged multipliers; network_activate promotes them after the removal pass so fuse removal subtracts the delta that was applied
         self.modules = {}
         self.mismatch = 0 # deltas dropped for not fitting their target module; try_load_chain refuses the file when non-zero
@@ -195,15 +196,19 @@ class NetworkModule:
     def multiplier(self):
         unet_multiplier = 3 * [self.network.unet_multiplier] if not isinstance(self.network.unet_multiplier, list) else self.network.unet_multiplier
         if self.sd_key.startswith('lora_te') or 'transformer' in self.sd_key[:20]:
-            return self.network.te_multiplier
-        if "down_blocks" in self.sd_key:
-            return unet_multiplier[0]
-        if "mid_block" in self.sd_key:
-            return unet_multiplier[1]
-        if "up_blocks" in self.sd_key:
-            return unet_multiplier[2]
+            base = self.network.te_multiplier
+        elif "down_blocks" in self.sd_key:
+            base = unet_multiplier[0]
+        elif "mid_block" in self.sd_key:
+            base = unet_multiplier[1]
+        elif "up_blocks" in self.sd_key:
+            base = unet_multiplier[2]
         else:
-            return unet_multiplier[0]
+            base = unet_multiplier[0]
+        if getattr(self.network, 'block_spec', None) is None: # per-block strength is off for this network; no shared access on this path
+            return base
+        from modules.lora import lora_blocks
+        return base * lora_blocks.factor(self.sd_key, self.network)
 
     def calc_scale(self):
         if self.scale is not None:

modules/lora/networks.py

@@ -2,9 +2,11 @@ from contextlib import nullcontext
 import time
 import rich.progress as rp
 from modules.errorlimiter import limit_errors
+from modules.lora import lora_blocks
 from modules.lora import lora_common as l
 from modules.lora import lora_overrides
 from modules.lora import lora_sdnq
+from modules.lora import lora_stack
 from modules.lora.lora_apply import network_apply_weights, network_apply_direct, network_backup_weights, network_calc_weights
 from modules import shared, devices, sd_models
 from modules.logger import log, console
@@ -54,6 +56,7 @@ def network_activate(include=None, exclude=None):
             net.te_multiplier = pending['te']
             net.unet_multiplier = pending['unet']
             net.dyn_dim = pending['dyn']
+            net.block_spec = pending.get('blocks', None)
     t0 = time.time()
     fuse = lora_overrides.fuse_native() # resolve once: backup and apply passes must agree
     with limit_errors("network_activate") as elimit:
@@ -62,7 +65,7 @@ def network_activate(include=None, exclude=None):
             sd_models.disable_offload(sd_model)
             sd_models.move_model(sd_model, device=devices.cpu)
         elif shared.opts.diffusers_offload_mode == "balanced":
-            sd_model = sd_models.apply_balanced_offload(sd_model, force=True) # dispatched modules hold meta tensors backed by the offload map; rebuild them real on cpu with hooks intact before touching weights
+            sd_model = sd_models.apply_balanced_offload(sd_model, force=True, silent=True) # dispatched modules hold meta tensors backed by the offload map; rebuild them real on cpu with hooks intact before touching weights
         group_offload = shared.opts.diffusers_offload_mode == "group"
         group_stripped = {}
         device = None
@@ -89,10 +92,13 @@ def network_activate(include=None, exclude=None):
         refused = 0
         with devices.inference_context(), pbar:
             wanted_names = tuple((x.name, x.te_multiplier, x.unet_multiplier, x.dyn_dim) for x in l.loaded_networks) if len(l.loaded_networks) > 0 else ()
-            stack_sig = lora_sdnq.signature() # apply-mechanism token tracked beside network_current_names so a settings-only flip re-applies
+            stack_sig = lora_stack.signature() + lora_blocks.signature() + lora_sdnq.signature() # tracked beside network_current_names so stack-setting, block-weight and mechanism changes re-apply
+            select_active = len(l.loaded_networks) > 0 and lora_stack.active_select(len(l.loaded_networks)) # restore-only walks have nothing to stack; the count warning would fire on every network-free generation
             applied_layers.clear()
             lora_sdnq.fallback_layers.clear() # a raise mid-pass leaves stale entries behind
             lora_sdnq.hosted_layers.clear()
+            lora_sdnq.factor_layers.clear()
+            lora_sdnq.select_layers.clear()
             backup_size = 0
             for component in modules.keys():
                 component_wanted = wanted_names if component in components else ()
@@ -100,13 +106,62 @@ def network_activate(include=None, exclude=None):
                 for _, module in modules[component]:
                     network_layer_name = getattr(module, 'network_layer_name', None)
                     current_names = getattr(module, "network_current_names", ())
-                    if getattr(module, 'weight', None) is None or shared.state.interrupted or (network_layer_name is None) or (current_names == component_wanted and getattr(module, 'network_current_stack', '') == stack_sig):
+                    if getattr(module, 'weight', None) is None or shared.state.interrupted or (network_layer_name is None) or (current_names == component_wanted and getattr(module, 'network_current_stack', 'sum') == stack_sig):
                         if task is not None:
                             pbar.update(task, advance=1)
                         continue
+                    lora_stack.drop(network_layer_name) # re-application invalidates any live selection schedule; the select branch re-registers
                     if group_offload and component not in group_stripped and group_will_mutate(module, network_layer_name, l.loaded_networks):
                         device = group_offload_strip(sd_model, component, group_stripped)
                     calced = False # tracks whether this iteration assembled the delta, so the fallthrough reuses it instead of recomputing
+                    if select_active and component_wanted and not network_layer_name.startswith('lora_te'):
+                        if lora_sdnq.select_candidate(module, network_layer_name, component_wanted): # SDNQ pairs ride the channel as separate segments at any bit width; weight rewrites cannot flip a quantized layer
+                            weights_backup = getattr(module, "network_weights_backup", None)
+                            if weights_backup is not None and not isinstance(weights_backup, bool):
+                                network_apply_weights(module, None, None, device=device)
+                            applied = lora_sdnq.apply_select_cached(module, network_layer_name, component_wanted) # a stored score record and factor pair serve before the deltas are assembled
+                            if applied is None:
+                                per_net, sel_bias = network_calc_weights(module, network_layer_name, elimit=elimit, per_net=True)
+                                if sel_bias is None:
+                                    applied = lora_sdnq.apply_select(module, network_layer_name, per_net, component_wanted)
+                            if applied is not None:
+                                if applied and component_wanted:
+                                    applied_layers.append(network_layer_name)
+                                    applied_weight += 1
+                                module.network_current_names = component_wanted
+                                module.network_current_stack = stack_sig
+                                if task is not None:
+                                    pbar.update(task, advance=1)
+                                continue
+                            lora_stack.warn_once('select-unridable', f'Network stack: mode={lora_stack.mode()} layer="{network_layer_name}" fallback=sum') # a pair the channel cannot carry (bias delta or malformed member) sums like any unsupported set
+                        elif getattr(module, 'sdnq_dequantizer', None) is not None: # hosting disabled: quantized layers have no side-channel to carry segments and packed backups cannot flip, so the sum paths below take the layer
+                            if any(net.modules.get(network_layer_name, None) is not None for net in l.loaded_networks):
+                                lora_stack.warn_once('select-host-disabled', f'Network stack: mode={lora_stack.mode()} quant=sdnq host=disabled fallback=sum')
+                        else: # other layers select by recomputing the winner from the pristine backup at schedule time
+                            sel_backup = network_backup_weights(module, network_layer_name, component_wanted, fuse)
+                            weights_backup = getattr(module, "network_weights_backup", None)
+                            if weights_backup is not None and not isinstance(weights_backup, bool):
+                                if lora_stack.register_weight_pair_cached(network_layer_name, module, component_wanted): # a stored score record registers without assembling the pair
+                                    backup_size += sel_backup
+                                    network_apply_weights(module, None, None, device=device) # pristine until the schedule applies the winner
+                                    applied_layers.append(network_layer_name)
+                                    applied_weight += 1
+                                    module.network_current_names = component_wanted
+                                    module.network_current_stack = stack_sig
+                                    if task is not None:
+                                        pbar.update(task, advance=1)
+                                    continue
+                                per_net, sel_bias = network_calc_weights(module, network_layer_name, elimit=elimit, per_net=True)
+                                if sel_bias is None and lora_stack.register_weight_pair(network_layer_name, module, per_net, component_wanted):
+                                    backup_size += sel_backup # counted only when this branch keeps the layer; the fallthrough re-enters the shared backup call below, which counts it then
+                                    network_apply_weights(module, None, None, device=device) # pristine until the schedule applies the winner
+                                    applied_layers.append(network_layer_name)
+                                    applied_weight += 1
+                                    module.network_current_names = component_wanted
+                                    module.network_current_stack = stack_sig
+                                    if task is not None:
+                                        pbar.update(task, advance=1)
+                                    continue
                     if lora_sdnq.factor_candidate(module, network_layer_name, component_wanted):
                         weights_backup = getattr(module, "network_weights_backup", None)
                         if weights_backup is not None and not isinstance(weights_backup, bool):
@@ -152,6 +207,7 @@ def network_activate(include=None, exclude=None):
                         continue
                     backup_size += network_backup_weights(module, network_layer_name, component_wanted, fuse)
                     if not component_wanted:
+                        lora_stack.drop(network_layer_name) # a restored layer must leave the selection schedule
                         weights_backup = getattr(module, "network_weights_backup", None)
                         if weights_backup is None or isinstance(weights_backup, bool): # fuse mode has no tensor backup, restore stays with network_deactivate
                             if task is not None:
@@ -186,6 +242,7 @@ def network_activate(include=None, exclude=None):
     lora_sdnq.report_fallbacks()
     native_active = len(l.loaded_networks) > 0
     refused_writes = refused
+    l.last_backup_size = backup_size
     l.timer.activate += time.time() - t0
     if refused > 0:
         log.error(f'Network load: type=LoRA networks={[n.name for n in l.loaded_networks]} weights={applied_weight} bias={applied_bias} refused={refused} network partially applied')
@@ -196,6 +253,15 @@ def network_activate(include=None, exclude=None):
         sd_models.set_diffuser_offload(sd_model, op="model")
 
 
+def effective_mode():
+    """Weight-state label for load logs: backup and fuse say how touched weights restore, factor means the whole load rode the svd channel and unload just drops factors."""
+    if getattr(l, 'last_backup_size', 0) > 0:
+        return 'backup'
+    if lora_overrides.fuse_native():
+        return 'fuse'
+    return 'factor'
+
+
 def network_deactivate(include=None, exclude=None):
     if exclude is None:
         exclude = []
@@ -213,7 +279,7 @@ def network_deactivate(include=None, exclude=None):
             sd_models.disable_offload(sd_model)
             sd_models.move_model(sd_model, device=devices.cpu)
         elif shared.opts.diffusers_offload_mode == "balanced":
-            sd_model = sd_models.apply_balanced_offload(sd_model, force=True) # dispatched modules hold meta tensors backed by the offload map; rebuild them real on cpu with hooks intact before touching weights
+            sd_model = sd_models.apply_balanced_offload(sd_model, force=True, silent=True) # dispatched modules hold meta tensors backed by the offload map; rebuild them real on cpu with hooks intact before touching weights
         group_offload = shared.opts.diffusers_offload_mode == "group"
         group_stripped = {}
         modules = {}

modules/options_handler.py

@@ -17,7 +17,7 @@ if TYPE_CHECKING:
     import builtins
 
 cmd_opts = cmd_args.parse_args()
-compatibility_opts = ['clip_skip', 'uni_pc_lower_order_final', 'uni_pc_order', 'xformers_options']
+compatibility_opts = ['clip_skip', 'uni_pc_lower_order_final', 'uni_pc_order', 'xformers_options', 'lora_apply_te']
 removed_values = { # a stored choice that no longer exists is kept by validate, so it has to be rewritten or it selects nothing
     'cross_attention_optimization': (['Batch matrix-matrix', 'Dynamic Attention BMM'], 'Scaled-Dot-Product'),
 }

modules/processing_callbacks.py

@@ -17,6 +17,8 @@ def set_callbacks_p(processing):
     global p, warned # pylint: disable=global-statement
     p = processing
     warned = False
+    from modules.lora import lora_stack
+    lora_stack.reset(int(getattr(processing, 'steps', 0) or 0)) # per-pass: restore initial selections and reschedule flips before any step runs
 
 
 def prompt_callback(step, kwargs):
@@ -37,6 +39,8 @@ def prompt_callback(step, kwargs):
 def diffusers_callback_legacy(step: int, timestep: int, latents: torch.FloatTensor | np.ndarray):
     if p is None:
         return
+    from modules.lora import lora_stack
+    lora_stack.on_step(step)
     if isinstance(latents, np.ndarray): # latents from Onnx pipelines is ndarray.
         latents = torch.from_numpy(latents)
     shared.state.sampling_step = step
@@ -56,6 +60,8 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict | No
     if kwargs is None:
         kwargs = {}
     t0 = time.time()
+    from modules.lora import lora_stack
+    lora_stack.on_step(step)
 
     if shared.opts.torch_sync:
         if devices.backend == "ipex":

modules/processing_class.py

@@ -254,7 +254,6 @@ class StableDiffusionProcessing:
                  prompt_mean_norm: bool | None = None,
                  diffusers_zeros_prompt_pad: bool | None = None,
                  te_pooled_embeds: bool | None = None,
-                 lora_apply_te: bool | None = None,
                  te_complex_human_instruction: str | None = None,
                  te_use_mask: bool | None = None,
                  # generation modifier overrides (hijack)
@@ -543,7 +542,6 @@ class StableDiffusionProcessing:
         self.prompt_mean_norm = prompt_mean_norm
         self.diffusers_zeros_prompt_pad = diffusers_zeros_prompt_pad
         self.te_pooled_embeds = te_pooled_embeds
-        self.lora_apply_te = lora_apply_te
         self.te_complex_human_instruction = te_complex_human_instruction
         self.te_use_mask = te_use_mask
         # generation modifier overrides (hijack)

modules/processing_diffusers.py

@@ -149,7 +149,7 @@ def process_base(p: processing.StableDiffusionProcessing):
     if 'detailer' in p.ops:
         desc = 'Detail'
     p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts, p.network_data)
-    extra_networks.activate_filtered(p) # networks must patch weights before prompt encode so te loras affect embeds
+    extra_networks.activate(p) # networks must patch weights before prompt encode so te loras affect embeds
     base_args = set_pipeline_args(
         p=p,
         model=shared.sd_model,
@@ -313,7 +313,7 @@ def process_hires(p: processing.StableDiffusionProcessing, output):
                 prompts, p.network_data = extra_networks.parse_prompts(prompts)
                 reset_prompts = True
             if reset_prompts or ('base' in p.skip):
-                extra_networks.activate_filtered(p)
+                extra_networks.activate(p)
 
             hires_args = set_pipeline_args(
                 p=p,

modules/prompt_parser_diffusers.py

@@ -130,11 +130,8 @@ class PromptEmbedder:
         # unpack EN data in case of TE LoRA
         en_data = p.network_data
         en_data = [idx.items for item in en_data.values() for idx in item]
-        apply_te = getattr(p, 'lora_apply_te', None)
-        if apply_te is None:
-            apply_te = shared.opts.lora_apply_te
         effective_batch = 1 if self.allsame else self.batchsize
-        key = str([self.prompts, self.negative_prompts, effective_batch, self.clip_skip, self.steps, en_data, apply_te])
+        key = str([self.prompts, self.negative_prompts, effective_batch, self.clip_skip, self.steps, en_data])
         item = cache.get(key)
         if not item:
             if not any(flatten(emb) for emb in [self.prompt_embeds,

modules/sd_offload_balanced.py

@@ -14,7 +14,7 @@ import modules.sd_offload_state as s
 
 
 class OffloadHook(accelerate.hooks.ModelHook):
-    def __init__(self, checkpoint_name):
+    def __init__(self, checkpoint_name, silent=False):
         if shared.opts.diffusers_offload_max_gpu_memory > 1:
             shared.opts.diffusers_offload_max_gpu_memory = 0.75
         if shared.opts.diffusers_offload_max_cpu_memory > 1:
@@ -32,8 +32,9 @@ class OffloadHook(accelerate.hooks.ModelHook):
         self.last_pre = None
         self.last_post = None
         self.last_cls = None
-        gpu = f'{(shared.gpu_memory * shared.opts.diffusers_offload_min_gpu_memory):.2f}-{(shared.gpu_memory * shared.opts.diffusers_offload_max_gpu_memory):.2f}:{shared.gpu_memory:.2f}'
-        log.info(f'Offload: type=balanced op=init watermark={self.min_watermark}-{self.max_watermark} gpu={gpu} cpu={shared.cpu_memory:.3f} limit={shared.opts.cuda_mem_fraction:.2f} always={self.offload_always} never={self.offload_never} pre={shared.opts.diffusers_offload_pre} streams={shared.opts.diffusers_offload_streams}')
+        if not silent:
+            gpu = f'{(shared.gpu_memory * shared.opts.diffusers_offload_min_gpu_memory):.2f}-{(shared.gpu_memory * shared.opts.diffusers_offload_max_gpu_memory):.2f}:{shared.gpu_memory:.2f}'
+            log.info(f'Offload: type=balanced op=init watermark={self.min_watermark}-{self.max_watermark} gpu={gpu} cpu={shared.cpu_memory:.3f} limit={shared.opts.cuda_mem_fraction:.2f} always={self.offload_always} never={self.offload_never} pre={shared.opts.diffusers_offload_pre} streams={shared.opts.diffusers_offload_streams}')
         self.validate()
         super().__init__()
 
@@ -255,7 +256,7 @@ def apply_balanced_offload(sd_model=None, exclude: list[str] | None = None, forc
     checkpoint_name = sd_model.sd_checkpoint_info.name if getattr(sd_model, "sd_checkpoint_info", None) is not None else sd_model.__class__.__name__
     if force or (s.offload_hook_instance is None) or (s.offload_hook_instance.min_watermark != shared.opts.diffusers_offload_min_gpu_memory) or (s.offload_hook_instance.max_watermark != shared.opts.diffusers_offload_max_gpu_memory) or (checkpoint_name != s.offload_hook_instance.checkpoint_name):
         cached = False
-        s.offload_hook_instance = OffloadHook(checkpoint_name)
+        s.offload_hook_instance = OffloadHook(checkpoint_name, silent=silent)
 
     if cached and shared.opts.diffusers_offload_pre:
         s.debug_move('Offload: type=balanced op=apply skip')
@@ -277,6 +278,6 @@ def apply_balanced_offload(sd_model=None, exclude: list[str] | None = None, forc
     process_timer.add('offload', t1 - t0)
     fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access
     s.debug_move(f'Apply offload: time={t1 - t0:.2f} type=balanced fn={fn}')
-    if not cached:
+    if not cached and not silent:
         log.info(f'Model class={sd_model.__class__.__name__} modules={len(s.offload_hook_instance.offload_map)} size={s.offload_hook_instance.model_size():.3f}')
     return sd_model

modules/ui_definitions.py

@@ -708,7 +708,6 @@ def create_settings(cmd_opts):
         "lora_apply_tags": OptionInfo(0, "LoRA auto-apply tags", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}),
 
         "lora_apply_sep": OptionInfo("<h2>Apply method</h2>", "", gr.HTML),
-        "lora_apply_te": OptionInfo(False, "LoRA native apply to text encoder"),
         "lora_fuse_native": OptionInfo(True, "LoRA native fuse with model"),
         "lora_fuse_diffusers": OptionInfo(False, "LoRA diffusers fuse with model"),
 
@@ -718,6 +717,12 @@ def create_settings(cmd_opts):
         "lora_sdnq_host_calib": OptionInfo(True, "LoRA quantized host calibration"),
         "lora_sdnq_host_cache": OptionInfo(10, "LoRA quantized host cache", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1}),
 
+        "lora_stack_sep": OptionInfo("<h2>Stacking options</h2>", "", gr.HTML),
+        "lora_stack_mode": OptionInfo("sum", "LoRA stack mode", gr.Dropdown, {"choices": ["sum", "ties", "dare_ties", "dare_linear", "magnitude_prune", "klora", "estlora"]}),
+        "lora_stack_density": OptionInfo(0.5, "LoRA stack density", gr.Slider, {"minimum": 0.05, "maximum": 1.0, "step": 0.05}),
+        "lora_stack_alpha": OptionInfo(0.0, "LoRA stack ramp", gr.Slider, {"minimum": 0.0, "maximum": 3.0, "step": 0.1}),
+        "lora_stack_discrepancy": OptionInfo(0.5, "LoRA stack discrepancy", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.05}),
+
         "lora_meta_sep": OptionInfo("<h2>Metadata</h2>", "", gr.HTML),
         "lora_add_hashes_to_infotext": OptionInfo(False, "LoRA add hash info to metadata"),
     }))

scripts/xyz/xyz_grid_classes.py

@@ -25,6 +25,9 @@ from scripts.xyz.xyz_grid_shared import ( # pylint: disable=no-name-in-module, u
     list_lora,
     apply_lora,
     apply_lora_strength,
+    list_lora_blocks,
+    apply_lora_blocks,
+    format_value_trim,
     apply_te,
     apply_guidance,
     apply_styles,
@@ -110,6 +113,10 @@ class SharedSettingsStackHelper():
     todo_ratio = None
     teacache_thresh = None
     extra_networks_default_multiplier = None
+    lora_stack_mode = None
+    lora_stack_density = None
+    lora_stack_alpha = None
+    lora_stack_discrepancy = None
     disable_apply_metadata = None
     disable_apply_params = None
     sdnq_quant_mode = None
@@ -143,6 +150,10 @@ class SharedSettingsStackHelper():
         self.sd_unet = shared.opts.sd_unet
         self.sd_text_encoder = shared.opts.sd_text_encoder
         self.extra_networks_default_multiplier = shared.opts.extra_networks_default_multiplier
+        self.lora_stack_mode = shared.opts.lora_stack_mode
+        self.lora_stack_density = shared.opts.lora_stack_density
+        self.lora_stack_alpha = shared.opts.lora_stack_alpha
+        self.lora_stack_discrepancy = shared.opts.lora_stack_discrepancy
         self.teacache_thresh = shared.opts.teacache_thresh
         self.disable_apply_metadata = shared.opts.disable_apply_metadata
         self.disable_apply_params = shared.opts.disable_apply_params
@@ -156,6 +167,10 @@ class SharedSettingsStackHelper():
         shared.opts.data["disable_apply_metadata"] = self.disable_apply_metadata
         shared.opts.data["disable_apply_params"] = self.disable_apply_params
         shared.opts.data["extra_networks_default_multiplier"] = self.extra_networks_default_multiplier
+        shared.opts.data["lora_stack_mode"] = self.lora_stack_mode
+        shared.opts.data["lora_stack_density"] = self.lora_stack_density
+        shared.opts.data["lora_stack_alpha"] = self.lora_stack_alpha
+        shared.opts.data["lora_stack_discrepancy"] = self.lora_stack_discrepancy
         shared.opts.data["prompt_attention"] = self.prompt_attention
         shared.opts.data["schedulers_solver_order"] = self.schedulers_solver_order
         shared.opts.data["schedulers_sigma_adjust"] = self.schedulers_sigma_adjust
@@ -214,6 +229,11 @@ axis_options = [
     AxisOption("[Prompt] Prompt parser", str, apply_setting("prompt_attention"), choices=lambda: ["native", "compel", "xhinker", "a1111", "fixed"]),
     AxisOption("[Network] LoRA", str, apply_lora, cost=0.5, choices=list_lora),
     AxisOption("[Network] LoRA strength", float, apply_lora_strength, cost=0.6),
+    AxisOption("[Network] LoRA block weight", str, apply_lora_blocks, cost=0.6, fmt=format_value_trim, choices=list_lora_blocks),
+    AxisOption("[Network] LoRA stack mode", str, apply_setting("lora_stack_mode"), cost=0.6, choices=lambda: ["sum", "ties", "dare_ties", "dare_linear", "magnitude_prune", "klora", "estlora"]),
+    AxisOption("[Network] LoRA stack density", float, apply_setting("lora_stack_density"), cost=0.6),
+    AxisOption("[Network] LoRA stack ramp", float, apply_setting("lora_stack_alpha"), cost=0.6),
+    AxisOption("[Network] LoRA stack discrepancy", float, apply_setting("lora_stack_discrepancy"), cost=0.6),
     AxisOption("[Network] Styles", str, apply_styles, choices=lambda: [s.name for s in shared.prompt_styles.styles.values()]),
     AxisOption("[Param] Width", int, apply_field("width")),
     AxisOption("[Param] Height", int, apply_field("height")),

scripts/xyz/xyz_grid_shared.py

@@ -329,6 +329,31 @@ def apply_lora_strength(p, x, xs):
     shared.opts.data['extra_networks_default_multiplier'] = x
 
 
+def list_lora_blocks():
+    from modules.lora import lora_blocks
+    from modules.merging.merge_presets import BLOCK_WEIGHTS_PRESETS, SDXL_BLOCK_WEIGHTS_PRESETS
+    return ['None'] + list(lora_blocks.CLASSIC) + list(lora_blocks.CHAIN_NAMES) + sorted(BLOCK_WEIGHTS_PRESETS) + sorted(SDXL_BLOCK_WEIGHTS_PRESETS)
+
+
+re_lora_tag = re.compile(r'<lora:([^>]+)>')
+
+
+def apply_lora_blocks(p, x, xs):
+    x = str(x or '').strip()
+    if ':' in x or '>' in x:
+        log.error(f'XYZ grid apply LoRA block weight: value="{x}" invalid characters')
+        return
+    def rewrite(m):
+        items = [i for i in m.group(1).split(':') if not i.lower().startswith('lbw=')]
+        if x and x.lower() != 'none':
+            items.append(f'lbw={x}')
+        return '<lora:' + ':'.join(items) + '>'
+    p.prompt = re_lora_tag.sub(rewrite, p.prompt)
+    p.all_prompts = None # a populated list would shadow the edited prompt in processing
+    p.all_negative_prompts = None
+    log.debug(f'XYZ grid apply LoRA block weight: "{x}"')
+
+
 def apply_te(p, x, xs):
     shared.opts.data["sd_text_encoder"] = x
     sd_models.reload_text_encoder()
@@ -452,6 +477,13 @@ def format_value_join_list(p, opt, x):
     return ", ".join(x)
 
 
+def format_value_trim(p, opt, x):
+    x = str(x)
+    if len(x) > 40:
+        x = x[:37] + '...' # block-weight vectors would flood the grid legend
+    return f"{opt.label}: {x}"
+
+
 def do_nothing(p, x, xs):
     pass
 

ui/locale/locale_en.json

@@ -861,13 +861,16 @@
     {"id":"","label":"Load custom Diffusers pipeline","localized":"","hint":"","ui":"settings_huggingface"},
     {"id":"","label":"LoRA force reload always","localized":"","hint":"Forces LoRA networks to reload from storage on every generation, even if already cached.<br>Useful for debugging or when LoRA files are being modified externally.<br>Disable for normal use to benefit from caching.","ui":"settings_lora"},
     {"id":"","label":"LoRA load using Diffusers method","localized":"","hint":"Alternative method uses diffusers built-in LoRA capabilities instead of native SD.Next implementation (may reduce LoRA compatibility)","ui":"settings_lora"},
-    {"id":"","label":"LoRA native apply to text encoder","localized":"","hint":"","ui":"settings_lora"},
     {"id":"","label":"LoRA native fuse with model","localized":"","hint":"Merge LoRA into the model for lower memory usage.<br><br><b style=\"color: #ef4444\">Warning:</b> After removing or switching a LoRA, you may still see its style in generated images. To get a clean model, reload it from the model selector.","ui":"settings_lora"},
     {"id":"","label":"LoRA diffusers fuse with model","localized":"","hint":"Merge LoRA into the model for lower memory usage and torch.compile compatibility.<br><br><b style=\"color: #ef4444\">Warning:</b> After removing or switching a LoRA, you may still see its style in generated images. To get a clean model, reload it from the model selector.","ui":"settings_lora"},
     {"id":"","label":"LoRA quantized apply method","localized":"","hint":"How networks are applied to SDNQ-quantized model weights:<br>- <b>exact</b>: adapters are carried alongside the quantized weights at full precision; apply and removal are exact and the quantized weights are never modified. The carried factors take additional VRAM, growing with adapter rank, size and count<br>- <b>requantize</b>: adapters are merged into the quantized weights, matching the behavior of earlier releases. Uses no additional VRAM (a weight backup for network removal is held in system RAM); on models quantized below 8 bits rounding typically loses much of the adapter effect, with strong adapters retaining more<br><br>With <b>requantize</b> selected, the host rank, calibration and cache options below have no effect.<br><br>Default is <b>exact</b>.","ui":"settings_lora"},
-    {"id":"","label":"LoRA quantized host rank","localized":"","hint":"Maximum rank used to carry non-factorable adapter types (LoKR, LoHA, OFT, DoRA) on the side-channel of SDNQ models quantized below 8 bits, where merging would erase most of the adapter. Higher values keep more of the adapter at proportionally higher memory cost. Set to 0 to disable and merge into the quantized weights instead.","ui":"settings_lora"},
-    {"id":"","label":"LoRA quantized host calibration","localized":"","hint":"Collection of per-channel activation statistics during normal generations on models quantized below 8 bits, cached per checkpoint. When available, the statistics weight side-channel hosting of non-factorable adapter types toward the channels carrying the most activation energy, improving delivered fidelity at the same host rank.","ui":"settings_lora"},
-    {"id":"","label":"LoRA quantized host cache","localized":"","hint":"Disk budget in GB for caching the side-channel factors computed when hosting non-factorable adapter types on quantized models. A cached configuration skips the truncation math on reapply; least recently used entries are evicted past the budget. Set to 0 to disable.","ui":"settings_lora"},
+    {"id":"","label":"LoRA quantized host rank","localized":"","hint":"Maximum rank used to carry adapter types that are not natively low-rank (LoKR, LoHA, OFT, DoRA) alongside the quantized weights instead of merging them in.<br>Higher values retain more of the adapter at proportionally more memory. Plain LoRA files are carried exactly at their own rank.<br><br>Applies only to SDNQ models quantized below 8 bits, where merging erases most of the adapter; at 8 bits and above merging retains it and hosting is skipped.<br><br><b>0</b> disables hosting and merges every adapter into the quantized weights.<br><br>Default is <b>256</b>.","ui":"settings_lora"},
+    {"id":"","label":"LoRA quantized host calibration","localized":"","hint":"Collects per-channel activation statistics from the model's own generations and uses them to focus hosted-adapter truncation on the channels with the strongest activations.<br>Statistics accumulate in the background on models quantized below 8 bits, persist per checkpoint, and raise delivered adapter fidelity at the same <b><i>LoRA quantized host rank</i></b>, most at low ranks.<br><br>Capture is skipped while the model is compiled; previously cached statistics still apply.<br><br>Enabled by default.","ui":"settings_lora"},
+    {"id":"","label":"LoRA quantized host cache","localized":"","hint":"Disk space in GB for caching computed hosting factors.<br>A cached set skips the truncation math on the next load; least recently used entries are evicted once the budget is exceeded.<br><br><b>0</b> disables the cache.<br><br>Default is <b>10</b>.","ui":"settings_lora"},
+    {"id":"","label":"LoRA stack mode","localized":"","hint":"How multiple networks targeting the same layer are combined:<br>- <b>sum</b>: adds all contributions<br>- <b>ties</b>: keeps each network's strongest elements and merges only where signs agree<br>- <b>dare_ties</b>: randomly drops elements, rescales the survivors, then merges where signs agree<br>- <b>dare_linear</b>: randomly drops elements, rescales the survivors and sums<br>- <b>magnitude_prune</b>: keeps each network's strongest elements and sums<br>- <b>klora</b> / <b>estlora</b>: assign each layer to one of exactly two networks, the first in the prompt as subject and the second as style; <b><i>LoRA stack ramp</i></b> optionally shifts layers toward style over the sampling steps<br><br>Each layer is given to a single network at a time, so a subject and a style that both need sustained strength can end up under-applied. For reliable blending of two strong networks, <b>sum</b>, <b>ties</b> and <b>dare_ties</b> apply every network throughout and combine more fully.<br><br>Kept fractions are set by <b><i>LoRA stack density</i></b>; the subject-to-style shift by <b><i>LoRA stack ramp</i></b> and <b><i>LoRA stack discrepancy</i></b>.<br><br>Applies to the native load path; other load methods and text encoder networks always combine as <b>sum</b>. Selection modes fall back to <b>sum</b> unless exactly two networks are loaded, or when model compile is active.<br><br>Default is <b>sum</b>.","ui":"settings_lora"},
+    {"id":"","label":"LoRA stack density","localized":"","hint":"Fraction of elements each network keeps under the <b>ties</b>, <b>dare_ties</b>, <b>dare_linear</b> and <b>magnitude_prune</b> stack modes.<br>Lower values keep only the strongest contributions and reduce interference between networks at the cost of per-network detail. The dare variants drop at random and rescale the survivors to preserve expected strength.<br><br>Default is <b>0.5</b>.","ui":"settings_lora"},
+    {"id":"","label":"LoRA stack ramp","localized":"","hint":"Slope of the subject-to-style shift across the sampling steps in the <b>klora</b> and <b>estlora</b> stack modes.<br><br><b>0</b> keeps the layer assignment fixed for the whole generation: each layer stays with the network that is more salient there, which preserves the subject while the style keeps its own layers. Higher values hand layers to the style network progressively, ending in a style takeover; on few-step models the handover happens early enough to override the subject.<br><br>Default is <b>0</b>.","ui":"settings_lora"},
+    {"id":"","label":"LoRA stack discrepancy","localized":"","hint":"Stand-in for the measured style separation the <b>estlora</b> stack mode would otherwise derive from data.<br>Higher values keep layers with the subject network longer; lower values let the style network take layers earlier.<br><br>Layer scores are balanced by each network's overall strength, so a louder network does not take layers on magnitude alone.<br><br>Applies only when <b><i>LoRA stack mode</i></b> is <b>estlora</b>.<br><br>Default is <b>0.5</b>.","ui":"settings_lora"},
     {"id":"","label":"LoRA auto-apply tags","localized":"","hint":"Automatically add trigger words/tags from LoRA metadata to your prompt.<br>Set to the number of tags to auto-apply, e.g., 3 = add top 3 trigger tags.<br>Set to 0 to disable, -1 to add all available tags.","ui":"settings_lora"},
     {"id":"","label":"LoRA memory cache","localized":"","hint":"How many LoRAs to keep in network for future use before requiring reloading from storage","ui":"settings_lora"},
     {"id":"","label":"LoRA add hash info to metadata","localized":"","hint":"Include LoRA file hashes in generated image metadata.<br>Useful for reproducibility and tracking which exact LoRA versions were used.","ui":"settings_lora"},