feat(lora): host non-factorable adapters on the sdnq side-channel

Non-additive families (lokr, loha, oft, dora, full) merged into the
quantized weight and lost most of their delta on low-bit formats. On
sub-8-bit layers the set's calc_updown delta now rides the svd
side-channel as its top singular directions instead: factorable members
are subtracted out and appended exactly, so only the non-factorable
remainder is truncated. Truncation keeps the dominant part of the
effect and drops an orthogonal residual, where requantize keeps the
grid extrema and adds grid-shift noise of the delta's own magnitude;
on real lokr files retention rises from 0.04 to about 0.5 at the
default rank.

Hosted layers take no weight backup and unload bit-exactly. The svd
runs under a forked rng so generation seeds are unaffected. At 8 bits
and above requantize retains most of the delta and remains the path.
lora_sdnq_host_rank caps the hosted rank; 0 disables hosting.

modules/lora/lora_sdnq.py

@@ -21,19 +21,26 @@ fidelity floors at the compute dtype, because the dequantizer materializes
 ``base + factors`` in the result dtype and a delta below its ULP of the
 base rounds exactly as it would on an unquantized model of that dtype.
 
-Only additive low-rank modules qualify (plain LoRA: no DoRA, no CP ``mid``,
-no LyCORIS dense-bias, no ``diff_b``). Layers with any non-factorable
-contribution fall back to the dequantize-add-requantize path.
+Only additive low-rank modules ride the channel exactly (plain LoRA: no
+DoRA, no CP ``mid``, no LyCORIS dense-bias, no ``diff_b``). On sub-8-bit
+formats, sets with non-factorable contributions are hosted instead: the
+families' own ``calc_updown`` delta is truncated to its top singular
+directions and appended the same way. Truncation keeps the dominant part
+of the effect and drops an orthogonal residual, where requantize keeps
+only the grid extrema and adds grid-shift noise of the delta's own
+magnitude. At 8 bits and above requantize retains most of the delta, so
+hosting is skipped there and the requantize path remains.
 """
 
 import torch
 
-from modules import devices
+from modules import devices, shared
 from modules.lora import lora_common as l
 from modules.logger import log
 
 
 fallback_layers: list[str] = []
+hosted_layers: list[tuple[str, float]] = []
 
 
 def get_module_factors(module, device, dtype, original_shape=None):
@@ -126,7 +133,6 @@ def apply_factors(self, network_layer_name, wanted_names, use_previous=False):
         return changed
 
     deq = self.sdnq_dequantizer
-    device = self.scale.device
     dtype = deq.result_dtype
     loaded = l.loaded_networks if not use_previous else l.previously_loaded_networks
     ups, downs = [], []
@@ -144,7 +150,15 @@ def apply_factors(self, network_layer_name, wanted_names, use_previous=False):
         downs.append(down)
     if not ups:
         return changed
+    append_factors(self, ups, downs)
+    return True
 
+
+def append_factors(self, ups, downs):
+    """Concatenate ``[out, r]`` / ``[r, in]`` factor pairs onto the layer's svd channel and stash the originals."""
+    deq = self.sdnq_dequantizer
+    device = self.scale.device
+    dtype = deq.result_dtype
     orig_up, orig_down = self.svd_up, self.svd_down
     if deq.use_quantized_matmul:
         # matmul layout stores factors transposed: svd_up [r, out], svd_down [in, r]
@@ -157,10 +171,76 @@ def apply_factors(self, network_layer_name, wanted_names, use_previous=False):
         parts_down = ([orig_down.to(device=devices.device, dtype=dtype)] if orig_down is not None else []) + downs
         new_up = torch.cat(parts_up, dim=1).contiguous()
         new_down = torch.cat(parts_down, dim=0).contiguous()
-
     self.sdnq_lora_svd_stash = (orig_up, orig_down)
     self.svd_up = torch.nn.Parameter(new_up.to(device=device), requires_grad=False)
     self.svd_down = torch.nn.Parameter(new_down.to(device=device), requires_grad=False)
+
+
+def host_candidate(self, network_layer_name, wanted_names, use_previous=False):
+    """True when a non-factorable set on this layer should be hosted as a truncated svd."""
+    if int(getattr(shared.opts, 'lora_sdnq_host_rank', 0) or 0) <= 0:
+        return False
+    if getattr(self, 'sdnq_dequantizer', None) is None or self.__class__.__name__ != 'SDNQLinear':
+        return False
+    if wanted_names == ():
+        return False
+    from sdnq.common import dtype_dict
+    if dtype_dict[self.sdnq_dequantizer.weights_dtype]['num_bits'] >= 8:
+        return False # requantize retains most of the delta at 8 bits and above; truncation would lose more than it saves
+    loaded = l.loaded_networks if not use_previous else l.previously_loaded_networks
+    return any(net.modules.get(network_layer_name, None) is not None for net in loaded)
+
+
+def apply_hosted(self, network_layer_name, updown, wanted_names, use_previous=False):
+    """Host a set's delta on the svd channel: exact factors for factorable
+    members, the top-k singular directions of the remainder for the rest.
+
+    The delta comes from the families' own ``calc_updown``, so every family
+    and scaling quirk is included; factorable members are subtracted out and
+    appended exactly so they never compete with the hosted remainder for
+    rank. Returns None when the delta cannot ride the channel (wrong shape);
+    the caller falls back to requantize.
+    """
+    from sdnq.quant_utils import rotate_hadamard
+
+    deq = self.sdnq_dequantizer
+    changed = remove_factors(self)
+    if wanted_names == ():
+        return changed
+    if updown is None or updown.ndim != 2 or tuple(updown.shape) != tuple(deq.original_shape):
+        return None
+    dtype = deq.result_dtype
+    D = updown.detach().to(devices.device, torch.float32)
+
+    ups, downs = [], []
+    loaded = l.loaded_networks if not use_previous else l.previously_loaded_networks
+    for net in loaded:
+        module = net.modules.get(network_layer_name, None)
+        if module is None:
+            continue
+        factors = get_module_factors(module, devices.device, dtype, original_shape=deq.original_shape)
+        if factors is None:
+            continue
+        up_eff, down = factors
+        D = D.sub_(up_eff.to(torch.float32) @ down.to(torch.float32)) # factorable members ride exactly; host only the remainder
+        if deq.use_hadamard:
+            down = rotate_hadamard(down.to(dtype=torch.float32), group_size=deq.hadamard_group_size).to(dtype=dtype)
+        ups.append(up_eff)
+        downs.append(down)
+
+    cap = int(shared.opts.lora_sdnq_host_rank)
+    q = min(cap, *D.shape)
+    # svd_lowrank draws random projections; fork so user generation seeds are untouched and re-applies are deterministic
+    with torch.random.fork_rng(devices=[D.device] if D.device.type == 'cuda' else []):
+        torch.manual_seed(0)
+        U, S, V = torch.svd_lowrank(D, q=q, niter=2)
+    energy = float(S.square().sum() / D.square().sum().clamp(min=1e-30))
+    up_h = (U * S).to(dtype=dtype)
+    down_h = V.t()
+    if deq.use_hadamard:
+        down_h = rotate_hadamard(down_h, group_size=deq.hadamard_group_size)
+    append_factors(self, ups + [up_h], downs + [down_h.to(dtype=dtype)])
+    hosted_layers.append((network_layer_name, energy))
     return True
 
 
@@ -171,6 +251,13 @@ def note_fallback(self, network_layer_name):
 
 
 def report_fallbacks():
+    if len(hosted_layers) > 0:
+        energies = sorted(e for _name, e in hosted_layers)
+        median = energies[len(energies) // 2]
+        log.info(f'Network load: type=LoRA quant=sdnq hosted={len(hosted_layers)} rank={int(shared.opts.lora_sdnq_host_rank)} energy={median:.2f} min={energies[0]:.2f} non-factorable networks hosted on the svd side-channel')
+        if l.debug:
+            log.debug(f'Network load: type=LoRA quant=sdnq hosted={[(n, round(e, 3)) for n, e in hosted_layers[:8]]}{"..." if len(hosted_layers) > 8 else ""}')
+    hosted_layers.clear()
     if len(fallback_layers) > 0:
         log.warning(f'Network load: type=LoRA quant=sdnq layers={len(fallback_layers)} non-factorable networks requantized in place (reduced fidelity on quantized weights)')
         if l.debug:

modules/lora/networks.py

@@ -91,6 +91,7 @@ def network_activate(include=None, exclude=None):
             wanted_names = tuple((x.name, x.te_multiplier, x.unet_multiplier, x.dyn_dim) for x in l.loaded_networks) if len(l.loaded_networks) > 0 else ()
             applied_layers.clear()
             lora_sdnq.fallback_layers.clear() # a raise mid-pass leaves stale entries behind
+            lora_sdnq.hosted_layers.clear()
             backup_size = 0
             for component in modules.keys():
                 component_wanted = wanted_names if component in components else ()
@@ -109,7 +110,7 @@ def network_activate(include=None, exclude=None):
                         if weights_backup is not None and not isinstance(weights_backup, bool):
                             network_apply_weights(module, None, None, device=device) # an earlier non-factorable set requantized this layer, restore the pristine base before attaching factors
                         applied = lora_sdnq.apply_factors(module, network_layer_name, component_wanted)
-                        if applied is not None: # exact path took the layer; None falls through to requantize
+                        if applied is not None: # exact path took the layer; None falls through to hosting or requantize
                             if applied and component_wanted:
                                 applied_layers.append(network_layer_name)
                                 applied_weight += 1
@@ -117,6 +118,25 @@ def network_activate(include=None, exclude=None):
                             if task is not None:
                                 pbar.update(task, advance=1)
                             continue
+                    if lora_sdnq.host_candidate(module, network_layer_name, component_wanted):
+                        weights_backup = getattr(module, "network_weights_backup", None)
+                        if weights_backup is not None and not isinstance(weights_backup, bool):
+                            network_apply_weights(module, None, None, device=device) # the hosted delta is measured against the pristine base
+                        batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name, elimit=elimit)
+                        if batch_ex_bias is None: # bias deltas need the plain path; weight-only sets ride the side-channel without a weight backup
+                            hosted = lora_sdnq.apply_hosted(module, network_layer_name, batch_updown, component_wanted)
+                            if hosted is not None:
+                                if hosted and component_wanted:
+                                    applied_layers.append(network_layer_name)
+                                    applied_weight += 1
+                                module.network_current_names = component_wanted
+                                batch_updown, batch_ex_bias = None, None
+                                del batch_updown, batch_ex_bias
+                                if task is not None:
+                                    pbar.update(task, advance=1)
+                                continue
+                        batch_updown, batch_ex_bias = None, None
+                        del batch_updown, batch_ex_bias
                     backup_size += network_backup_weights(module, network_layer_name, component_wanted, fuse)
                     if not component_wanted:
                         weights_backup = getattr(module, "network_weights_backup", None)

modules/ui_definitions.py

@@ -684,6 +684,7 @@ def create_settings(cmd_opts):
         "lora_apply_te": OptionInfo(False, "LoRA native apply to text encoder"),
         "lora_fuse_native": OptionInfo(True, "LoRA native fuse with model"),
         "lora_fuse_diffusers": OptionInfo(False, "LoRA diffusers fuse with model"),
+        "lora_sdnq_host_rank": OptionInfo(256, "LoRA quantized host rank", gr.Slider, {"minimum": 0, "maximum": 1024, "step": 32}),
         "lora_apply_tags": OptionInfo(0, "LoRA auto-apply tags", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}),
         "lora_in_memory_limit": OptionInfo(1, "LoRA memory cache", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1}),
         "lora_add_hashes_to_infotext": OptionInfo(False, "LoRA add hash info to metadata"),

test/test-sdnq-lora-factors.py

@@ -23,6 +23,11 @@ for the per-model analyzer):
 - Robustness: factor removal restores onto the layer's current device after
   an offload-style move, and a shape-mismatched network stacked onto a
   factor-mode layer downgrades to the legacy path instead of raising.
+- Hosting: on sub-8-bit layers, non-factorable sets ride the side-channel as
+  a truncated svd of their calc_updown delta: low-rank content survives
+  whole, dense content beats the requantize floor by a wide margin, int8
+  and rank 0 keep the requantize path, removal stays bit-exact, and the
+  svd's random projections never touch the generation rng stream.
 
 All tensors are synthetic; no model files or running server required.
 
@@ -488,7 +493,7 @@ def test_mixed_family_transition_restores_base():
         real_report()
     lora_sdnq.report_fallbacks = capture_report
     try:
-        with mock_model(lin=layer, bystander=bystander):
+        with host_rank(0), mock_model(lin=layer, bystander=bystander): # pins the requantize fallback; hosted transitions are covered in the hosting category
             Wdq0 = dq(layer)
             activate(net_plain)
             assert hasattr(layer, 'sdnq_lora_svd_stash'), 'plain set must take the factor path'
@@ -525,7 +530,7 @@ def test_partial_coverage_layers_stay_independent():
     A2, B2, _ = make_delta(seed=5, sigma=3e-3)
     net_dora = make_net('dorafar', layer_dora, A2, B2, dora=True)
 
-    with mock_model(lin=layer_plain, other=layer_dora):
+    with host_rank(0), mock_model(lin=layer_plain, other=layer_dora): # pins the requantize fallback for the non-factorable layer
         Wdq0, Wdq0_dora = dq(layer_plain), dq(layer_dora)
         activate(net_plain, net_dora)
         assert hasattr(layer_plain, 'sdnq_lora_svd_stash') and getattr(layer_plain, 'network_weights_backup', None) is None, 'plain layer must stay on the factor path'
@@ -538,6 +543,111 @@ def test_partial_coverage_layers_stay_independent():
     return True
 
 
+CAT_HOST = category('hosting')
+
+
+@contextmanager
+def host_rank(rank):
+    old = getattr(shared.opts, 'lora_sdnq_host_rank', 0)
+    shared.opts.lora_sdnq_host_rank = rank
+    try:
+        yield
+    finally:
+        shared.opts.lora_sdnq_host_rank = old
+
+
+def make_dense_net(name, layer, D):
+    """A full-family (dense diff) network module: non-factorable by construction."""
+    from modules.lora import network_full
+    net = network.Network(name, MockNOD(name))
+    net.te_multiplier = 1.0
+    net.unet_multiplier = [1.0] * 3
+    nw = network.NetworkWeights(network_key=layer.network_layer_name, sd_key=layer.network_layer_name,
+                                w={'diff': D.cpu()}, sd_module=layer)
+    net.modules[layer.network_layer_name] = network_full.NetworkModuleFull(net, nw)
+    return net
+
+
+def test_hosted_low_rank_delta_is_kept():
+    layer = build_layer('uint4')
+    _A, _B, D = make_delta(sigma=3e-3)
+    net = make_dense_net('densenet', layer, D) # low-rank content in a non-factorable container
+    with host_rank(64), mock_model(lin=layer):
+        Wdq0 = dq(layer)
+        activate(net)
+        assert hasattr(layer, 'sdnq_lora_svd_stash'), 'hosted set must ride the side-channel'
+        assert getattr(layer, 'network_weights_backup', None) is None, 'hosted layers must not take a weight backup'
+        rho = rho_of(dq(layer) - Wdq0, D)
+        assert rho > 0.95, f'rank-8 delta under cap 64 must be kept nearly whole: rho={rho:.4f}'
+        activate()
+        assert torch.equal(dq(layer), Wdq0), 'unload must restore bit-exact'
+    return True
+
+
+def test_hosted_dense_delta_beats_requant():
+    layer = build_layer('uint4')
+    torch.manual_seed(3)
+    D = torch.randn(OUT_F, IN_F, device=DEVICE) * 3e-4 # full-rank, sub-step: requant erases it
+    requant_rho = rho_of(requant_effective(layer, D), D)
+    net = make_dense_net('densefull', layer, D)
+    with host_rank(256), mock_model(lin=layer):
+        Wdq0 = dq(layer)
+        activate(net)
+        hosted_rho = rho_of(dq(layer) - Wdq0, D)
+        assert hosted_rho > 0.4, f'hosted rho={hosted_rho:.3f}'
+        assert hosted_rho > requant_rho + 0.3, f'hosting must beat requant by a wide margin: {hosted_rho:.3f} vs {requant_rho:.3f}'
+        activate()
+        assert torch.equal(dq(layer), Wdq0)
+    return True
+
+
+def test_hosted_skips_int8():
+    layer = build_layer('int8')
+    _A, _B, D = make_delta(sigma=3e-3)
+    net = make_dense_net('int8net', layer, D)
+    with host_rank(256), mock_model(lin=layer):
+        activate(net)
+        assert not hasattr(layer, 'sdnq_lora_svd_stash'), 'int8 must keep the requantize path'
+        assert isinstance(getattr(layer, 'network_weights_backup', None), torch.Tensor), 'int8 fallback must take the backup'
+        activate()
+    return True
+
+
+def test_hosted_disabled_by_option():
+    layer = build_layer('uint4')
+    _A, _B, D = make_delta(sigma=3e-3)
+    net = make_dense_net('offnet', layer, D)
+    with host_rank(0), mock_model(lin=layer):
+        activate(net)
+        assert not hasattr(layer, 'sdnq_lora_svd_stash'), 'rank 0 must disable hosting'
+        activate()
+    return True
+
+
+def test_hosted_transitions_and_rng_isolation():
+    layer = build_layer('uint4')
+    A, B, D = make_delta()
+    net_plain = make_net('plainh', layer, A, B)
+    _A2, _B2, D2 = make_delta(seed=9, sigma=3e-3)
+    net_dense = make_dense_net('denseh', layer, D2)
+    with host_rank(256), mock_model(lin=layer):
+        Wdq0 = dq(layer)
+        rng0 = torch.cuda.get_rng_state() if DEVICE.type == 'cuda' else torch.get_rng_state()
+        activate(net_dense) # hosted
+        rng1 = torch.cuda.get_rng_state() if DEVICE.type == 'cuda' else torch.get_rng_state()
+        assert torch.equal(rng0, rng1), 'hosting must not consume the generation rng stream'
+        assert hasattr(layer, 'sdnq_lora_svd_stash')
+        activate(net_plain) # exact replaces hosted
+        rho = rho_of(dq(layer) - Wdq0, D)
+        assert rho > 0.99, f'exact set after hosted set: rho={rho:.4f}'
+        activate(net_plain, net_dense) # mixed set hosts the combined delta
+        rho_mix = rho_of(dq(layer) - Wdq0, D + D2)
+        assert rho_mix > 0.9, f'mixed hosted rho={rho_mix:.4f}'
+        activate()
+        assert torch.equal(dq(layer), Wdq0), 'unload must restore bit-exact'
+    return True
+
+
 CAT_ROBUST = category('robustness')
 
 
@@ -571,7 +681,7 @@ def test_stacked_shape_mismatch_falls_back():
     prev_enl = l_common.extra_network_lora
     l_common.extra_network_lora = SimpleNamespace(errors={}) # the error path reports through the extra-networks registry
     try:
-        with mock_model(lin=layer):
+        with host_rank(0), mock_model(lin=layer):
             Wdq0 = dq(layer)
             activate(net_good)
             assert hasattr(layer, 'sdnq_lora_svd_stash')
@@ -601,6 +711,10 @@ def run_tests():
     log.warning('=== Set transitions ===')
     for fn in [test_mixed_family_transition_restores_base, test_partial_coverage_layers_stay_independent]:
         run_test(CAT_TRANS, fn)
+    log.warning('=== Hosting ===')
+    for fn in [test_hosted_low_rank_delta_is_kept, test_hosted_dense_delta_beats_requant, test_hosted_skips_int8,
+               test_hosted_disabled_by_option, test_hosted_transitions_and_rng_isolation]:
+        run_test(CAT_HOST, fn)
     log.warning('=== Robustness ===')
     for fn in [test_remove_factors_after_device_move, test_stacked_shape_mismatch_falls_back]:
         run_test(CAT_ROBUST, fn)

ui/locale/locale_en.json

@@ -862,6 +862,7 @@
     {"id":"","label":"LoRA native apply to text encoder","localized":"","hint":"","ui":"settings_extra_networks"},
     {"id":"","label":"LoRA native fuse with model","localized":"","hint":"Merge LoRA into the model for lower memory usage.<br><br><b style=\"color: #ef4444\">Warning:</b> After removing or switching a LoRA, you may still see its style in generated images. To get a clean model, reload it from the model selector.","ui":"settings_extra_networks"},
     {"id":"","label":"LoRA diffusers fuse with model","localized":"","hint":"Merge LoRA into the model for lower memory usage and torch.compile compatibility.<br><br><b style=\"color: #ef4444\">Warning:</b> After removing or switching a LoRA, you may still see its style in generated images. To get a clean model, reload it from the model selector.","ui":"settings_extra_networks"},
+    {"id":"","label":"LoRA quantized host rank","localized":"","hint":"Maximum rank used to carry non-factorable adapter types (LoKR, LoHA, OFT, DoRA) on the side-channel of SDNQ models quantized below 8 bits, where merging would erase most of the adapter. Higher values keep more of the adapter at proportionally higher memory cost. Set to 0 to disable and merge into the quantized weights instead.","ui":"settings_extra_networks"},
     {"id":"","label":"LoRA auto-apply tags","localized":"","hint":"Automatically add trigger words/tags from LoRA metadata to your prompt.<br>Set to the number of tags to auto-apply, e.g., 3 = add top 3 trigger tags.<br>Set to 0 to disable, -1 to add all available tags.","ui":"settings_extra_networks"},
     {"id":"","label":"LoRA memory cache","localized":"","hint":"How many LoRAs to keep in network for future use before requiring reloading from storage","ui":"settings_extra_networks"},
     {"id":"","label":"LoRA add hash info to metadata","localized":"","hint":"Include LoRA file hashes in generated image metadata.<br>Useful for reproducibility and tracking which exact LoRA versions were used.","ui":"settings_extra_networks"},