diff --git a/modules/lora/lora_sdnq.py b/modules/lora/lora_sdnq.py
index d07c0db8a..c2a968630 100644
--- a/modules/lora/lora_sdnq.py
+++ b/modules/lora/lora_sdnq.py
@@ -21,19 +21,26 @@ fidelity floors at the compute dtype, because the dequantizer materializes
``base + factors`` in the result dtype and a delta below its ULP of the
base rounds exactly as it would on an unquantized model of that dtype.
-Only additive low-rank modules qualify (plain LoRA: no DoRA, no CP ``mid``,
-no LyCORIS dense-bias, no ``diff_b``). Layers with any non-factorable
-contribution fall back to the dequantize-add-requantize path.
+Only additive low-rank modules ride the channel exactly (plain LoRA: no
+DoRA, no CP ``mid``, no LyCORIS dense-bias, no ``diff_b``). On sub-8-bit
+formats, sets with non-factorable contributions are hosted instead: the
+families' own ``calc_updown`` delta is truncated to its top singular
+directions and appended the same way. Truncation keeps the dominant part
+of the effect and drops an orthogonal residual, where requantize keeps
+only the grid extrema and adds grid-shift noise of the delta's own
+magnitude. At 8 bits and above requantize retains most of the delta, so
+hosting is skipped there and the requantize path remains.
"""
import torch
-from modules import devices
+from modules import devices, shared
from modules.lora import lora_common as l
from modules.logger import log
fallback_layers: list[str] = []
+hosted_layers: list[tuple[str, float]] = []
def get_module_factors(module, device, dtype, original_shape=None):
@@ -126,7 +133,6 @@ def apply_factors(self, network_layer_name, wanted_names, use_previous=False):
return changed
deq = self.sdnq_dequantizer
- device = self.scale.device
dtype = deq.result_dtype
loaded = l.loaded_networks if not use_previous else l.previously_loaded_networks
ups, downs = [], []
@@ -144,7 +150,15 @@ def apply_factors(self, network_layer_name, wanted_names, use_previous=False):
downs.append(down)
if not ups:
return changed
+ append_factors(self, ups, downs)
+ return True
+
+def append_factors(self, ups, downs):
+ """Concatenate ``[out, r]`` / ``[r, in]`` factor pairs onto the layer's svd channel and stash the originals."""
+ deq = self.sdnq_dequantizer
+ device = self.scale.device
+ dtype = deq.result_dtype
orig_up, orig_down = self.svd_up, self.svd_down
if deq.use_quantized_matmul:
# matmul layout stores factors transposed: svd_up [r, out], svd_down [in, r]
@@ -157,10 +171,76 @@ def apply_factors(self, network_layer_name, wanted_names, use_previous=False):
parts_down = ([orig_down.to(device=devices.device, dtype=dtype)] if orig_down is not None else []) + downs
new_up = torch.cat(parts_up, dim=1).contiguous()
new_down = torch.cat(parts_down, dim=0).contiguous()
-
self.sdnq_lora_svd_stash = (orig_up, orig_down)
self.svd_up = torch.nn.Parameter(new_up.to(device=device), requires_grad=False)
self.svd_down = torch.nn.Parameter(new_down.to(device=device), requires_grad=False)
+
+
+def host_candidate(self, network_layer_name, wanted_names, use_previous=False):
+ """True when a non-factorable set on this layer should be hosted as a truncated svd."""
+ if int(getattr(shared.opts, 'lora_sdnq_host_rank', 0) or 0) <= 0:
+ return False
+ if getattr(self, 'sdnq_dequantizer', None) is None or self.__class__.__name__ != 'SDNQLinear':
+ return False
+ if wanted_names == ():
+ return False
+ from sdnq.common import dtype_dict
+ if dtype_dict[self.sdnq_dequantizer.weights_dtype]['num_bits'] >= 8:
+ return False # requantize retains most of the delta at 8 bits and above; truncation would lose more than it saves
+ loaded = l.loaded_networks if not use_previous else l.previously_loaded_networks
+ return any(net.modules.get(network_layer_name, None) is not None for net in loaded)
+
+
+def apply_hosted(self, network_layer_name, updown, wanted_names, use_previous=False):
+ """Host a set's delta on the svd channel: exact factors for factorable
+ members, the top-k singular directions of the remainder for the rest.
+
+ The delta comes from the families' own ``calc_updown``, so every family
+ and scaling quirk is included; factorable members are subtracted out and
+ appended exactly so they never compete with the hosted remainder for
+ rank. Returns None when the delta cannot ride the channel (wrong shape);
+ the caller falls back to requantize.
+ """
+ from sdnq.quant_utils import rotate_hadamard
+
+ deq = self.sdnq_dequantizer
+ changed = remove_factors(self)
+ if wanted_names == ():
+ return changed
+ if updown is None or updown.ndim != 2 or tuple(updown.shape) != tuple(deq.original_shape):
+ return None
+ dtype = deq.result_dtype
+ D = updown.detach().to(devices.device, torch.float32)
+
+ ups, downs = [], []
+ loaded = l.loaded_networks if not use_previous else l.previously_loaded_networks
+ for net in loaded:
+ module = net.modules.get(network_layer_name, None)
+ if module is None:
+ continue
+ factors = get_module_factors(module, devices.device, dtype, original_shape=deq.original_shape)
+ if factors is None:
+ continue
+ up_eff, down = factors
+ D = D.sub_(up_eff.to(torch.float32) @ down.to(torch.float32)) # factorable members ride exactly; host only the remainder
+ if deq.use_hadamard:
+ down = rotate_hadamard(down.to(dtype=torch.float32), group_size=deq.hadamard_group_size).to(dtype=dtype)
+ ups.append(up_eff)
+ downs.append(down)
+
+ cap = int(shared.opts.lora_sdnq_host_rank)
+ q = min(cap, *D.shape)
+ # svd_lowrank draws random projections; fork so user generation seeds are untouched and re-applies are deterministic
+ with torch.random.fork_rng(devices=[D.device] if D.device.type == 'cuda' else []):
+ torch.manual_seed(0)
+ U, S, V = torch.svd_lowrank(D, q=q, niter=2)
+ energy = float(S.square().sum() / D.square().sum().clamp(min=1e-30))
+ up_h = (U * S).to(dtype=dtype)
+ down_h = V.t()
+ if deq.use_hadamard:
+ down_h = rotate_hadamard(down_h, group_size=deq.hadamard_group_size)
+ append_factors(self, ups + [up_h], downs + [down_h.to(dtype=dtype)])
+ hosted_layers.append((network_layer_name, energy))
return True
@@ -171,6 +251,13 @@ def note_fallback(self, network_layer_name):
def report_fallbacks():
+ if len(hosted_layers) > 0:
+ energies = sorted(e for _name, e in hosted_layers)
+ median = energies[len(energies) // 2]
+ log.info(f'Network load: type=LoRA quant=sdnq hosted={len(hosted_layers)} rank={int(shared.opts.lora_sdnq_host_rank)} energy={median:.2f} min={energies[0]:.2f} non-factorable networks hosted on the svd side-channel')
+ if l.debug:
+ log.debug(f'Network load: type=LoRA quant=sdnq hosted={[(n, round(e, 3)) for n, e in hosted_layers[:8]]}{"..." if len(hosted_layers) > 8 else ""}')
+ hosted_layers.clear()
if len(fallback_layers) > 0:
log.warning(f'Network load: type=LoRA quant=sdnq layers={len(fallback_layers)} non-factorable networks requantized in place (reduced fidelity on quantized weights)')
if l.debug:
diff --git a/modules/lora/networks.py b/modules/lora/networks.py
index c472da4ea..d027a5f81 100644
--- a/modules/lora/networks.py
+++ b/modules/lora/networks.py
@@ -91,6 +91,7 @@ def network_activate(include=None, exclude=None):
wanted_names = tuple((x.name, x.te_multiplier, x.unet_multiplier, x.dyn_dim) for x in l.loaded_networks) if len(l.loaded_networks) > 0 else ()
applied_layers.clear()
lora_sdnq.fallback_layers.clear() # a raise mid-pass leaves stale entries behind
+ lora_sdnq.hosted_layers.clear()
backup_size = 0
for component in modules.keys():
component_wanted = wanted_names if component in components else ()
@@ -109,7 +110,7 @@ def network_activate(include=None, exclude=None):
if weights_backup is not None and not isinstance(weights_backup, bool):
network_apply_weights(module, None, None, device=device) # an earlier non-factorable set requantized this layer, restore the pristine base before attaching factors
applied = lora_sdnq.apply_factors(module, network_layer_name, component_wanted)
- if applied is not None: # exact path took the layer; None falls through to requantize
+ if applied is not None: # exact path took the layer; None falls through to hosting or requantize
if applied and component_wanted:
applied_layers.append(network_layer_name)
applied_weight += 1
@@ -117,6 +118,25 @@ def network_activate(include=None, exclude=None):
if task is not None:
pbar.update(task, advance=1)
continue
+ if lora_sdnq.host_candidate(module, network_layer_name, component_wanted):
+ weights_backup = getattr(module, "network_weights_backup", None)
+ if weights_backup is not None and not isinstance(weights_backup, bool):
+ network_apply_weights(module, None, None, device=device) # the hosted delta is measured against the pristine base
+ batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name, elimit=elimit)
+ if batch_ex_bias is None: # bias deltas need the plain path; weight-only sets ride the side-channel without a weight backup
+ hosted = lora_sdnq.apply_hosted(module, network_layer_name, batch_updown, component_wanted)
+ if hosted is not None:
+ if hosted and component_wanted:
+ applied_layers.append(network_layer_name)
+ applied_weight += 1
+ module.network_current_names = component_wanted
+ batch_updown, batch_ex_bias = None, None
+ del batch_updown, batch_ex_bias
+ if task is not None:
+ pbar.update(task, advance=1)
+ continue
+ batch_updown, batch_ex_bias = None, None
+ del batch_updown, batch_ex_bias
backup_size += network_backup_weights(module, network_layer_name, component_wanted, fuse)
if not component_wanted:
weights_backup = getattr(module, "network_weights_backup", None)
diff --git a/modules/ui_definitions.py b/modules/ui_definitions.py
index ed0dab9c6..8e9284566 100644
--- a/modules/ui_definitions.py
+++ b/modules/ui_definitions.py
@@ -684,6 +684,7 @@ def create_settings(cmd_opts):
"lora_apply_te": OptionInfo(False, "LoRA native apply to text encoder"),
"lora_fuse_native": OptionInfo(True, "LoRA native fuse with model"),
"lora_fuse_diffusers": OptionInfo(False, "LoRA diffusers fuse with model"),
+ "lora_sdnq_host_rank": OptionInfo(256, "LoRA quantized host rank", gr.Slider, {"minimum": 0, "maximum": 1024, "step": 32}),
"lora_apply_tags": OptionInfo(0, "LoRA auto-apply tags", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}),
"lora_in_memory_limit": OptionInfo(1, "LoRA memory cache", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1}),
"lora_add_hashes_to_infotext": OptionInfo(False, "LoRA add hash info to metadata"),
diff --git a/test/test-sdnq-lora-factors.py b/test/test-sdnq-lora-factors.py
index 4257f35fd..8514e9632 100644
--- a/test/test-sdnq-lora-factors.py
+++ b/test/test-sdnq-lora-factors.py
@@ -23,6 +23,11 @@ for the per-model analyzer):
- Robustness: factor removal restores onto the layer's current device after
an offload-style move, and a shape-mismatched network stacked onto a
factor-mode layer downgrades to the legacy path instead of raising.
+- Hosting: on sub-8-bit layers, non-factorable sets ride the side-channel as
+ a truncated svd of their calc_updown delta: low-rank content survives
+ whole, dense content beats the requantize floor by a wide margin, int8
+ and rank 0 keep the requantize path, removal stays bit-exact, and the
+ svd's random projections never touch the generation rng stream.
All tensors are synthetic; no model files or running server required.
@@ -488,7 +493,7 @@ def test_mixed_family_transition_restores_base():
real_report()
lora_sdnq.report_fallbacks = capture_report
try:
- with mock_model(lin=layer, bystander=bystander):
+ with host_rank(0), mock_model(lin=layer, bystander=bystander): # pins the requantize fallback; hosted transitions are covered in the hosting category
Wdq0 = dq(layer)
activate(net_plain)
assert hasattr(layer, 'sdnq_lora_svd_stash'), 'plain set must take the factor path'
@@ -525,7 +530,7 @@ def test_partial_coverage_layers_stay_independent():
A2, B2, _ = make_delta(seed=5, sigma=3e-3)
net_dora = make_net('dorafar', layer_dora, A2, B2, dora=True)
- with mock_model(lin=layer_plain, other=layer_dora):
+ with host_rank(0), mock_model(lin=layer_plain, other=layer_dora): # pins the requantize fallback for the non-factorable layer
Wdq0, Wdq0_dora = dq(layer_plain), dq(layer_dora)
activate(net_plain, net_dora)
assert hasattr(layer_plain, 'sdnq_lora_svd_stash') and getattr(layer_plain, 'network_weights_backup', None) is None, 'plain layer must stay on the factor path'
@@ -538,6 +543,111 @@ def test_partial_coverage_layers_stay_independent():
return True
+CAT_HOST = category('hosting')
+
+
+@contextmanager
+def host_rank(rank):
+ old = getattr(shared.opts, 'lora_sdnq_host_rank', 0)
+ shared.opts.lora_sdnq_host_rank = rank
+ try:
+ yield
+ finally:
+ shared.opts.lora_sdnq_host_rank = old
+
+
+def make_dense_net(name, layer, D):
+ """A full-family (dense diff) network module: non-factorable by construction."""
+ from modules.lora import network_full
+ net = network.Network(name, MockNOD(name))
+ net.te_multiplier = 1.0
+ net.unet_multiplier = [1.0] * 3
+ nw = network.NetworkWeights(network_key=layer.network_layer_name, sd_key=layer.network_layer_name,
+ w={'diff': D.cpu()}, sd_module=layer)
+ net.modules[layer.network_layer_name] = network_full.NetworkModuleFull(net, nw)
+ return net
+
+
+def test_hosted_low_rank_delta_is_kept():
+ layer = build_layer('uint4')
+ _A, _B, D = make_delta(sigma=3e-3)
+ net = make_dense_net('densenet', layer, D) # low-rank content in a non-factorable container
+ with host_rank(64), mock_model(lin=layer):
+ Wdq0 = dq(layer)
+ activate(net)
+ assert hasattr(layer, 'sdnq_lora_svd_stash'), 'hosted set must ride the side-channel'
+ assert getattr(layer, 'network_weights_backup', None) is None, 'hosted layers must not take a weight backup'
+ rho = rho_of(dq(layer) - Wdq0, D)
+ assert rho > 0.95, f'rank-8 delta under cap 64 must be kept nearly whole: rho={rho:.4f}'
+ activate()
+ assert torch.equal(dq(layer), Wdq0), 'unload must restore bit-exact'
+ return True
+
+
+def test_hosted_dense_delta_beats_requant():
+ layer = build_layer('uint4')
+ torch.manual_seed(3)
+ D = torch.randn(OUT_F, IN_F, device=DEVICE) * 3e-4 # full-rank, sub-step: requant erases it
+ requant_rho = rho_of(requant_effective(layer, D), D)
+ net = make_dense_net('densefull', layer, D)
+ with host_rank(256), mock_model(lin=layer):
+ Wdq0 = dq(layer)
+ activate(net)
+ hosted_rho = rho_of(dq(layer) - Wdq0, D)
+ assert hosted_rho > 0.4, f'hosted rho={hosted_rho:.3f}'
+ assert hosted_rho > requant_rho + 0.3, f'hosting must beat requant by a wide margin: {hosted_rho:.3f} vs {requant_rho:.3f}'
+ activate()
+ assert torch.equal(dq(layer), Wdq0)
+ return True
+
+
+def test_hosted_skips_int8():
+ layer = build_layer('int8')
+ _A, _B, D = make_delta(sigma=3e-3)
+ net = make_dense_net('int8net', layer, D)
+ with host_rank(256), mock_model(lin=layer):
+ activate(net)
+ assert not hasattr(layer, 'sdnq_lora_svd_stash'), 'int8 must keep the requantize path'
+ assert isinstance(getattr(layer, 'network_weights_backup', None), torch.Tensor), 'int8 fallback must take the backup'
+ activate()
+ return True
+
+
+def test_hosted_disabled_by_option():
+ layer = build_layer('uint4')
+ _A, _B, D = make_delta(sigma=3e-3)
+ net = make_dense_net('offnet', layer, D)
+ with host_rank(0), mock_model(lin=layer):
+ activate(net)
+ assert not hasattr(layer, 'sdnq_lora_svd_stash'), 'rank 0 must disable hosting'
+ activate()
+ return True
+
+
+def test_hosted_transitions_and_rng_isolation():
+ layer = build_layer('uint4')
+ A, B, D = make_delta()
+ net_plain = make_net('plainh', layer, A, B)
+ _A2, _B2, D2 = make_delta(seed=9, sigma=3e-3)
+ net_dense = make_dense_net('denseh', layer, D2)
+ with host_rank(256), mock_model(lin=layer):
+ Wdq0 = dq(layer)
+ rng0 = torch.cuda.get_rng_state() if DEVICE.type == 'cuda' else torch.get_rng_state()
+ activate(net_dense) # hosted
+ rng1 = torch.cuda.get_rng_state() if DEVICE.type == 'cuda' else torch.get_rng_state()
+ assert torch.equal(rng0, rng1), 'hosting must not consume the generation rng stream'
+ assert hasattr(layer, 'sdnq_lora_svd_stash')
+ activate(net_plain) # exact replaces hosted
+ rho = rho_of(dq(layer) - Wdq0, D)
+ assert rho > 0.99, f'exact set after hosted set: rho={rho:.4f}'
+ activate(net_plain, net_dense) # mixed set hosts the combined delta
+ rho_mix = rho_of(dq(layer) - Wdq0, D + D2)
+ assert rho_mix > 0.9, f'mixed hosted rho={rho_mix:.4f}'
+ activate()
+ assert torch.equal(dq(layer), Wdq0), 'unload must restore bit-exact'
+ return True
+
+
CAT_ROBUST = category('robustness')
@@ -571,7 +681,7 @@ def test_stacked_shape_mismatch_falls_back():
prev_enl = l_common.extra_network_lora
l_common.extra_network_lora = SimpleNamespace(errors={}) # the error path reports through the extra-networks registry
try:
- with mock_model(lin=layer):
+ with host_rank(0), mock_model(lin=layer):
Wdq0 = dq(layer)
activate(net_good)
assert hasattr(layer, 'sdnq_lora_svd_stash')
@@ -601,6 +711,10 @@ def run_tests():
log.warning('=== Set transitions ===')
for fn in [test_mixed_family_transition_restores_base, test_partial_coverage_layers_stay_independent]:
run_test(CAT_TRANS, fn)
+ log.warning('=== Hosting ===')
+ for fn in [test_hosted_low_rank_delta_is_kept, test_hosted_dense_delta_beats_requant, test_hosted_skips_int8,
+ test_hosted_disabled_by_option, test_hosted_transitions_and_rng_isolation]:
+ run_test(CAT_HOST, fn)
log.warning('=== Robustness ===')
for fn in [test_remove_factors_after_device_move, test_stacked_shape_mismatch_falls_back]:
run_test(CAT_ROBUST, fn)
diff --git a/ui/locale/locale_en.json b/ui/locale/locale_en.json
index 089b239b6..eb289bcdf 100644
--- a/ui/locale/locale_en.json
+++ b/ui/locale/locale_en.json
@@ -862,6 +862,7 @@
{"id":"","label":"LoRA native apply to text encoder","localized":"","hint":"","ui":"settings_extra_networks"},
{"id":"","label":"LoRA native fuse with model","localized":"","hint":"Merge LoRA into the model for lower memory usage.
Warning: After removing or switching a LoRA, you may still see its style in generated images. To get a clean model, reload it from the model selector.","ui":"settings_extra_networks"},
{"id":"","label":"LoRA diffusers fuse with model","localized":"","hint":"Merge LoRA into the model for lower memory usage and torch.compile compatibility.
Warning: After removing or switching a LoRA, you may still see its style in generated images. To get a clean model, reload it from the model selector.","ui":"settings_extra_networks"},
+ {"id":"","label":"LoRA quantized host rank","localized":"","hint":"Maximum rank used to carry non-factorable adapter types (LoKR, LoHA, OFT, DoRA) on the side-channel of SDNQ models quantized below 8 bits, where merging would erase most of the adapter. Higher values keep more of the adapter at proportionally higher memory cost. Set to 0 to disable and merge into the quantized weights instead.","ui":"settings_extra_networks"},
{"id":"","label":"LoRA auto-apply tags","localized":"","hint":"Automatically add trigger words/tags from LoRA metadata to your prompt.
Set to the number of tags to auto-apply, e.g., 3 = add top 3 trigger tags.
Set to 0 to disable, -1 to add all available tags.","ui":"settings_extra_networks"},
{"id":"","label":"LoRA memory cache","localized":"","hint":"How many LoRAs to keep in network for future use before requiring reloading from storage","ui":"settings_extra_networks"},
{"id":"","label":"LoRA add hash info to metadata","localized":"","hint":"Include LoRA file hashes in generated image metadata.
Useful for reproducibility and tracking which exact LoRA versions were used.","ui":"settings_extra_networks"},