From 6c0dd0b15fc3433c3a45825728ef8e959fbb69aa Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Fri, 17 Jul 2026 21:00:58 +0100 Subject: [PATCH] feat(lora): host non-factorable adapters on the sdnq side-channel Non-additive families (lokr, loha, oft, dora, full) merged into the quantized weight and lost most of their delta on low-bit formats. On sub-8-bit layers the set's calc_updown delta now rides the svd side-channel as its top singular directions instead: factorable members are subtracted out and appended exactly, so only the non-factorable remainder is truncated. Truncation keeps the dominant part of the effect and drops an orthogonal residual, where requantize keeps the grid extrema and adds grid-shift noise of the delta's own magnitude; on real lokr files retention rises from 0.04 to about 0.5 at the default rank. Hosted layers take no weight backup and unload bit-exactly. The svd runs under a forked rng so generation seeds are unaffected. At 8 bits and above requantize retains most of the delta and remains the path. lora_sdnq_host_rank caps the hosted rank; 0 disables hosting. --- modules/lora/lora_sdnq.py | 99 +++++++++++++++++++++++++-- modules/lora/networks.py | 22 +++++- modules/ui_definitions.py | 1 + test/test-sdnq-lora-factors.py | 120 ++++++++++++++++++++++++++++++++- ui/locale/locale_en.json | 1 + 5 files changed, 233 insertions(+), 10 deletions(-) diff --git a/modules/lora/lora_sdnq.py b/modules/lora/lora_sdnq.py index d07c0db8a..c2a968630 100644 --- a/modules/lora/lora_sdnq.py +++ b/modules/lora/lora_sdnq.py @@ -21,19 +21,26 @@ fidelity floors at the compute dtype, because the dequantizer materializes ``base + factors`` in the result dtype and a delta below its ULP of the base rounds exactly as it would on an unquantized model of that dtype. -Only additive low-rank modules qualify (plain LoRA: no DoRA, no CP ``mid``, -no LyCORIS dense-bias, no ``diff_b``). Layers with any non-factorable -contribution fall back to the dequantize-add-requantize path. +Only additive low-rank modules ride the channel exactly (plain LoRA: no +DoRA, no CP ``mid``, no LyCORIS dense-bias, no ``diff_b``). On sub-8-bit +formats, sets with non-factorable contributions are hosted instead: the +families' own ``calc_updown`` delta is truncated to its top singular +directions and appended the same way. Truncation keeps the dominant part +of the effect and drops an orthogonal residual, where requantize keeps +only the grid extrema and adds grid-shift noise of the delta's own +magnitude. At 8 bits and above requantize retains most of the delta, so +hosting is skipped there and the requantize path remains. """ import torch -from modules import devices +from modules import devices, shared from modules.lora import lora_common as l from modules.logger import log fallback_layers: list[str] = [] +hosted_layers: list[tuple[str, float]] = [] def get_module_factors(module, device, dtype, original_shape=None): @@ -126,7 +133,6 @@ def apply_factors(self, network_layer_name, wanted_names, use_previous=False): return changed deq = self.sdnq_dequantizer - device = self.scale.device dtype = deq.result_dtype loaded = l.loaded_networks if not use_previous else l.previously_loaded_networks ups, downs = [], [] @@ -144,7 +150,15 @@ def apply_factors(self, network_layer_name, wanted_names, use_previous=False): downs.append(down) if not ups: return changed + append_factors(self, ups, downs) + return True + +def append_factors(self, ups, downs): + """Concatenate ``[out, r]`` / ``[r, in]`` factor pairs onto the layer's svd channel and stash the originals.""" + deq = self.sdnq_dequantizer + device = self.scale.device + dtype = deq.result_dtype orig_up, orig_down = self.svd_up, self.svd_down if deq.use_quantized_matmul: # matmul layout stores factors transposed: svd_up [r, out], svd_down [in, r] @@ -157,10 +171,76 @@ def apply_factors(self, network_layer_name, wanted_names, use_previous=False): parts_down = ([orig_down.to(device=devices.device, dtype=dtype)] if orig_down is not None else []) + downs new_up = torch.cat(parts_up, dim=1).contiguous() new_down = torch.cat(parts_down, dim=0).contiguous() - self.sdnq_lora_svd_stash = (orig_up, orig_down) self.svd_up = torch.nn.Parameter(new_up.to(device=device), requires_grad=False) self.svd_down = torch.nn.Parameter(new_down.to(device=device), requires_grad=False) + + +def host_candidate(self, network_layer_name, wanted_names, use_previous=False): + """True when a non-factorable set on this layer should be hosted as a truncated svd.""" + if int(getattr(shared.opts, 'lora_sdnq_host_rank', 0) or 0) <= 0: + return False + if getattr(self, 'sdnq_dequantizer', None) is None or self.__class__.__name__ != 'SDNQLinear': + return False + if wanted_names == (): + return False + from sdnq.common import dtype_dict + if dtype_dict[self.sdnq_dequantizer.weights_dtype]['num_bits'] >= 8: + return False # requantize retains most of the delta at 8 bits and above; truncation would lose more than it saves + loaded = l.loaded_networks if not use_previous else l.previously_loaded_networks + return any(net.modules.get(network_layer_name, None) is not None for net in loaded) + + +def apply_hosted(self, network_layer_name, updown, wanted_names, use_previous=False): + """Host a set's delta on the svd channel: exact factors for factorable + members, the top-k singular directions of the remainder for the rest. + + The delta comes from the families' own ``calc_updown``, so every family + and scaling quirk is included; factorable members are subtracted out and + appended exactly so they never compete with the hosted remainder for + rank. Returns None when the delta cannot ride the channel (wrong shape); + the caller falls back to requantize. + """ + from sdnq.quant_utils import rotate_hadamard + + deq = self.sdnq_dequantizer + changed = remove_factors(self) + if wanted_names == (): + return changed + if updown is None or updown.ndim != 2 or tuple(updown.shape) != tuple(deq.original_shape): + return None + dtype = deq.result_dtype + D = updown.detach().to(devices.device, torch.float32) + + ups, downs = [], [] + loaded = l.loaded_networks if not use_previous else l.previously_loaded_networks + for net in loaded: + module = net.modules.get(network_layer_name, None) + if module is None: + continue + factors = get_module_factors(module, devices.device, dtype, original_shape=deq.original_shape) + if factors is None: + continue + up_eff, down = factors + D = D.sub_(up_eff.to(torch.float32) @ down.to(torch.float32)) # factorable members ride exactly; host only the remainder + if deq.use_hadamard: + down = rotate_hadamard(down.to(dtype=torch.float32), group_size=deq.hadamard_group_size).to(dtype=dtype) + ups.append(up_eff) + downs.append(down) + + cap = int(shared.opts.lora_sdnq_host_rank) + q = min(cap, *D.shape) + # svd_lowrank draws random projections; fork so user generation seeds are untouched and re-applies are deterministic + with torch.random.fork_rng(devices=[D.device] if D.device.type == 'cuda' else []): + torch.manual_seed(0) + U, S, V = torch.svd_lowrank(D, q=q, niter=2) + energy = float(S.square().sum() / D.square().sum().clamp(min=1e-30)) + up_h = (U * S).to(dtype=dtype) + down_h = V.t() + if deq.use_hadamard: + down_h = rotate_hadamard(down_h, group_size=deq.hadamard_group_size) + append_factors(self, ups + [up_h], downs + [down_h.to(dtype=dtype)]) + hosted_layers.append((network_layer_name, energy)) return True @@ -171,6 +251,13 @@ def note_fallback(self, network_layer_name): def report_fallbacks(): + if len(hosted_layers) > 0: + energies = sorted(e for _name, e in hosted_layers) + median = energies[len(energies) // 2] + log.info(f'Network load: type=LoRA quant=sdnq hosted={len(hosted_layers)} rank={int(shared.opts.lora_sdnq_host_rank)} energy={median:.2f} min={energies[0]:.2f} non-factorable networks hosted on the svd side-channel') + if l.debug: + log.debug(f'Network load: type=LoRA quant=sdnq hosted={[(n, round(e, 3)) for n, e in hosted_layers[:8]]}{"..." if len(hosted_layers) > 8 else ""}') + hosted_layers.clear() if len(fallback_layers) > 0: log.warning(f'Network load: type=LoRA quant=sdnq layers={len(fallback_layers)} non-factorable networks requantized in place (reduced fidelity on quantized weights)') if l.debug: diff --git a/modules/lora/networks.py b/modules/lora/networks.py index c472da4ea..d027a5f81 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -91,6 +91,7 @@ def network_activate(include=None, exclude=None): wanted_names = tuple((x.name, x.te_multiplier, x.unet_multiplier, x.dyn_dim) for x in l.loaded_networks) if len(l.loaded_networks) > 0 else () applied_layers.clear() lora_sdnq.fallback_layers.clear() # a raise mid-pass leaves stale entries behind + lora_sdnq.hosted_layers.clear() backup_size = 0 for component in modules.keys(): component_wanted = wanted_names if component in components else () @@ -109,7 +110,7 @@ def network_activate(include=None, exclude=None): if weights_backup is not None and not isinstance(weights_backup, bool): network_apply_weights(module, None, None, device=device) # an earlier non-factorable set requantized this layer, restore the pristine base before attaching factors applied = lora_sdnq.apply_factors(module, network_layer_name, component_wanted) - if applied is not None: # exact path took the layer; None falls through to requantize + if applied is not None: # exact path took the layer; None falls through to hosting or requantize if applied and component_wanted: applied_layers.append(network_layer_name) applied_weight += 1 @@ -117,6 +118,25 @@ def network_activate(include=None, exclude=None): if task is not None: pbar.update(task, advance=1) continue + if lora_sdnq.host_candidate(module, network_layer_name, component_wanted): + weights_backup = getattr(module, "network_weights_backup", None) + if weights_backup is not None and not isinstance(weights_backup, bool): + network_apply_weights(module, None, None, device=device) # the hosted delta is measured against the pristine base + batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name, elimit=elimit) + if batch_ex_bias is None: # bias deltas need the plain path; weight-only sets ride the side-channel without a weight backup + hosted = lora_sdnq.apply_hosted(module, network_layer_name, batch_updown, component_wanted) + if hosted is not None: + if hosted and component_wanted: + applied_layers.append(network_layer_name) + applied_weight += 1 + module.network_current_names = component_wanted + batch_updown, batch_ex_bias = None, None + del batch_updown, batch_ex_bias + if task is not None: + pbar.update(task, advance=1) + continue + batch_updown, batch_ex_bias = None, None + del batch_updown, batch_ex_bias backup_size += network_backup_weights(module, network_layer_name, component_wanted, fuse) if not component_wanted: weights_backup = getattr(module, "network_weights_backup", None) diff --git a/modules/ui_definitions.py b/modules/ui_definitions.py index ed0dab9c6..8e9284566 100644 --- a/modules/ui_definitions.py +++ b/modules/ui_definitions.py @@ -684,6 +684,7 @@ def create_settings(cmd_opts): "lora_apply_te": OptionInfo(False, "LoRA native apply to text encoder"), "lora_fuse_native": OptionInfo(True, "LoRA native fuse with model"), "lora_fuse_diffusers": OptionInfo(False, "LoRA diffusers fuse with model"), + "lora_sdnq_host_rank": OptionInfo(256, "LoRA quantized host rank", gr.Slider, {"minimum": 0, "maximum": 1024, "step": 32}), "lora_apply_tags": OptionInfo(0, "LoRA auto-apply tags", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), "lora_in_memory_limit": OptionInfo(1, "LoRA memory cache", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1}), "lora_add_hashes_to_infotext": OptionInfo(False, "LoRA add hash info to metadata"), diff --git a/test/test-sdnq-lora-factors.py b/test/test-sdnq-lora-factors.py index 4257f35fd..8514e9632 100644 --- a/test/test-sdnq-lora-factors.py +++ b/test/test-sdnq-lora-factors.py @@ -23,6 +23,11 @@ for the per-model analyzer): - Robustness: factor removal restores onto the layer's current device after an offload-style move, and a shape-mismatched network stacked onto a factor-mode layer downgrades to the legacy path instead of raising. +- Hosting: on sub-8-bit layers, non-factorable sets ride the side-channel as + a truncated svd of their calc_updown delta: low-rank content survives + whole, dense content beats the requantize floor by a wide margin, int8 + and rank 0 keep the requantize path, removal stays bit-exact, and the + svd's random projections never touch the generation rng stream. All tensors are synthetic; no model files or running server required. @@ -488,7 +493,7 @@ def test_mixed_family_transition_restores_base(): real_report() lora_sdnq.report_fallbacks = capture_report try: - with mock_model(lin=layer, bystander=bystander): + with host_rank(0), mock_model(lin=layer, bystander=bystander): # pins the requantize fallback; hosted transitions are covered in the hosting category Wdq0 = dq(layer) activate(net_plain) assert hasattr(layer, 'sdnq_lora_svd_stash'), 'plain set must take the factor path' @@ -525,7 +530,7 @@ def test_partial_coverage_layers_stay_independent(): A2, B2, _ = make_delta(seed=5, sigma=3e-3) net_dora = make_net('dorafar', layer_dora, A2, B2, dora=True) - with mock_model(lin=layer_plain, other=layer_dora): + with host_rank(0), mock_model(lin=layer_plain, other=layer_dora): # pins the requantize fallback for the non-factorable layer Wdq0, Wdq0_dora = dq(layer_plain), dq(layer_dora) activate(net_plain, net_dora) assert hasattr(layer_plain, 'sdnq_lora_svd_stash') and getattr(layer_plain, 'network_weights_backup', None) is None, 'plain layer must stay on the factor path' @@ -538,6 +543,111 @@ def test_partial_coverage_layers_stay_independent(): return True +CAT_HOST = category('hosting') + + +@contextmanager +def host_rank(rank): + old = getattr(shared.opts, 'lora_sdnq_host_rank', 0) + shared.opts.lora_sdnq_host_rank = rank + try: + yield + finally: + shared.opts.lora_sdnq_host_rank = old + + +def make_dense_net(name, layer, D): + """A full-family (dense diff) network module: non-factorable by construction.""" + from modules.lora import network_full + net = network.Network(name, MockNOD(name)) + net.te_multiplier = 1.0 + net.unet_multiplier = [1.0] * 3 + nw = network.NetworkWeights(network_key=layer.network_layer_name, sd_key=layer.network_layer_name, + w={'diff': D.cpu()}, sd_module=layer) + net.modules[layer.network_layer_name] = network_full.NetworkModuleFull(net, nw) + return net + + +def test_hosted_low_rank_delta_is_kept(): + layer = build_layer('uint4') + _A, _B, D = make_delta(sigma=3e-3) + net = make_dense_net('densenet', layer, D) # low-rank content in a non-factorable container + with host_rank(64), mock_model(lin=layer): + Wdq0 = dq(layer) + activate(net) + assert hasattr(layer, 'sdnq_lora_svd_stash'), 'hosted set must ride the side-channel' + assert getattr(layer, 'network_weights_backup', None) is None, 'hosted layers must not take a weight backup' + rho = rho_of(dq(layer) - Wdq0, D) + assert rho > 0.95, f'rank-8 delta under cap 64 must be kept nearly whole: rho={rho:.4f}' + activate() + assert torch.equal(dq(layer), Wdq0), 'unload must restore bit-exact' + return True + + +def test_hosted_dense_delta_beats_requant(): + layer = build_layer('uint4') + torch.manual_seed(3) + D = torch.randn(OUT_F, IN_F, device=DEVICE) * 3e-4 # full-rank, sub-step: requant erases it + requant_rho = rho_of(requant_effective(layer, D), D) + net = make_dense_net('densefull', layer, D) + with host_rank(256), mock_model(lin=layer): + Wdq0 = dq(layer) + activate(net) + hosted_rho = rho_of(dq(layer) - Wdq0, D) + assert hosted_rho > 0.4, f'hosted rho={hosted_rho:.3f}' + assert hosted_rho > requant_rho + 0.3, f'hosting must beat requant by a wide margin: {hosted_rho:.3f} vs {requant_rho:.3f}' + activate() + assert torch.equal(dq(layer), Wdq0) + return True + + +def test_hosted_skips_int8(): + layer = build_layer('int8') + _A, _B, D = make_delta(sigma=3e-3) + net = make_dense_net('int8net', layer, D) + with host_rank(256), mock_model(lin=layer): + activate(net) + assert not hasattr(layer, 'sdnq_lora_svd_stash'), 'int8 must keep the requantize path' + assert isinstance(getattr(layer, 'network_weights_backup', None), torch.Tensor), 'int8 fallback must take the backup' + activate() + return True + + +def test_hosted_disabled_by_option(): + layer = build_layer('uint4') + _A, _B, D = make_delta(sigma=3e-3) + net = make_dense_net('offnet', layer, D) + with host_rank(0), mock_model(lin=layer): + activate(net) + assert not hasattr(layer, 'sdnq_lora_svd_stash'), 'rank 0 must disable hosting' + activate() + return True + + +def test_hosted_transitions_and_rng_isolation(): + layer = build_layer('uint4') + A, B, D = make_delta() + net_plain = make_net('plainh', layer, A, B) + _A2, _B2, D2 = make_delta(seed=9, sigma=3e-3) + net_dense = make_dense_net('denseh', layer, D2) + with host_rank(256), mock_model(lin=layer): + Wdq0 = dq(layer) + rng0 = torch.cuda.get_rng_state() if DEVICE.type == 'cuda' else torch.get_rng_state() + activate(net_dense) # hosted + rng1 = torch.cuda.get_rng_state() if DEVICE.type == 'cuda' else torch.get_rng_state() + assert torch.equal(rng0, rng1), 'hosting must not consume the generation rng stream' + assert hasattr(layer, 'sdnq_lora_svd_stash') + activate(net_plain) # exact replaces hosted + rho = rho_of(dq(layer) - Wdq0, D) + assert rho > 0.99, f'exact set after hosted set: rho={rho:.4f}' + activate(net_plain, net_dense) # mixed set hosts the combined delta + rho_mix = rho_of(dq(layer) - Wdq0, D + D2) + assert rho_mix > 0.9, f'mixed hosted rho={rho_mix:.4f}' + activate() + assert torch.equal(dq(layer), Wdq0), 'unload must restore bit-exact' + return True + + CAT_ROBUST = category('robustness') @@ -571,7 +681,7 @@ def test_stacked_shape_mismatch_falls_back(): prev_enl = l_common.extra_network_lora l_common.extra_network_lora = SimpleNamespace(errors={}) # the error path reports through the extra-networks registry try: - with mock_model(lin=layer): + with host_rank(0), mock_model(lin=layer): Wdq0 = dq(layer) activate(net_good) assert hasattr(layer, 'sdnq_lora_svd_stash') @@ -601,6 +711,10 @@ def run_tests(): log.warning('=== Set transitions ===') for fn in [test_mixed_family_transition_restores_base, test_partial_coverage_layers_stay_independent]: run_test(CAT_TRANS, fn) + log.warning('=== Hosting ===') + for fn in [test_hosted_low_rank_delta_is_kept, test_hosted_dense_delta_beats_requant, test_hosted_skips_int8, + test_hosted_disabled_by_option, test_hosted_transitions_and_rng_isolation]: + run_test(CAT_HOST, fn) log.warning('=== Robustness ===') for fn in [test_remove_factors_after_device_move, test_stacked_shape_mismatch_falls_back]: run_test(CAT_ROBUST, fn) diff --git a/ui/locale/locale_en.json b/ui/locale/locale_en.json index 089b239b6..eb289bcdf 100644 --- a/ui/locale/locale_en.json +++ b/ui/locale/locale_en.json @@ -862,6 +862,7 @@ {"id":"","label":"LoRA native apply to text encoder","localized":"","hint":"","ui":"settings_extra_networks"}, {"id":"","label":"LoRA native fuse with model","localized":"","hint":"Merge LoRA into the model for lower memory usage.

Warning: After removing or switching a LoRA, you may still see its style in generated images. To get a clean model, reload it from the model selector.","ui":"settings_extra_networks"}, {"id":"","label":"LoRA diffusers fuse with model","localized":"","hint":"Merge LoRA into the model for lower memory usage and torch.compile compatibility.

Warning: After removing or switching a LoRA, you may still see its style in generated images. To get a clean model, reload it from the model selector.","ui":"settings_extra_networks"}, + {"id":"","label":"LoRA quantized host rank","localized":"","hint":"Maximum rank used to carry non-factorable adapter types (LoKR, LoHA, OFT, DoRA) on the side-channel of SDNQ models quantized below 8 bits, where merging would erase most of the adapter. Higher values keep more of the adapter at proportionally higher memory cost. Set to 0 to disable and merge into the quantized weights instead.","ui":"settings_extra_networks"}, {"id":"","label":"LoRA auto-apply tags","localized":"","hint":"Automatically add trigger words/tags from LoRA metadata to your prompt.
Set to the number of tags to auto-apply, e.g., 3 = add top 3 trigger tags.
Set to 0 to disable, -1 to add all available tags.","ui":"settings_extra_networks"}, {"id":"","label":"LoRA memory cache","localized":"","hint":"How many LoRAs to keep in network for future use before requiring reloading from storage","ui":"settings_extra_networks"}, {"id":"","label":"LoRA add hash info to metadata","localized":"","hint":"Include LoRA file hashes in generated image metadata.
Useful for reproducibility and tracking which exact LoRA versions were used.","ui":"settings_extra_networks"},