Merge pull request #5067 from vladmandic/feat/lora-sdnq-cache

perf(lora): factor cache and hosting improvements for sdnq exact apply

cli/lora-quant-fidelity.py

@@ -252,7 +252,7 @@ class Bf16Repo:
         return f.get_tensor(key)
 
 
-def analyze_module(W_dq, deq_params, mods, calib_rms=None):
+def analyze_module(W_dq, deq_params, mods, calib_rms=None, step_live=None):
     """Return fidelity metrics for one quantized module and the adapters targeting it.
 
     Deltas come from each module's production calc_updown and sum the way the
@@ -260,6 +260,9 @@ def analyze_module(W_dq, deq_params, mods, calib_rms=None):
     is measured as applied. A module is factor-path eligible only when every
     contribution is a plain additive lora. With ``calib_rms``, hosting mirrors
     the calibrated production path and its rho is scored in the weighted norm.
+    With ``step_live`` (the layer's own pre-add scale), the production routing
+    rule applies: a delta fat against the grid whose truncation capture is low
+    reports the requantize path, the way the loader would route it.
     """
     D = None
     for mod in mods:
@@ -315,18 +318,24 @@ def analyze_module(W_dq, deq_params, mods, calib_rms=None):
                 Dw = D * rms if rms is not None else D
                 with torch.random.fork_rng(devices=[D.device] if D.device.type == 'cuda' else []):
                     torch.manual_seed(0)
-                    U, S, V = torch.svd_lowrank(Dw, q=q, niter=2)
-                Dk = (U * S) @ V.t()
-                if rms is not None:
-                    Dk = Dk / rms
-                base16 = W_dq.to(torch.bfloat16).float()
-                realized = (W_dq.to(torch.bfloat16) + Dk.to(torch.bfloat16)).float() - base16
-                if rms is not None: # weighted norm: the diagonal-covariance output-error proxy the calibrated truncation optimizes
-                    Dr = D * rms
-                    applied_rho = float((realized * rms).flatten() @ Dr.flatten() / Dr.square().sum())
-                else:
-                    applied_rho = float(realized.flatten() @ D.flatten() / nD.square())
-                hosted = True
+                    U, S, V = torch.svd_lowrank(Dw, q=min(q + 64, *D.shape), niter=8)
+                energy = float(S[:q].square().sum() / Dw.square().sum().clamp(min=1e-30))
+                routed = False
+                if step_live is not None and not deq_params.get('use_svd', False):
+                    sr = float(D.square().mean().sqrt() / step_live.float().mean())
+                    routed = sr > lora_sdnq.REQUANT_RATIO and energy < lora_sdnq.REQUANT_ENERGY
+                if not routed: # the loader routes fat, genuinely-truncated deltas back to requantize
+                    Dk = (U[:, :q] * S[:q]) @ V[:, :q].t()
+                    if rms is not None:
+                        Dk = Dk / rms
+                    base16 = W_dq.to(torch.bfloat16).float()
+                    realized = (W_dq.to(torch.bfloat16) + Dk.to(torch.bfloat16)).float() - base16
+                    if rms is not None: # weighted norm: the diagonal-covariance output-error proxy the calibrated truncation optimizes
+                        Dr = D * rms
+                        applied_rho = float((realized * rms).flatten() @ Dr.flatten() / Dr.square().sum())
+                    else:
+                        applied_rho = float(realized.flatten() @ D.flatten() / nD.square())
+                    hosted = True
     return dict(rank=getattr(mods[0], 'dim', None), rms_delta=float(D.pow(2).mean().sqrt()), rms_weight=float(W_dq.pow(2).mean().sqrt()),
                 step_ratio=step_ratio, crossers=crossers, requant_rho=rho, requant_resid=resid,
                 factor_eligible=factor_eligible, hosted=hosted, applied_rho=applied_rho,
@@ -398,6 +407,7 @@ def main():
                                skip_quantized_matmul=deq.use_quantized_matmul, dtype=torch.float32, skip_compile=True).to(device)
                     params = dict(weights_dtype=deq.weights_dtype, group_size=deq.group_size, hadamard_group_size=deq.hadamard_group_size,
                                   use_hadamard=deq.use_hadamard, use_svd=layer.svd_up is not None, svd_rank=deq.svd_rank, svd_steps=deq.svd_steps)
+                    step_live = layer.scale.detach().to(device)
                     sd_module = layer
                 else:
                     W = bf16_repo.get(f'{path}.weight')
@@ -412,16 +422,18 @@ def main():
                     if args.dtype == 'bf16':
                         W_dq = W.to(device, torch.bfloat16).float()
                         params = dict(weights_dtype='bf16', group_size=0, hadamard_group_size=0, use_hadamard=False)
+                        step_live = None
                     else:
                         deq0, data0 = sdnq_quantize_layer_weight(W.to(device, torch.float32), layer_class_name='Linear', weights_dtype=args.dtype,
                                                                  group_size=args.group, hadamard_group_size=args.hadamard_group, use_hadamard=args.hadamard_group > 0,
                                                                  use_svd=False, use_quantized_matmul=False, dequantize_fp32=False, torch_dtype=torch.bfloat16)
                         W_dq = deq0(data0['weight'], data0['scale'], zero_point=data0['zero_point'], svd_up=None, svd_down=None, dtype=torch.float32, skip_compile=True)
                         params = dict(weights_dtype=args.dtype, group_size=deq0.group_size, hadamard_group_size=deq0.hadamard_group_size, use_hadamard=deq0.use_hadamard)
+                        step_live = data0['scale'].detach()
                     sd_module = make_stub(W.shape)
                 try:
                     mods = [build_module(fam, path, w, net, sd_module) for fam, w in entries]
-                    row = analyze_module(W_dq, params, mods, calib_rms=calib_stats.get(lname))
+                    row = analyze_module(W_dq, params, mods, calib_rms=calib_stats.get(lname), step_live=step_live)
                 except Exception as e: # a family the tool cannot rebuild must not read as a clean module
                     failed.append(f'{path}: {type(e).__name__}: {e}')
                     del W_dq

modules/lora/lora_calib.py

@@ -13,11 +13,15 @@ MLP down projections whose inputs carry the largest outlier channels.
 Statistics come from the model's own forwards: when a sub-8-bit SDNQ model
 loads and no calibration is cached for it, streaming sum-of-squares hooks
 attach to its quantized linears, accumulate during normal generations,
-persist once enough tokens are seen, and go inert. Cached statistics load
-at model load and sit on each layer as ``sdnq_calib_rms``; the hosting path
-reads them through ``rms_for``. Capture is skipped when the model is
-compiled (hooks would break the graph) and everything is gated by the
-``lora_sdnq_host_calib`` option.
+persist, and go inert. Persist fires when every layer reaches the token
+quota, or at a bounded number of denoiser forwards for models where some
+projections take pooled or modulation vectors (a few tokens per forward)
+and could never reach an absolute quota; layers still under a small token
+floor at the deadline are omitted and stay on plain truncation. Cached
+statistics load at model load and sit on each layer as ``sdnq_calib_rms``;
+the hosting path reads them through ``rms_for``. Capture is skipped when
+the model is compiled (hooks would break the graph) and everything is
+gated by the ``lora_sdnq_host_calib`` option.
 """
 
 import os
@@ -40,12 +44,15 @@ class CaptureState(TypedDict):
     model: Optional[str]
     recs: dict[str, CaptureRecord]
     handles: list[torch.utils.hooks.RemovableHandle]
+    forwards: int
     complete: bool
 
 
 TOKENS_DONE = 65536
+FORWARDS_DEADLINE = 48 # ~2 generations; token-rich layers normally finish their quota well inside it
+TOKENS_FLOOR = 32 # below this mass the rms estimate is noise; the layer is omitted and stays on plain truncation
 calib_root = os.path.join(paths.models_path, 'calibration')
-capture: CaptureState = {'model': None, 'recs': {}, 'handles': [], 'complete': False}
+capture: CaptureState = {'model': None, 'recs': {}, 'handles': [], 'forwards': 0, 'complete': False}
 
 
 def enabled():
@@ -62,14 +69,20 @@ def checkpoint_name(sd_model):
     return getattr(info, 'name', None)
 
 
+def denoiser_root(sd_model):
+    """The model's denoiser component, transformer first, unet otherwise."""
+    root = getattr(sd_model, 'transformer', None)
+    return root if root is not None else getattr(sd_model, 'unet', None)
+
+
 def eligible_modules(sd_model):
-    """Sub-8-bit 2-D SDNQ linears of the model's transformer: the layers hosting applies to."""
-    transformer = getattr(sd_model, 'transformer', None)
-    if transformer is None:
+    """Sub-8-bit 2-D SDNQ linears of the model's denoiser: the layers hosting applies to."""
+    root = denoiser_root(sd_model)
+    if root is None:
         return []
     from sdnq.common import dtype_dict
     out = []
-    for name, m in transformer.named_modules():
+    for name, m in root.named_modules():
         deq = getattr(m, 'sdnq_dequantizer', None)
         if deq is None or len(deq.original_shape) != 2:
             continue
@@ -85,9 +98,24 @@ def detach_capture():
     capture['handles'].clear()
     capture['recs'].clear()
     capture['model'] = None
+    capture['forwards'] = 0
     capture['complete'] = False
 
 
+def deadline_hook(module, hook_args): # pylint: disable=unused-argument
+    """Count denoiser forwards and close capture at the deadline.
+
+    Layers taking pooled or modulation vectors see a few tokens per forward
+    and can never reach the token quota; a global forward count bounds
+    capture for them and for modules the generation path never runs.
+    """
+    if capture['complete']:
+        return
+    capture['forwards'] += 1
+    if capture['forwards'] >= FORWARDS_DEADLINE:
+        persist()
+
+
 def hook_for(rec, in_features):
     def hook(module, hook_args): # pylint: disable=unused-argument
         if rec['done'] or capture['complete']:
@@ -111,10 +139,12 @@ def hook_for(rec, in_features):
 
 
 def persist():
-    """Write completed statistics and stamp them onto the layers.
+    """Write accumulated statistics and stamp them onto the layers.
 
-    Runs from the last completing hook, inside a forward; the write is a few
-    MB once per checkpoint ever. Handles stay registered but inert until the
+    Runs from the last hook to complete its quota or from the forward
+    deadline, inside a forward; the write is a few MB once per checkpoint
+    ever. Layers under the token floor are omitted rather than saved with
+    meaningless statistics. Handles stay registered but inert until the
     next safe point removes them (hook removal here would mutate the hook
     dict the forward is iterating).
     """
@@ -124,15 +154,20 @@ def persist():
     from safetensors.torch import save_file
     tensors, min_n = {}, None
     for name, rec in capture['recs'].items():
-        rms = (rec['ss'] / max(rec['n'], 1)).sqrt().float().cpu().contiguous().clone()
+        if rec['ss'] is None or rec['n'] < TOKENS_FLOOR:
+            continue
+        rms = (rec['ss'] / rec['n']).sqrt().float().cpu().contiguous().clone()
         tensors[name] = rms
         rec['m'].sdnq_calib_rms = rms
         min_n = rec['n'] if min_n is None else min(min_n, rec['n'])
+    if not tensors:
+        log.warning(f'Network calibration: model="{capture["model"]}" no layer reached {TOKENS_FLOOR} tokens; nothing saved')
+        return
     path = calib_file(capture['model'])
     try:
         os.makedirs(calib_root, exist_ok=True)
         save_file(tensors, path, metadata={'version': '1', 'model': capture['model'], 'tokens': str(min_n)})
-        log.info(f'Network calibration: model="{capture["model"]}" layers={len(tensors)} tokens={min_n} saved="{path}"')
+        log.info(f'Network calibration: model="{capture["model"]}" layers={len(tensors)}/{len(capture["recs"])} tokens={min_n} saved="{path}"')
     except Exception as e:
         log.warning(f'Network calibration: save failed path="{path}" {e}')
 
@@ -172,6 +207,7 @@ def on_model_loaded(sd_model):
     if 'Model' in (getattr(shared.opts, 'cuda_compile', None) or []):
         return # hooks inside a compiled module graph-break or misbehave; skip capture entirely
     capture['model'] = name
+    capture['handles'].append(denoiser_root(sd_model).register_forward_pre_hook(deadline_hook))
     for mod_name, m in modules_list:
         rec = {'m': m, 'ss': None, 'n': 0, 'done': False}
         capture['recs'][mod_name] = rec

modules/lora/lora_factor_cache.py

@@ -0,0 +1,219 @@
+"""Disk cache for hosted svd factors.
+
+Hosting a non-factorable adapter set costs one truncated svd per targeted
+layer (tens of ms each, seconds per file) every time the set is applied
+fresh. The resulting factors are deterministic in the checkpoint, the loaded
+set (files, multipliers, dyn_dim), the host rank and the calibration
+statistics, so they are cached on disk keyed by exactly that identity and
+replayed bit-identically on the next apply of the same configuration.
+
+One safetensors file per configuration under ``models/lora-factor-cache``,
+holding every hosted layer's post-rotation factor pair as rowwise int8
+with fp32 scales (measured fidelity-free in output space, half the bytes
+of bf16). Files are named by the model and network set with an
+identity-hash suffix, and the exact signature is embedded in the file
+metadata. Factors are quantized before first use: ``store`` returns the
+dequantized round-trip for the caller to apply, so a fresh compute and a
+later cache hit attach bit-identical tensors. The ``lora_sdnq_host_cache``
+option is the size budget in GB (0 disables); least-recently-used entries
+are evicted past the budget. Any doubt about identity (unknown checkpoint,
+unreadable lora file, signature mismatch) disables caching for the pass
+rather than risking a stale hit.
+"""
+
+import os
+import json
+import hashlib
+
+import torch
+
+from modules import paths, shared
+from modules.lora import lora_common as l
+from modules.logger import log
+
+
+cache_root = os.path.join(paths.models_path, 'lora-factor-cache')
+state = {'wn': None, 'sig': None, 'path': None, 'store': {}, 'dirty': False, 'hits': 0, 'misses': 0}
+FMT = '5' # bump on entry-layout changes so older files recompute instead of replaying short
+
+
+def budget_gb():
+    try:
+        return float(getattr(shared.opts, 'lora_sdnq_host_cache', 0) or 0)
+    except Exception:
+        return 0.0
+
+
+def signature(wanted_names):
+    """Content identity of a hosted-apply configuration, or None when caching is unsafe."""
+    from modules.lora import lora_calib
+    model_name = lora_calib.checkpoint_name(getattr(shared, 'sd_model', None))
+    if model_name is None:
+        return None
+    calib_path = lora_calib.calib_file(model_name)
+    parts = {
+        'model': model_name,
+        'rank': int(getattr(shared.opts, 'lora_sdnq_host_rank', 0) or 0),
+        'calib': int(os.path.getmtime(calib_path)) if lora_calib.enabled() and os.path.isfile(calib_path) else None, # the toggle is part of the identity: factors computed under the other setting must not replay
+        'nets': [],
+    }
+    for name, te, unet, dyn in wanted_names:
+        net = next((n for n in l.loaded_networks if n.name == name), None)
+        filename = getattr(getattr(net, 'network_on_disk', None), 'filename', None)
+        try:
+            st = os.stat(filename)
+        except Exception:
+            return None
+        parts['nets'].append([name, repr(te), repr(unet), repr(dyn), filename, int(st.st_mtime), st.st_size])
+    return parts
+
+
+def label(parts):
+    """Filename prefix from the model and net names, so the cache folder reads without tooling."""
+    names = [parts['model'].replace('\\', '/').split('/')[-1]] + [n[0] for n in parts['nets']]
+    text = '-'.join(names)
+    text = ''.join(c if c.isalnum() or c in '._-' else '-' for c in text)
+    return text[:96]
+
+
+def begin_pass(wanted_names):
+    """Bind the pass to its cache entry; identity-memoized on the wanted_names tuple."""
+    if wanted_names is state['wn']:
+        return
+    state['wn'] = wanted_names
+    state.update(sig=None, path=None, dirty=False)
+    state['store'] = {}
+    if budget_gb() <= 0 or wanted_names == ():
+        return
+    parts = signature(wanted_names)
+    if parts is None:
+        return
+    sig = json.dumps(parts, sort_keys=True)
+    key = hashlib.sha256(sig.encode()).hexdigest()[:24]
+    path = os.path.join(cache_root, f'{label(parts)}-{key}.safetensors')
+    entries = {}
+    if os.path.isfile(path):
+        try:
+            from safetensors import safe_open
+            with safe_open(path, framework='pt', device='cpu') as f:
+                meta = f.metadata() or {}
+                if meta.get('sig') == sig and meta.get('fmt') == FMT:
+                    for k in f.keys():
+                        entries[k] = f.get_tensor(k)
+            os.utime(path, None) # freshness for LRU eviction
+        except Exception as e:
+            log.debug(f'Network cache: read failed path="{path}" {e}')
+            entries = {}
+    state.update(sig=sig, path=path)
+    state['store'] = entries
+    log.debug(f'Network cache: entry="{path}" keys={len(entries)}')
+
+
+def quantize_rowwise(t):
+    t32 = t.detach().to(torch.float32)
+    scale = t32.abs().amax(dim=1, keepdim=True).clamp(min=1e-12) / 127.0
+    q = (t32 / scale).round().clamp(-127, 127).to(torch.int8)
+    return q, scale
+
+
+def dequantize_rowwise(q, scale):
+    # int8 * fp32 with a single fp32 rounding: identical on any device, so hit and miss replay the same values
+    return q.to(torch.float32) * scale
+
+
+def lookup(network_layer_name):
+    """Cached (up, down, energy, calibrated, rms) for a layer, or None; factors return as fp32.
+
+    Pure lookup with no hit/miss accounting: the fast-path probe uses it so a
+    layer is only counted once, by whichever caller consumes the answer.
+    """
+    if state['sig'] is None:
+        return None
+    st = state['store']
+    up_q, up_s = st.get(f'{network_layer_name}.up_q'), st.get(f'{network_layer_name}.up_s')
+    down_q, down_s = st.get(f'{network_layer_name}.down_q'), st.get(f'{network_layer_name}.down_s')
+    energy = st.get(f'{network_layer_name}.energy')
+    calib = st.get(f'{network_layer_name}.calib')
+    rms = st.get(f'{network_layer_name}.rms')
+    if up_q is None or up_s is None or down_q is None or down_s is None or energy is None or calib is None or rms is None:
+        return None
+    return dequantize_rowwise(up_q, up_s), dequantize_rowwise(down_q, down_s), float(energy), bool(calib), float(rms)
+
+
+def note_hit():
+    state['hits'] += 1
+
+
+def fetch(network_layer_name):
+    """``lookup`` with accounting: a usable entry counts a hit, anything else a miss."""
+    entry = lookup(network_layer_name)
+    if entry is None:
+        if state['sig'] is not None:
+            state['misses'] += 1
+        return None
+    state['hits'] += 1
+    return entry
+
+
+def store(network_layer_name, up, down, energy, calibrated, rms):
+    """Quantize-before-use: returns the dequantized round-trip the caller must apply.
+
+    The factors quantize to rowwise int8 whether or not a cache entry can be
+    written, so the factors applied now, the factors a later hit replays, and a
+    cache-off apply are the same tensors (the round-trip also zeroes null-tail
+    columns the attach-side trim relies on). ``rms`` is the assembled delta's
+    rms, kept so replays can evaluate the requantize routing rule without
+    assembling the delta.
+    """
+    up_q, up_s = quantize_rowwise(up)
+    down_q, down_s = quantize_rowwise(down)
+    if state['sig'] is not None:
+        st = state['store']
+        st[f'{network_layer_name}.up_q'] = up_q.to('cpu').contiguous()
+        st[f'{network_layer_name}.up_s'] = up_s.to('cpu').contiguous()
+        st[f'{network_layer_name}.down_q'] = down_q.to('cpu').contiguous()
+        st[f'{network_layer_name}.down_s'] = down_s.to('cpu').contiguous()
+        st[f'{network_layer_name}.energy'] = torch.tensor(float(energy))
+        st[f'{network_layer_name}.calib'] = torch.tensor(1 if calibrated else 0, dtype=torch.uint8)
+        st[f'{network_layer_name}.rms'] = torch.tensor(float(rms))
+        state['dirty'] = True
+    return dequantize_rowwise(up_q, up_s).to(up.dtype), dequantize_rowwise(down_q, down_s).to(down.dtype)
+
+
+def evict():
+    budget = budget_gb() * 2**30
+    try:
+        files = [os.path.join(cache_root, f) for f in os.listdir(cache_root) if f.endswith('.safetensors')]
+        sizes = {p: os.path.getsize(p) for p in files}
+    except Exception:
+        return
+    total = sum(sizes.values())
+    for p in sorted(files, key=os.path.getmtime):
+        if total <= budget:
+            break
+        if p == state['path']:
+            continue # never evict the entry of the live pass
+        try:
+            os.remove(p)
+            total -= sizes[p]
+        except Exception:
+            pass
+
+
+def flush():
+    """Persist a dirty pass store; returns (hits, misses) since the last flush."""
+    hits, misses = state['hits'], state['misses']
+    state['hits'] = state['misses'] = 0
+    if not state['dirty'] or state['path'] is None:
+        return hits, misses
+    state['dirty'] = False
+    try:
+        from safetensors.torch import save_file
+        os.makedirs(cache_root, exist_ok=True)
+        tmp = state['path'] + '.tmp'
+        save_file(state['store'], tmp, metadata={'sig': state['sig'], 'fmt': FMT})
+        os.replace(tmp, state['path'])
+        evict()
+    except Exception as e:
+        log.warning(f'Network cache: write failed path="{state["path"]}" {e}')
+    return hits, misses

modules/lora/lora_sdnq.py

@@ -25,7 +25,9 @@ Only additive low-rank modules ride the channel exactly (plain LoRA: no
 DoRA, no CP ``mid``, no LyCORIS dense-bias, no ``diff_b``). On sub-8-bit
 formats, sets with non-factorable contributions are hosted instead: the
 families' own ``calc_updown`` delta is truncated to its top singular
-directions and appended the same way. Truncation keeps the dominant part
+directions and appended the same way, stored at the delta's effective
+rank when the spectrum ends in a numerically null tail (dense-combined
+plain pairs, low-rank LyCORIS). Truncation keeps the dominant part
 of the effect and drops an orthogonal residual, where requantize keeps
 only the grid extrema and adds grid-shift noise of the delta's own
 magnitude. When activation statistics for the checkpoint exist (see
@@ -33,18 +35,47 @@ magnitude. When activation statistics for the checkpoint exist (see
 error instead of weight error. At 8 bits and above requantize retains
 most of the delta, so hosting is skipped there and the requantize path
 remains.
+
+A small tail of deltas inverts the tradeoff: when the delta is large
+against the grid step AND the truncation genuinely cuts it, requantize
+retains more than hosting drops, and the layer routes back to the
+requantize path (``REQUANT_RATIO``/``REQUANT_ENERGY``). Both terms must
+agree: a thin delta rounds away on the grid however low its capture, and
+a low-rank delta hosts exactly however fat it is.
 """
 
 import torch
 
 from modules import devices, shared
-from modules.lora import lora_calib
+from modules.lora import lora_calib, lora_factor_cache
 from modules.lora import lora_common as l
 from modules.logger import log
 
 
 fallback_layers: list[str] = []
 hosted_layers: list[tuple[str, float, bool]] = []
+hosted_ranks: list[int] = []
+routed_layers: list[str] = []
+
+REQUANT_RATIO = 0.30 # delta rms over mean grid step above which requantize can retain the delta
+REQUANT_ENERGY = 0.90 # sketch capture below which truncation genuinely loses part of it
+NULL_TAIL_EPS = 1e-6 # spectrum tail below this fraction of the capture is numerically null; dropping it keeps stored rank at the delta's effective rank
+
+def rank_bucket(r):
+    """Fixed rank ladder for compiled-graph reuse: powers of two up to 256, multiples of 64 above (hosted rank plus exact members)."""
+    if r <= 8:
+        return 8
+    if r <= 256:
+        return 1 << (r - 1).bit_length()
+    return -(-r // 64) * 64
+
+
+def pad_rank(t, dim, bucket):
+    if t.shape[dim] >= bucket:
+        return t
+    shape = list(t.shape)
+    shape[dim] = bucket - t.shape[dim]
+    return torch.cat([t, t.new_zeros(shape)], dim=dim)
 
 
 def enabled():
@@ -57,6 +88,16 @@ def signature():
     return '' if enabled() else '|quant=requantize'
 
 
+def trim_null_tail(up_h, down_h):
+    """Cache entries stored before tail slicing carry null ranks as exact zero columns; trim to the effective rank on attach."""
+    nz = (up_h != 0).any(dim=0)
+    if not bool(nz.all()):
+        k = max(1, int(nz.nonzero().max().item()) + 1) if bool(nz.any()) else 1
+        if k < up_h.shape[1]:
+            return up_h[:, :k].contiguous(), down_h[:k].contiguous()
+    return up_h, down_h
+
+
 def get_module_factors(module, device, dtype, original_shape=None):
     """Return ``(up_eff, down)`` reproducing ``calc_updown`` exactly, or None.
 
@@ -85,7 +126,7 @@ def get_module_factors(module, device, dtype, original_shape=None):
     return up_eff.to(dtype=dtype), down.to(device=device, dtype=dtype)
 
 
-def factor_candidate(self, network_layer_name, wanted_names, use_previous=False):
+def factor_candidate(self, network_layer_name, wanted_names):
     """True when this layer should take the exact svd-append path.
 
     Requires an SDNQ linear layer whose active networks all contribute plain
@@ -100,9 +141,8 @@ def factor_candidate(self, network_layer_name, wanted_names, use_previous=False)
         return True
     if wanted_names == ():  # nothing attached, nothing to remove
         return False
-    loaded = l.loaded_networks if not use_previous else l.previously_loaded_networks
     seen = False
-    for net in loaded:
+    for net in l.loaded_networks:
         module = net.modules.get(network_layer_name, None)
         if module is None:
             continue
@@ -134,7 +174,7 @@ def remove_factors(self):
     return True
 
 
-def apply_factors(self, network_layer_name, wanted_names, use_previous=False):
+def apply_factors(self, network_layer_name, wanted_names):
     """Attach the active networks' LoRA factors to this layer's svd side-channel.
 
     Replaces any previously attached factors (multiplier changes re-enter
@@ -150,9 +190,9 @@ def apply_factors(self, network_layer_name, wanted_names, use_previous=False):
 
     deq = self.sdnq_dequantizer
     dtype = deq.result_dtype
-    loaded = l.loaded_networks if not use_previous else l.previously_loaded_networks
+
     ups, downs = [], []
-    for net in loaded:
+    for net in l.loaded_networks:
         module = net.modules.get(network_layer_name, None)
         if module is None:
             continue
@@ -187,12 +227,19 @@ def append_factors(self, ups, downs):
         parts_down = ([orig_down.to(device=devices.device, dtype=dtype)] if orig_down is not None else []) + downs
         new_up = torch.cat(parts_up, dim=1).contiguous()
         new_down = torch.cat(parts_down, dim=0).contiguous()
+    from sdnq.common import use_torch_compile
+    if use_torch_compile:
+        # the compiled dequant specializes per factor rank; pad to a fixed bucket so set switches inside a bucket reuse the graph (zero columns contribute exactly nothing)
+        dim_up, dim_down = (0, 1) if deq.use_quantized_matmul else (1, 0)
+        bucket = rank_bucket(new_up.shape[dim_up])
+        new_up = pad_rank(new_up, dim_up, bucket)
+        new_down = pad_rank(new_down, dim_down, bucket)
     self.sdnq_lora_svd_stash = (orig_up, orig_down)
     self.svd_up = torch.nn.Parameter(new_up.to(device=device), requires_grad=False)
     self.svd_down = torch.nn.Parameter(new_down.to(device=device), requires_grad=False)
 
 
-def host_candidate(self, network_layer_name, wanted_names, use_previous=False):
+def host_candidate(self, network_layer_name, wanted_names):
     """True when a non-factorable set on this layer should be hosted as a truncated svd."""
     if not enabled():
         return False
@@ -205,11 +252,57 @@ def host_candidate(self, network_layer_name, wanted_names, use_previous=False):
     from sdnq.common import dtype_dict
     if dtype_dict[self.sdnq_dequantizer.weights_dtype]['num_bits'] >= 8:
         return False # requantize retains most of the delta at 8 bits and above; truncation would lose more than it saves
-    loaded = l.loaded_networks if not use_previous else l.previously_loaded_networks
-    return any(net.modules.get(network_layer_name, None) is not None for net in loaded)
+
+    return any(net.modules.get(network_layer_name, None) is not None for net in l.loaded_networks)
 
 
-def apply_hosted(self, network_layer_name, updown, wanted_names, use_previous=False):
+def apply_cached(self, network_layer_name, wanted_names):
+    """Attach a hosted set straight from the factor cache, before the delta exists.
+
+    Probed by the walk ahead of delta assembly: on a usable entry the routing
+    rule is evaluated from the stored delta rms and the cached factors attach
+    exactly as a fetch inside ``apply_hosted`` would, so the pass skips
+    ``calc_updown`` for the layer entirely. Returns True when the layer was
+    served; None sends the caller down the assemble-and-host path (no entry,
+    or the rule wants the grid).
+    """
+    from sdnq.quant_utils import rotate_hadamard
+
+    lora_factor_cache.begin_pass(wanted_names)
+    entry = lora_factor_cache.lookup(network_layer_name)
+    if entry is None:
+        return None
+    up_h, down_h, energy, calibrated, rms = entry
+    up_h, down_h = trim_null_tail(up_h, down_h)
+    deq = self.sdnq_dequantizer
+    dtype = deq.result_dtype
+    remove_factors(self) # before the rule: the svd-channel check must see the checkpoint's own state, and a declined layer must fall through pristine
+    members = []
+    for net in l.loaded_networks:
+        module = net.modules.get(network_layer_name, None)
+        if module is None:
+            continue
+        factors = get_module_factors(module, devices.device, dtype, original_shape=deq.original_shape)
+        if factors is not None:
+            members.append(factors)
+    if len(members) == 0 and self.svd_up is None:
+        step = float(self.scale.detach().float().mean())
+        if step > 0 and rms / step > REQUANT_RATIO and energy < REQUANT_ENERGY:
+            return None # routed to the grid: the caller assembles the delta and requantizes
+    ups, downs = [], []
+    for up_eff, down in members:
+        if deq.use_hadamard:
+            down = rotate_hadamard(down.to(dtype=torch.float32), group_size=deq.hadamard_group_size).to(dtype=dtype)
+        ups.append(up_eff)
+        downs.append(down)
+    lora_factor_cache.note_hit()
+    append_factors(self, ups + [up_h.to(device=devices.device, dtype=dtype)], downs + [down_h.to(device=devices.device, dtype=dtype)])
+    hosted_layers.append((network_layer_name, energy, calibrated))
+    hosted_ranks.append(int(up_h.shape[1]))
+    return True
+
+
+def apply_hosted(self, network_layer_name, updown, wanted_names):
     """Host a set's delta on the svd channel: exact factors for factorable
     members, the top-k singular directions of the remainder for the rest.
 
@@ -218,9 +311,11 @@ def apply_hosted(self, network_layer_name, updown, wanted_names, use_previous=Fa
     appended exactly so they never compete with the hosted remainder for
     rank. When per-checkpoint activation statistics exist (``lora_calib``),
     input channels are weighted by their RMS before truncation so the kept
-    directions minimize output error rather than weight error. Returns None
-    when the delta cannot ride the channel (wrong shape); the caller falls
-    back to requantize.
+    directions minimize output error rather than weight error. Computed
+    factors are disk-cached per configuration (``lora_factor_cache``) and
+    replayed bit-identically on later applies. Returns None when the delta
+    cannot ride the channel (wrong shape) or when the routing rule prefers
+    the grid for it; the caller falls back to requantize.
     """
     from sdnq.quant_utils import rotate_hadamard
 
@@ -231,24 +326,73 @@ def apply_hosted(self, network_layer_name, updown, wanted_names, use_previous=Fa
     if updown is None or updown.ndim != 2 or tuple(updown.shape) != tuple(deq.original_shape):
         return None
     dtype = deq.result_dtype
-    D = updown.detach().to(devices.device, torch.float32)
 
-    ups, downs = [], []
-    loaded = l.loaded_networks if not use_previous else l.previously_loaded_networks
-    for net in loaded:
+    members = []
+    for net in l.loaded_networks:
         module = net.modules.get(network_layer_name, None)
         if module is None:
             continue
         factors = get_module_factors(module, devices.device, dtype, original_shape=deq.original_shape)
-        if factors is None:
-            continue
-        up_eff, down = factors
-        D = D.sub_(up_eff.to(torch.float32) @ down.to(torch.float32)) # factorable members ride exactly; host only the remainder
+        if factors is not None:
+            members.append(factors)
+
+    # requantize keeps a delta the grid can resolve and that truncation would genuinely
+    # cut: both terms must agree, since a thin delta rounds away on the grid however
+    # low its capture, and a low-rank delta hosts exactly however fat it is. Scoped to
+    # sets the side-channel would otherwise carry whole: factorable members ride
+    # exactly.
+    delta_rms = float(updown.detach().float().square().mean().sqrt())
+    maybe_requant = len(members) == 0 and self.svd_up is None
+    if maybe_requant:
+        step = float(self.scale.detach().float().mean())
+        maybe_requant = step > 0 and delta_rms / step > REQUANT_RATIO
+
+    lora_factor_cache.begin_pass(wanted_names)
+    cached = lora_factor_cache.fetch(network_layer_name)
+    D = None if cached is not None else updown.detach().to(devices.device, torch.float32)
+
+    ups, downs = [], []
+    for up_eff, down in members:
+        if D is not None:
+            D = D.sub_(up_eff.to(torch.float32) @ down.to(torch.float32)) # factorable members ride exactly; host only the remainder
         if deq.use_hadamard:
             down = rotate_hadamard(down.to(dtype=torch.float32), group_size=deq.hadamard_group_size).to(dtype=dtype)
         ups.append(up_eff)
         downs.append(down)
 
+    if cached is not None:
+        up_h, down_h, energy, calibrated, _cached_rms = cached
+        if maybe_requant and energy < REQUANT_ENERGY:
+            routed_layers.append(network_layer_name)
+            return None
+        up_h, down_h = trim_null_tail(up_h, down_h)
+        append_factors(self, ups + [up_h.to(device=devices.device, dtype=dtype)], downs + [down_h.to(device=devices.device, dtype=dtype)])
+        hosted_layers.append((network_layer_name, energy, calibrated))
+        hosted_ranks.append(int(up_h.shape[1]))
+        return True
+
+    up_h, down_h, energy, calibrated = truncate_delta(self, D, dtype)
+    up_h, down_h = lora_factor_cache.store(network_layer_name, up_h, down_h, energy, calibrated, delta_rms)
+    if maybe_requant and energy < REQUANT_ENERGY:
+        routed_layers.append(network_layer_name) # the stored entry memoizes the routing; replays skip the sketch
+        return None
+    up_h, down_h = trim_null_tail(up_h, down_h) # the int8 roundtrip zeroes the numeric tail the eps slice keeps; fresh and replayed attaches must trim alike
+    append_factors(self, ups + [up_h], downs + [down_h])
+    hosted_layers.append((network_layer_name, energy, calibrated))
+    hosted_ranks.append(int(up_h.shape[1]))
+    return True
+
+
+def truncate_delta(self, D, dtype):
+    """Truncate one dense fp32 delta to hosted factors in the layer's channel layout; consumes ``D``.
+
+    Calibration-weighted when statistics exist; the sketch is oversampled past
+    the kept rank so the truncation sits within noise of exact svd. Returns
+    ``(up_h, down_h, energy, calibrated)`` with the down factor rotated into the
+    layer's hadamard domain.
+    """
+    from sdnq.quant_utils import rotate_hadamard
+    deq = self.sdnq_dequantizer
     cap = int(shared.opts.lora_sdnq_host_rank)
     q = min(cap, *D.shape)
     rms = lora_calib.rms_for(self)
@@ -261,7 +405,17 @@ def apply_hosted(self, network_layer_name, updown, wanted_names, use_previous=Fa
     # svd_lowrank draws random projections; fork so user generation seeds are untouched and re-applies are deterministic
     with torch.random.fork_rng(devices=[D.device] if D.device.type == 'cuda' else []):
         torch.manual_seed(0)
-        U, S, V = torch.svd_lowrank(D, q=q, niter=2)
+        # oversampled sketch with extra power iterations lands within noise of exact svd; only the top q columns are kept
+        U, S, V = torch.svd_lowrank(D, q=min(q + 64, *D.shape), niter=8)
+    U, S, V = U[:, :q], S[:q], V[:, :q]
+    e = S.square()
+    total_e = e.sum()
+    if float(total_e) > 0:
+        # an exactly low-rank delta (dense-combined plain pairs, low-rank LyCORIS) fills the tail with
+        # numerical zeros; storing them would pad the channel to the cap for nothing
+        k = int((torch.cumsum(e, 0) < (1.0 - NULL_TAIL_EPS) * total_e).sum().item()) + 1
+        if k < q:
+            U, S, V = U[:, :k], S[:k], V[:, :k]
     energy = float(S.square().sum() / D.square().sum().clamp(min=1e-30)) # captured fraction, in the weighted domain when calibrated
     up_h = (U * S).to(dtype=dtype)
     down_h = V.t()
@@ -269,26 +423,38 @@ def apply_hosted(self, network_layer_name, updown, wanted_names, use_previous=Fa
         down_h = down_h / rms # unscale in the original input basis, before any rotation
     if deq.use_hadamard:
         down_h = rotate_hadamard(down_h, group_size=deq.hadamard_group_size)
-    append_factors(self, ups + [up_h], downs + [down_h.to(dtype=dtype)])
-    hosted_layers.append((network_layer_name, energy, rms is not None))
-    return True
+    down_h = down_h.to(dtype=dtype)
+    return up_h, down_h, energy, rms is not None
 
 
 def note_fallback(self, network_layer_name):
-    """Record a quantized layer taking the lossy requantize path (summary-logged per pass)."""
-    if getattr(self, 'sdnq_dequantizer', None) is not None:
+    """Record a quantized layer taking the requantize path (summary-logged per pass); layers the routing rule sent there are counted apart."""
+    if getattr(self, 'sdnq_dequantizer', None) is not None and network_layer_name not in routed_layers:
         fallback_layers.append(network_layer_name)
 
 
 def report_fallbacks():
+    hits, misses = lora_factor_cache.flush()
+    if hits > 0 or misses > 0:
+        log.info(f'Network load: type=LoRA quant=sdnq cache hits={hits} misses={misses}')
     if len(hosted_layers) > 0:
         energies = sorted(e for _name, e, _c in hosted_layers)
         median = energies[len(energies) // 2]
         calibrated = sum(1 for _name, _e, c in hosted_layers if c)
-        log.info(f'Network load: type=LoRA quant=sdnq hosted={len(hosted_layers)} rank={int(shared.opts.lora_sdnq_host_rank)}{f" calib={calibrated}" if calibrated else ""} energy={median:.2f} min={energies[0]:.2f} non-factorable networks hosted on the svd side-channel')
+        ranks = ''
+        if len(hosted_ranks) > 0 and min(hosted_ranks) < int(shared.opts.lora_sdnq_host_rank):
+            rs = sorted(hosted_ranks)
+            ranks = f' k={rs[0]}-{rs[len(rs) // 2]}-{rs[-1]}' # realized rank spread; shown only when a spectrum collapsed below the cap
+        log.info(f'Network load: type=LoRA quant=sdnq hosted={len(hosted_layers)} rank={int(shared.opts.lora_sdnq_host_rank)}{ranks}{f" calib={calibrated}" if calibrated else ""} energy={median:.2f} min={energies[0]:.2f} non-factorable networks hosted on the svd side-channel')
         if l.debug:
             log.debug(f'Network load: type=LoRA quant=sdnq hosted={[(n, round(e, 3)) for n, e, _c in hosted_layers[:8]]}{"..." if len(hosted_layers) > 8 else ""}')
     hosted_layers.clear()
+    hosted_ranks.clear()
+    if len(routed_layers) > 0:
+        log.info(f'Network load: type=LoRA quant=sdnq apply=requantize layers={len(routed_layers)} routed=fat-delta')
+        if l.debug:
+            log.debug(f'Network load: type=LoRA quant=sdnq routed={routed_layers[:8]}{"..." if len(routed_layers) > 8 else ""}')
+    routed_layers.clear()
     if len(fallback_layers) > 0:
         if enabled():
             log.warning(f'Network load: type=LoRA quant=sdnq layers={len(fallback_layers)} non-factorable networks requantized in place (reduced fidelity on quantized weights)')

modules/lora/networks.py

@@ -106,6 +106,7 @@ def network_activate(include=None, exclude=None):
                         continue
                     if group_offload and component not in group_stripped and group_will_mutate(module, network_layer_name, l.loaded_networks):
                         device = group_offload_strip(sd_model, component, group_stripped)
+                    calced = False # tracks whether this iteration assembled the delta, so the fallthrough reuses it instead of recomputing
                     if lora_sdnq.factor_candidate(module, network_layer_name, component_wanted):
                         weights_backup = getattr(module, "network_weights_backup", None)
                         if weights_backup is not None and not isinstance(weights_backup, bool):
@@ -124,22 +125,24 @@ def network_activate(include=None, exclude=None):
                         weights_backup = getattr(module, "network_weights_backup", None)
                         if weights_backup is not None and not isinstance(weights_backup, bool):
                             network_apply_weights(module, None, None, device=device) # the hosted delta is measured against the pristine base
-                        batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name, elimit=elimit)
-                        if batch_ex_bias is None: # bias deltas need the plain path; weight-only sets ride the side-channel without a weight backup
-                            hosted = lora_sdnq.apply_hosted(module, network_layer_name, batch_updown, component_wanted)
+                        hosted = lora_sdnq.apply_cached(module, network_layer_name, component_wanted) # a stored entry serves the layer before the delta is assembled
+                        if hosted is None:
+                            batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name, elimit=elimit)
+                            calced = True
+                            if batch_ex_bias is None: # bias deltas need the plain path; weight-only sets ride the side-channel without a weight backup
+                                hosted = lora_sdnq.apply_hosted(module, network_layer_name, batch_updown, component_wanted)
                             if hosted is not None:
-                                if hosted and component_wanted:
-                                    applied_layers.append(network_layer_name)
-                                    applied_weight += 1
-                                module.network_current_names = component_wanted
-                                module.network_current_stack = stack_sig
                                 batch_updown, batch_ex_bias = None, None
                                 del batch_updown, batch_ex_bias
-                                if task is not None:
-                                    pbar.update(task, advance=1)
-                                continue
-                        batch_updown, batch_ex_bias = None, None
-                        del batch_updown, batch_ex_bias
+                        if hosted is not None:
+                            if hosted and component_wanted:
+                                applied_layers.append(network_layer_name)
+                                applied_weight += 1
+                            module.network_current_names = component_wanted
+                            module.network_current_stack = stack_sig
+                            if task is not None:
+                                pbar.update(task, advance=1)
+                            continue
                     stripped = lora_sdnq.remove_factors(module) # the mechanism gate can decline a layer still carrying attached factors; the weight path must start from the pristine channel
                     if stripped and not component_wanted: # factor-mode layers have no tensor backup, dropping the factors is the whole restore
                         module.network_current_names = ()
@@ -156,7 +159,8 @@ def network_activate(include=None, exclude=None):
                             continue
                         batch_updown, batch_ex_bias = None, None # restore-only pass, apply with no weights reverts to backup
                     else:
-                        batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name, elimit=elimit)
+                        if not calced: # the host branch may have assembled the delta already; a declined layer reuses it
+                            batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name, elimit=elimit)
                         if batch_updown is not None:
                             lora_sdnq.note_fallback(module, network_layer_name) # only layers whose quantized weight actually takes a delta
                     if fuse:

modules/ui_definitions.py

@@ -707,6 +707,7 @@ def create_settings(cmd_opts):
         "lora_sdnq_apply": OptionInfo("exact", "LoRA quantized apply method", gr.Radio, {"choices": ["exact", "requantize"]}),
         "lora_sdnq_host_rank": OptionInfo(256, "LoRA quantized host rank", gr.Slider, {"minimum": 0, "maximum": 1024, "step": 32}),
         "lora_sdnq_host_calib": OptionInfo(True, "LoRA quantized host calibration"),
+        "lora_sdnq_host_cache": OptionInfo(10, "LoRA quantized host cache", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1}),
 
         "lora_meta_sep": OptionInfo("<h2>Metadata</h2>", "", gr.HTML),
         "lora_add_hashes_to_infotext": OptionInfo(False, "LoRA add hash info to metadata"),

test/test-sdnq-lora-factors.py

@@ -34,6 +34,14 @@ for the per-model analyzer):
   plain truncation bit-exact, and the capture hooks accumulate, persist
   and reload statistics correctly, gated by option, format width and
   model compile.
+- Factor cache: hosted factors replay bit-identically from the disk cache
+  without re-running the svd, a configuration change (multiplier) misses
+  and writes a separate entry, and budget 0 writes nothing.
+- Compile: the factor add runs inside the single compiled dequant graph
+  (fullgraph, no breaks) and matches the eager result; factor ranks pad to
+  a fixed bucket ladder so set switches inside a bucket reuse the compiled
+  graph while a novel bucket compiles exactly once, and padding changes
+  the dequantized weight by nothing beyond reduction-order ulp.
 
 All tensors are synthetic; no model files or running server required.
 
@@ -698,6 +706,34 @@ def test_hosted_dense_delta_beats_requant():
     return True
 
 
+def test_hosted_null_tail_collapses_to_effective_rank():
+    layer = build_layer('uint4')
+    _A, _B, D = make_delta(sigma=3e-3) # exact rank-8 content in a non-factorable container
+    net = make_dense_net('nulltail', layer, D)
+    with host_rank(256), mock_model(lin=layer):
+        Wdq0 = dq(layer)
+        activate(net)
+        assert layer.svd_up.shape[1] == 8, f'rank-8 delta under cap 256 must store 8 ranks, got {layer.svd_up.shape[1]}'
+        assert layer.svd_down.shape[0] == 8, f'down factor must slice with the up factor, got {layer.svd_down.shape[0]}'
+        rho = rho_of(dq(layer) - Wdq0, D)
+        assert rho > 0.95, f'collapsing the null tail must not cost fidelity: rho={rho:.4f}'
+        activate()
+        assert torch.equal(dq(layer), Wdq0), 'unload must restore bit-exact'
+    return True
+
+
+def test_hosted_flat_spectrum_keeps_cap():
+    layer = build_layer('uint4')
+    torch.manual_seed(13)
+    D = torch.randn(OUT_F, IN_F, device=DEVICE) * 3e-4 # full-rank gaussian: no null tail inside the cap
+    net = make_dense_net('flattail', layer, D)
+    with host_rank(64), mock_model(lin=layer):
+        activate(net)
+        assert layer.svd_up.shape[1] == 64, f'a flat spectrum must keep the full cap, got {layer.svd_up.shape[1]}'
+        activate()
+    return True
+
+
 def test_hosted_skips_int8():
     layer = build_layer('int8')
     _A, _B, D = make_delta(sigma=3e-3)
@@ -745,6 +781,116 @@ def test_hosted_transitions_and_rng_isolation():
     return True
 
 
+@contextmanager
+def requant_rule(ratio, energy):
+    old_r, old_e = lora_sdnq.REQUANT_RATIO, lora_sdnq.REQUANT_ENERGY
+    lora_sdnq.REQUANT_RATIO, lora_sdnq.REQUANT_ENERGY = ratio, energy
+    try:
+        yield
+    finally:
+        lora_sdnq.REQUANT_RATIO, lora_sdnq.REQUANT_ENERGY = old_r, old_e
+
+
+def test_route_fat_dense_delta_requantizes():
+    layer = build_layer('uint4')
+    torch.manual_seed(21)
+    D = torch.randn(OUT_F, IN_F, device=DEVICE) * 1e-2 # full-rank and well above the grid step: the grid retains it, truncation would cut it
+    net = make_dense_net('fatnet', layer, D)
+    with host_rank(256), mock_model(lin=layer):
+        Wdq0 = dq(layer)
+        activate(net)
+        assert not hasattr(layer, 'sdnq_lora_svd_stash'), 'a fat full-rank delta must route to requantize'
+        assert isinstance(getattr(layer, 'network_weights_backup', None), torch.Tensor), 'the routed layer takes the requantize backup'
+        rho = rho_of(dq(layer) - Wdq0, D)
+        assert rho > 0.7, f'the grid must retain the routed delta: rho={rho:.3f}'
+        activate()
+        assert torch.equal(dq(layer), Wdq0), 'restore from backup must be bit-exact'
+    return True
+
+
+def test_route_rule_terms_gate_both_ways():
+    layer = build_layer('uint4')
+    torch.manual_seed(23)
+    D = torch.randn(OUT_F, IN_F, device=DEVICE) * 1e-2 # sr about 0.8, capture about 0.8 at cap 256: each term alone can hold it hosted
+    net = make_dense_net('gatenet', layer, D)
+    with host_rank(256), mock_model(lin=layer):
+        with requant_rule(ratio=10.0, energy=0.90):
+            activate(net)
+            assert hasattr(layer, 'sdnq_lora_svd_stash'), 'sr below the ratio must host regardless of capture'
+            activate()
+        with requant_rule(ratio=0.30, energy=0.0):
+            activate(net)
+            assert hasattr(layer, 'sdnq_lora_svd_stash'), 'capture above the energy floor must host regardless of sr'
+            activate()
+        with requant_rule(ratio=0.30, energy=0.90):
+            activate(net)
+            assert not hasattr(layer, 'sdnq_lora_svd_stash'), 'both terms crossed must requantize'
+            activate()
+    return True
+
+
+def test_route_low_rank_fat_delta_stays_hosted():
+    layer = build_layer('uint4')
+    _A, _B, D = make_delta(seed=22, sigma=3e-3) # rank-8: fat against the grid, exact under the cap
+    net = make_dense_net('fatlow', layer, D)
+    with host_rank(64), mock_model(lin=layer):
+        Wdq0 = dq(layer)
+        activate(net)
+        assert hasattr(layer, 'sdnq_lora_svd_stash'), 'a low-rank delta hosts exactly at any magnitude'
+        rho = rho_of(dq(layer) - Wdq0, D)
+        assert rho > 0.95, f'rho={rho:.4f}'
+        activate()
+    return True
+
+
+def test_route_mixed_set_keeps_hosting():
+    layer = build_layer('uint4')
+    A, B, _D1 = make_delta(seed=24)
+    torch.manual_seed(25)
+    D2 = torch.randn(OUT_F, IN_F, device=DEVICE) * 1e-2
+    net1 = make_net('mixp', layer, A, B)
+    net2 = make_dense_net('mixf', layer, D2)
+    with host_rank(256), mock_model(lin=layer):
+        Wdq0 = dq(layer)
+        activate(net1, net2)
+        assert hasattr(layer, 'sdnq_lora_svd_stash'), 'a set with factorable members keeps the side-channel'
+        activate()
+        assert torch.equal(dq(layer), Wdq0)
+    return True
+
+
+def test_route_svd_checkpoint_keeps_hosting():
+    layer = build_layer('uint4', use_svd=True)
+    torch.manual_seed(26)
+    D = torch.randn(OUT_F, IN_F, device=DEVICE) * 1e-2
+    net = make_dense_net('svdfat', layer, D)
+    with host_rank(256), mock_model(lin=layer):
+        activate(net)
+        assert hasattr(layer, 'sdnq_lora_svd_stash'), 'svd checkpoints keep hosting; the rule is not grounded there'
+        activate()
+    return True
+
+
+def test_route_replay_from_cache():
+    import tempfile
+    layer = build_layer('uint4')
+    with tempfile.TemporaryDirectory() as tmp:
+        with host_rank(256), host_cache(10, os.path.join(tmp, 'cache')), mock_model(lin=layer):
+            net, _D = cache_fixture(tmp, layer, name='fatcache', sigma=1e-2, seed=28)
+            activate(net)
+            assert not hasattr(layer, 'sdnq_lora_svd_stash'), 'fat delta must route on the fresh-sketch path'
+            activate()
+            real_svd = torch.svd_lowrank
+            torch.svd_lowrank = raise_no_svd
+            try:
+                activate(net) # the stored entry memoizes the routing: same decision, no sketch
+            finally:
+                torch.svd_lowrank = real_svd
+            assert not hasattr(layer, 'sdnq_lora_svd_stash'), 'cache replay must route the same way'
+            activate()
+    return True
+
+
 CAT_CALIB = category('calibration')
 
 
@@ -850,7 +996,7 @@ def test_calib_capture_persist_roundtrip():
             lora_calib.calib_root = tmp
             lora_calib.TOKENS_DONE = 2048
             lora_calib.on_model_loaded(sd)
-            assert len(lora_calib.capture['handles']) == 2, 'both sub-8-bit linears must hook'
+            assert len(lora_calib.capture['handles']) == 3, 'both sub-8-bit linears plus the root forward counter must hook'
             torch.manual_seed(51)
             scale = torch.linspace(0.1, 4.0, IN_F, device=DEVICE)
             xs = []
@@ -897,6 +1043,462 @@ def test_calib_capture_gates():
     return True
 
 
+class MockCalibDenoiser(torch.nn.Module):
+    """Denoiser whose forward feeds one token-rich linear and one token-starved one, like a DiT block beside its modulation projection."""
+    def __init__(self, rich, starved, starved_tokens):
+        super().__init__()
+        self.rich = rich
+        self.starved = starved
+        self.starved_tokens = starved_tokens
+
+    def forward(self, x):
+        self.rich(x)
+        self.starved(x[:self.starved_tokens])
+        return x
+
+
+def test_calib_deadline_persists_starved_layers():
+    import tempfile
+    from safetensors import safe_open
+    from modules.lora import lora_calib
+    rich = build_layer('uint4', seed=45)
+    starved = build_layer('uint4', seed=46)
+    root = MockCalibDenoiser(rich, starved, starved_tokens=8)
+    sd = MockCalibSd('test/calib-deadline')
+    sd.transformer = root
+    old = (lora_calib.calib_root, lora_calib.TOKENS_DONE, lora_calib.FORWARDS_DEADLINE)
+    with tempfile.TemporaryDirectory() as tmp, host_calib(True):
+        try:
+            lora_calib.calib_root = tmp
+            lora_calib.TOKENS_DONE = 2048
+            lora_calib.FORWARDS_DEADLINE = 6
+            lora_calib.on_model_loaded(sd)
+            assert len(lora_calib.capture['handles']) == 3, 'two layer hooks plus the root forward counter must attach'
+            torch.manual_seed(52)
+            xs = []
+            for i in range(6):
+                x = torch.randn(1024, IN_F, device=DEVICE).to(torch.bfloat16)
+                if i < 5: # the deadline fires at the start of the sixth forward, before its layer hooks run
+                    xs.append(x[:8].float())
+                root(x)
+            assert lora_calib.capture['complete'], 'the forward deadline must close capture'
+            assert lora_calib.capture['forwards'] == 6, f'root counter must track denoiser forwards, got {lora_calib.capture["forwards"]}'
+            path = lora_calib.calib_file('test/calib-deadline')
+            assert os.path.isfile(path), 'deadline persist must write the statistics file'
+            assert getattr(starved, 'sdnq_calib_rms', None) is not None, 'the starved layer must carry statistics'
+            expected = torch.cat(xs).square().mean(dim=0).sqrt().cpu()
+            assert torch.allclose(starved.sdnq_calib_rms, expected, rtol=1e-3, atol=1e-5), 'starved rms must match exactly the tokens it saw'
+            with safe_open(path, framework='pt', device='cpu') as f:
+                assert set(f.keys()) == {'rich', 'starved'}, f'both layers must persist, got {sorted(f.keys())}'
+                assert f.metadata()['tokens'] == '40', f'metadata must report the weakest saved layer, got {f.metadata()["tokens"]}'
+        finally:
+            lora_calib.calib_root, lora_calib.TOKENS_DONE, lora_calib.FORWARDS_DEADLINE = old
+            lora_calib.detach_capture()
+    return True
+
+
+def test_calib_deadline_omits_subfloor_layers():
+    import tempfile
+    from safetensors import safe_open
+    from modules.lora import lora_calib
+    rich = build_layer('uint4', seed=48)
+    starved = build_layer('uint4', seed=49)
+    root = MockCalibDenoiser(rich, starved, starved_tokens=2) # 2 tokens x 5 counted forwards = 10, under the floor of 32
+    sd = MockCalibSd('test/calib-subfloor')
+    sd.transformer = root
+    old = (lora_calib.calib_root, lora_calib.TOKENS_DONE, lora_calib.FORWARDS_DEADLINE)
+    with tempfile.TemporaryDirectory() as tmp, host_calib(True):
+        try:
+            lora_calib.calib_root = tmp
+            lora_calib.TOKENS_DONE = 2048
+            lora_calib.FORWARDS_DEADLINE = 6
+            lora_calib.on_model_loaded(sd)
+            torch.manual_seed(53)
+            for _ in range(6):
+                root(torch.randn(1024, IN_F, device=DEVICE).to(torch.bfloat16))
+            assert lora_calib.capture['complete'], 'the forward deadline must close capture'
+            path = lora_calib.calib_file('test/calib-subfloor')
+            with safe_open(path, framework='pt', device='cpu') as f:
+                assert set(f.keys()) == {'rich'}, f'a layer under the token floor must be omitted, got {sorted(f.keys())}'
+            assert getattr(starved, 'sdnq_calib_rms', None) is None, 'an omitted layer must not carry statistics'
+            del rich.sdnq_calib_rms
+            lora_calib.on_model_loaded(sd) # second load takes the cached path with the partial file
+            assert len(lora_calib.capture['handles']) == 0, 'a partial file still counts as cached; capture must not re-attach'
+            assert getattr(rich, 'sdnq_calib_rms', None) is not None, 'the saved layer must reload from the partial file'
+            assert getattr(starved, 'sdnq_calib_rms', None) is None, 'the omitted layer must stay on plain truncation after reload'
+        finally:
+            lora_calib.calib_root, lora_calib.TOKENS_DONE, lora_calib.FORWARDS_DEADLINE = old
+            lora_calib.detach_capture()
+    return True
+
+
+def test_calib_unet_root_walk():
+    import tempfile
+    from modules.lora import lora_calib
+    layer = build_layer('uint4', seed=47)
+    sd = MockCalibSd('test/calib-unet', lin=layer)
+    sd.unet = sd.transformer
+    sd.transformer = None
+    mods = lora_calib.eligible_modules(sd)
+    assert [n for n, _ in mods] == ['lin'], f'the unet root must be walked when no transformer exists, got {[n for n, _ in mods]}'
+    both = MockCalibSd('test/calib-both')
+    both.unet = sd.unet
+    assert lora_calib.eligible_modules(both) == [], 'a transformer root wins even when it holds no eligible linears'
+    old_root = lora_calib.calib_root
+    with tempfile.TemporaryDirectory() as tmp, host_calib(True):
+        try:
+            lora_calib.calib_root = tmp
+            lora_calib.on_model_loaded(sd)
+            assert len(lora_calib.capture['handles']) == 2, 'the layer hook plus the root counter must attach on a unet model'
+        finally:
+            lora_calib.calib_root = old_root
+            lora_calib.detach_capture()
+    return True
+
+
+CAT_FCACHE = category('factor-cache')
+
+
+@contextmanager
+def host_cache(gb, root):
+    from modules.lora import lora_factor_cache
+    old_gb = getattr(shared.opts, 'lora_sdnq_host_cache', 0)
+    old_root = lora_factor_cache.cache_root
+    shared.opts.lora_sdnq_host_cache = gb
+    lora_factor_cache.cache_root = root
+    lora_factor_cache.state.update(wn=None, sig=None, path=None, dirty=False, hits=0, misses=0)
+    lora_factor_cache.state['store'] = {}
+    try:
+        yield lora_factor_cache
+    finally:
+        shared.opts.lora_sdnq_host_cache = old_gb
+        lora_factor_cache.cache_root = old_root
+        lora_factor_cache.state.update(wn=None, sig=None, path=None, dirty=False, hits=0, misses=0)
+        lora_factor_cache.state['store'] = {}
+
+
+def cache_fixture(tmp, layer, name='cachenet', sigma=3e-4, seed=61):
+    """Dense net whose on-disk file exists (signature needs a stat-able path) plus a mock checkpoint identity."""
+    torch.manual_seed(seed)
+    D = torch.randn(OUT_F, IN_F, device=DEVICE) * sigma
+    net = make_dense_net(name, layer, D)
+    lora_file = os.path.join(tmp, f'{name}.safetensors')
+    with open(lora_file, 'wb') as f:
+        f.write(b'0' * 64)
+    net.network_on_disk.filename = lora_file
+    from modules.modeldata import model_data
+    model_data.sd_model.sd_checkpoint_info = MockCheckpointInfo('test/cache-model')
+    return net, D
+
+
+def raise_no_svd(*_args, **_kwargs):
+    raise AssertionError('svd must not run on a cache hit')
+
+
+def test_factor_cache_roundtrip_bitexact():
+    import tempfile
+    layer = build_layer('uint4')
+    with tempfile.TemporaryDirectory() as tmp:
+        with host_rank(64), host_cache(10, os.path.join(tmp, 'cache')), mock_model(lin=layer):
+            net, _D = cache_fixture(tmp, layer)
+            Wdq0 = dq(layer)
+            activate(net)
+            first_up = layer.svd_up.detach().clone()
+            first_down = layer.svd_down.detach().clone()
+            activate() # pass end flushed the entry; unload restores the base
+            files = os.listdir(os.path.join(tmp, 'cache'))
+            assert len(files) == 1, f'one cache entry expected, got {files}'
+            bf16_bytes = (first_up.numel() + first_down.numel()) * 2
+            entry_bytes = os.path.getsize(os.path.join(tmp, 'cache', files[0]))
+            assert entry_bytes < bf16_bytes * 0.62 + 8192, f'int8 entry must be about half the bf16 factor bytes: {entry_bytes} vs {bf16_bytes}'
+            real_svd = torch.svd_lowrank
+            torch.svd_lowrank = raise_no_svd
+            try:
+                activate(net) # same configuration: must replay from disk without touching the svd
+            finally:
+                torch.svd_lowrank = real_svd
+            assert torch.equal(layer.svd_up, first_up), 'cache hit must replay bit-identical up factors'
+            assert torch.equal(layer.svd_down, first_down), 'cache hit must replay bit-identical down factors'
+            activate()
+            assert torch.equal(dq(layer), Wdq0), 'unload must restore bit-exact'
+    return True
+
+
+def test_factor_cache_invalidates_on_multiplier():
+    import tempfile
+    layer = build_layer('uint4')
+    with tempfile.TemporaryDirectory() as tmp:
+        with host_rank(64), host_cache(10, os.path.join(tmp, 'cache')), mock_model(lin=layer):
+            net, _D = cache_fixture(tmp, layer)
+            activate(net)
+            up_full = layer.svd_up.detach().clone()
+            activate()
+            net.te_multiplier = 0.7
+            net.unet_multiplier = [0.7] * 3
+            activate(net) # different multiplier: different signature, fresh svd, second entry
+            assert not torch.equal(layer.svd_up, up_full), 'multiplier change must produce different factors'
+            activate()
+            files = os.listdir(os.path.join(tmp, 'cache'))
+            assert len(files) == 2, f'two cache entries expected, got {files}'
+    return True
+
+
+def test_attach_trims_stored_null_tail():
+    """Entries written before tail slicing pad the channel with null ranks: zero up
+    columns (and junk down rows behind them). Attach must trim to the effective rank
+    and replay the same resident tensors and weights as the unpadded entry."""
+    import tempfile
+    layer = build_layer('uint4')
+    with tempfile.TemporaryDirectory() as tmp:
+        with host_rank(64), host_cache(10, os.path.join(tmp, 'cache')), mock_model(lin=layer):
+            net, _D = cache_fixture(tmp, layer, name='padnet')
+            activate(net)
+            up0 = layer.svd_up.detach().clone()
+            down0 = layer.svd_down.detach().clone()
+            Wl0 = dq(layer)
+            activate()
+            cache_dir = os.path.join(tmp, 'cache')
+            entry = os.path.join(cache_dir, os.listdir(cache_dir)[0])
+            from safetensors import safe_open
+            from safetensors.torch import save_file
+            with safe_open(entry, framework='pt', device='cpu') as f:
+                meta = dict(f.metadata())
+                tensors = {k: f.get_tensor(k) for k in f.keys()}
+            for k in [k for k in tensors if k.endswith('.up_q')]:
+                base = k[: -len('.up_q')]
+                torch.manual_seed(5)
+                tensors[f'{base}.up_q'] = torch.cat([tensors[k], torch.zeros(tensors[k].shape[0], 64, dtype=torch.int8)], dim=1)
+                tensors[f'{base}.down_q'] = torch.cat([tensors[f'{base}.down_q'], torch.randint(-127, 128, (64, IN_F), dtype=torch.int8)], dim=0)
+                tensors[f'{base}.down_s'] = torch.cat([tensors[f'{base}.down_s'], torch.ones(64, 1)], dim=0)
+            save_file(tensors, entry, metadata=meta)
+            real_svd = torch.svd_lowrank
+            torch.svd_lowrank = raise_no_svd
+            try:
+                activate(net)
+            finally:
+                torch.svd_lowrank = real_svd
+            assert layer.svd_up.shape[1] == 64, f'attach must trim the padded tail back to the effective rank, got {layer.svd_up.shape[1]}'
+            assert torch.equal(layer.svd_up, up0) and torch.equal(layer.svd_down, down0), 'trimmed factors must match the unpadded entry'
+            assert torch.equal(dq(layer), Wl0), 'trimmed attach must materialize the same weight'
+            activate()
+    return True
+
+
+def test_factor_cache_int8_quantization():
+    from modules.lora import lora_factor_cache as fc
+    torch.manual_seed(71)
+    t = torch.randn(64, 128, device=DEVICE) * torch.logspace(-3, 0, 64, device=DEVICE)[:, None] # rows spanning magnitudes
+    q, s = fc.quantize_rowwise(t)
+    assert q.dtype == torch.int8
+    dq = fc.dequantize_rowwise(q, s)
+    err = (dq - t).abs().max(dim=1).values
+    assert bool((err <= s.squeeze(1) * 0.51).all()), 'rowwise int8 error must stay within half a step'
+    cos = torch.nn.functional.cosine_similarity(dq.flatten(), t.flatten(), dim=0)
+    assert float(cos) > 0.99995, f'int8 roundtrip cosine {float(cos):.6f}'
+    return True
+
+
+def test_factor_cache_disabled_at_zero():
+    import tempfile
+    layer = build_layer('uint4')
+    with tempfile.TemporaryDirectory() as tmp:
+        with host_rank(64), host_cache(0, os.path.join(tmp, 'cache')), mock_model(lin=layer):
+            net, _D = cache_fixture(tmp, layer)
+            activate(net)
+            activate()
+            assert not os.path.isdir(os.path.join(tmp, 'cache')), 'budget 0 must write nothing'
+    return True
+
+
+def test_factor_cache_invalidates_on_calib_toggle():
+    import tempfile
+    from modules.lora import lora_calib
+    layer = build_layer('uint4')
+    with tempfile.TemporaryDirectory() as tmp:
+        with host_rank(64), host_cache(10, os.path.join(tmp, 'cache')), mock_model(lin=layer):
+            net, _D = cache_fixture(tmp, layer)
+            old_root = lora_calib.calib_root
+            lora_calib.calib_root = os.path.join(tmp, 'calib')
+            os.makedirs(lora_calib.calib_root, exist_ok=True)
+            with open(lora_calib.calib_file('test/cache-model'), 'wb') as f:
+                f.write(b'0' * 64) # the signature stats this file; its content is never read here
+            torch.manual_seed(77)
+            layer.sdnq_calib_rms = torch.rand(IN_F) * 4 + 0.1
+            try:
+                with host_calib(True):
+                    activate(net)
+                    up_cal = layer.svd_up.detach().clone()
+                    activate()
+                with host_calib(False):
+                    activate(net) # same set with calibration off: the entry keyed under the other setting must miss
+                    up_plain = layer.svd_up.detach().clone()
+                    activate()
+                assert not torch.equal(up_cal, up_plain), 'toggling calibration must not replay factors computed under the other setting'
+                assert len(os.listdir(os.path.join(tmp, 'cache'))) == 2, 'the two settings must key separate cache entries'
+            finally:
+                del layer.sdnq_calib_rms
+                lora_calib.calib_root = old_root
+    return True
+
+
+@contextmanager
+def counting_calc():
+    """Count NetworkModuleFull.calc_updown calls: zero on a pass proves the walk skipped delta assembly."""
+    from modules.lora import network_full
+    calls = {'n': 0}
+    real = network_full.NetworkModuleFull.calc_updown
+    def wrapper(self, *args, **kwargs):
+        calls['n'] += 1
+        return real(self, *args, **kwargs)
+    network_full.NetworkModuleFull.calc_updown = wrapper
+    try:
+        yield calls
+    finally:
+        network_full.NetworkModuleFull.calc_updown = real
+
+
+def test_cache_fastpath_skips_calc():
+    import tempfile
+    layer = build_layer('uint4')
+    with tempfile.TemporaryDirectory() as tmp:
+        with host_rank(64), host_cache(10, os.path.join(tmp, 'cache')), mock_model(lin=layer):
+            net, _D = cache_fixture(tmp, layer)
+            Wdq0 = dq(layer)
+            with counting_calc() as calls:
+                activate(net)
+                assert calls['n'] > 0, 'a fresh apply must assemble the delta'
+                first = dq(layer)
+                activate()
+                calls['n'] = 0
+                activate(net)
+                assert calls['n'] == 0, f'a cache replay must not assemble the delta: calc_updown ran {calls["n"]} times'
+                assert hasattr(layer, 'sdnq_lora_svd_stash'), 'the fast path must attach the cached factors'
+                assert torch.equal(dq(layer), first), 'fast-path replay must be bit-identical to the fresh apply'
+                activate()
+                assert torch.equal(dq(layer), Wdq0)
+    return True
+
+
+def test_cache_fastpath_serves_mixed_set():
+    import tempfile
+    layer = build_layer('uint4')
+    A, B, _D1 = make_delta(seed=63)
+    with tempfile.TemporaryDirectory() as tmp:
+        with host_rank(64), host_cache(10, os.path.join(tmp, 'cache')), mock_model(lin=layer):
+            net_full, _D2 = cache_fixture(tmp, layer, name='mixfull', seed=64)
+            net_plain = make_net('mixlora', layer, A, B)
+            lora_file = os.path.join(tmp, 'mixlora.safetensors')
+            with open(lora_file, 'wb') as f:
+                f.write(b'0' * 64)
+            net_plain.network_on_disk.filename = lora_file
+            activate(net_plain, net_full)
+            first = dq(layer)
+            activate()
+            with counting_calc() as calls:
+                activate(net_plain, net_full) # the factorable member re-extracts from its own weights; the hosted remainder replays
+                assert calls['n'] == 0, 'a mixed-set replay must not assemble the delta'
+            assert hasattr(layer, 'sdnq_lora_svd_stash')
+            assert torch.equal(dq(layer), first), 'mixed-set replay must be bit-identical to the fresh apply'
+            activate()
+    return True
+
+
+CAT_COMPILE = category('compile')
+
+
+def dq_compiled(layer):
+    # the production entry: skip_compile left at its default so the shared compiled dequant runs
+    return layer.sdnq_dequantizer(layer.weight, layer.scale, zero_point=layer.zero_point,
+                                  svd_up=layer.svd_up, svd_down=layer.svd_down,
+                                  skip_quantized_matmul=layer.sdnq_dequantizer.use_quantized_matmul,
+                                  dtype=torch.float32)
+
+
+def graph_stats():
+    from torch._dynamo.utils import counters
+    return int(counters['stats']['unique_graphs']), sum(counters['graph_break'].values())
+
+
+def test_factor_add_inside_compiled_graph():
+    from sdnq.common import use_torch_compile
+    if not use_torch_compile:
+        return True # compile disabled at sdnq import (no triton); nothing to pin
+    import torch._dynamo
+    from torch._dynamo.utils import counters
+    layer = build_layer('uint4')
+    A, B, _D = make_delta()
+    dtype = layer.sdnq_dequantizer.result_dtype
+    torch._dynamo.reset()
+    counters.clear()
+    lora_sdnq.append_factors(layer, [B.to(dtype)], [A.to(dtype)])
+    W_c = dq_compiled(layer)
+    graphs, breaks = graph_stats()
+    assert breaks == 0, f'graph breaks in the compiled dequant: {breaks}'
+    assert graphs == 1, f'factor-bearing dequant must be one compiled region, got {graphs} graphs'
+    W_e = dq(layer)
+    assert torch.allclose(W_c, W_e, rtol=1e-3, atol=1e-4), f'compiled vs eager dequant diverged, max {float((W_c - W_e).abs().max()):.3e}'
+    lora_sdnq.remove_factors(layer)
+    return True
+
+
+def test_rank_bucket_graph_reuse():
+    from sdnq.common import use_torch_compile
+    if not use_torch_compile:
+        return True
+    import torch._dynamo
+    from torch._dynamo.utils import counters
+    import sdnq.common as sdnq_common
+    layer = build_layer('uint4', use_hadamard=False)
+    dtype = layer.sdnq_dequantizer.result_dtype
+    torch.manual_seed(13)
+    mk = lambda r: (torch.randn(OUT_F, r, device=DEVICE, dtype=dtype) * 0.01, torch.randn(r, IN_F, device=DEVICE, dtype=dtype) * 0.01)
+    B8, A8 = mk(8)
+    B6, A6 = mk(6)
+    B24, A24 = mk(24)
+
+    torch._dynamo.reset()
+    counters.clear()
+    lora_sdnq.append_factors(layer, [B8], [A8])
+    assert layer.svd_up.shape[1] == 8, f'rank 8 must bucket to 8, got {layer.svd_up.shape[1]}'
+    dq_compiled(layer)
+    g_first, _ = graph_stats()
+
+    lora_sdnq.remove_factors(layer)
+    lora_sdnq.append_factors(layer, [B6], [A6])
+    assert layer.svd_up.shape[1] == 8, f'rank 6 must pad to bucket 8, got {layer.svd_up.shape[1]}'
+    assert float(layer.svd_up[:, 6:].abs().sum()) == 0.0, 'pad columns must be exact zeros'
+    dq_compiled(layer)
+    g_same, _ = graph_stats()
+    assert g_same == g_first, f'same bucket must reuse the graph: {g_first} -> {g_same}'
+
+    lora_sdnq.remove_factors(layer)
+    lora_sdnq.append_factors(layer, [B24], [A24])
+    assert layer.svd_up.shape[1] == 32, f'rank 24 must pad to bucket 32, got {layer.svd_up.shape[1]}'
+    dq_compiled(layer)
+    g_novel, _ = graph_stats()
+    assert g_novel == g_first + 1, f'novel bucket must compile exactly one new graph: {g_first} -> {g_novel}'
+
+    lora_sdnq.remove_factors(layer)
+    lora_sdnq.append_factors(layer, [B8], [A8])
+    dq_compiled(layer)
+    g_back, _ = graph_stats()
+    assert g_back == g_novel, f'returning to a seen bucket must be free: {g_novel} -> {g_back}'
+
+    W_padded = dq(layer)
+    lora_sdnq.remove_factors(layer)
+    old_flag = sdnq_common.use_torch_compile
+    sdnq_common.use_torch_compile = False
+    try:
+        lora_sdnq.append_factors(layer, [B8], [A8])
+        assert layer.svd_up.shape[1] == 8
+        W_unpadded = dq(layer)
+    finally:
+        sdnq_common.use_torch_compile = old_flag
+        lora_sdnq.remove_factors(layer)
+    assert torch.allclose(W_padded, W_unpadded, rtol=0.0, atol=1e-6), f'padding must be inert beyond reduction-order ulp, max {float((W_padded - W_unpadded).abs().max()):.3e}'
+    return True
+
+
 CAT_ROBUST = category('robustness')
 
 
@@ -964,12 +1566,25 @@ def run_tests():
         run_test(CAT_TRANS, fn)
     log.warning('=== Hosting ===')
     for fn in [test_hosted_low_rank_delta_is_kept, test_hosted_dense_delta_beats_requant, test_hosted_skips_int8,
-               test_hosted_disabled_by_option, test_hosted_transitions_and_rng_isolation]:
+               test_hosted_disabled_by_option, test_hosted_transitions_and_rng_isolation,
+               test_route_fat_dense_delta_requantizes, test_route_rule_terms_gate_both_ways, test_route_low_rank_fat_delta_stays_hosted,
+               test_route_mixed_set_keeps_hosting, test_route_svd_checkpoint_keeps_hosting,
+               test_route_replay_from_cache, test_hosted_null_tail_collapses_to_effective_rank, test_hosted_flat_spectrum_keeps_cap]:
         run_test(CAT_HOST, fn)
     log.warning('=== Calibration ===')
     for fn in [test_calibrated_hosting_beats_plain, test_calibrated_low_rank_delta_survives, test_calib_option_off_matches_plain,
-               test_calib_capture_persist_roundtrip, test_calib_capture_gates]:
+               test_calib_capture_persist_roundtrip, test_calib_capture_gates, test_calib_deadline_persists_starved_layers,
+               test_calib_deadline_omits_subfloor_layers, test_calib_unet_root_walk]:
         run_test(CAT_CALIB, fn)
+    log.warning('=== Factor cache ===')
+    for fn in [test_factor_cache_roundtrip_bitexact, test_factor_cache_invalidates_on_multiplier,
+               test_factor_cache_int8_quantization, test_factor_cache_disabled_at_zero, test_factor_cache_invalidates_on_calib_toggle,
+               test_cache_fastpath_skips_calc, test_cache_fastpath_serves_mixed_set,
+               test_attach_trims_stored_null_tail]:
+        run_test(CAT_FCACHE, fn)
+    log.warning('=== Compile ===')
+    for fn in [test_factor_add_inside_compiled_graph, test_rank_bucket_graph_reuse]:
+        run_test(CAT_COMPILE, fn)
     log.warning('=== Robustness ===')
     for fn in [test_remove_factors_after_device_move, test_stacked_shape_mismatch_falls_back]:
         run_test(CAT_ROBUST, fn)

ui/locale/locale_en.json

@@ -864,9 +864,10 @@
     {"id":"","label":"LoRA native apply to text encoder","localized":"","hint":"","ui":"settings_lora"},
     {"id":"","label":"LoRA native fuse with model","localized":"","hint":"Merge LoRA into the model for lower memory usage.<br><br><b style=\"color: #ef4444\">Warning:</b> After removing or switching a LoRA, you may still see its style in generated images. To get a clean model, reload it from the model selector.","ui":"settings_lora"},
     {"id":"","label":"LoRA diffusers fuse with model","localized":"","hint":"Merge LoRA into the model for lower memory usage and torch.compile compatibility.<br><br><b style=\"color: #ef4444\">Warning:</b> After removing or switching a LoRA, you may still see its style in generated images. To get a clean model, reload it from the model selector.","ui":"settings_lora"},
-    {"id":"","label":"LoRA quantized apply method","localized":"","hint":"How networks are applied to SDNQ-quantized model weights:<br>- <b>exact</b>: adapters are carried alongside the quantized weights at full precision; apply and removal are exact and the quantized weights are never modified. The carried factors take additional VRAM, growing with adapter rank, size and count<br>- <b>requantize</b>: adapters are merged into the quantized weights, matching the behavior of earlier releases. Uses no additional VRAM (a weight backup for network removal is held in system RAM); on models quantized below 8 bits rounding typically loses much of the adapter effect, with strong adapters retaining more<br><br>With <b>requantize</b> selected, the host rank and calibration options below have no effect.<br><br>Default is <b>exact</b>.","ui":"settings_lora"},
+    {"id":"","label":"LoRA quantized apply method","localized":"","hint":"How networks are applied to SDNQ-quantized model weights:<br>- <b>exact</b>: adapters are carried alongside the quantized weights at full precision; apply and removal are exact and the quantized weights are never modified. The carried factors take additional VRAM, growing with adapter rank, size and count<br>- <b>requantize</b>: adapters are merged into the quantized weights, matching the behavior of earlier releases. Uses no additional VRAM (a weight backup for network removal is held in system RAM); on models quantized below 8 bits rounding typically loses much of the adapter effect, with strong adapters retaining more<br><br>With <b>requantize</b> selected, the host rank, calibration and cache options below have no effect.<br><br>Default is <b>exact</b>.","ui":"settings_lora"},
     {"id":"","label":"LoRA quantized host rank","localized":"","hint":"Maximum rank used to carry non-factorable adapter types (LoKR, LoHA, OFT, DoRA) on the side-channel of SDNQ models quantized below 8 bits, where merging would erase most of the adapter. Higher values keep more of the adapter at proportionally higher memory cost. Set to 0 to disable and merge into the quantized weights instead.","ui":"settings_lora"},
     {"id":"","label":"LoRA quantized host calibration","localized":"","hint":"Collection of per-channel activation statistics during normal generations on models quantized below 8 bits, cached per checkpoint. When available, the statistics weight side-channel hosting of non-factorable adapter types toward the channels carrying the most activation energy, improving delivered fidelity at the same host rank.","ui":"settings_lora"},
+    {"id":"","label":"LoRA quantized host cache","localized":"","hint":"Disk budget in GB for caching the side-channel factors computed when hosting non-factorable adapter types on quantized models. A cached configuration skips the truncation math on reapply; least recently used entries are evicted past the budget. Set to 0 to disable.","ui":"settings_lora"},
     {"id":"","label":"LoRA auto-apply tags","localized":"","hint":"Automatically add trigger words/tags from LoRA metadata to your prompt.<br>Set to the number of tags to auto-apply, e.g., 3 = add top 3 trigger tags.<br>Set to 0 to disable, -1 to add all available tags.","ui":"settings_lora"},
     {"id":"","label":"LoRA memory cache","localized":"","hint":"How many LoRAs to keep in network for future use before requiring reloading from storage","ui":"settings_lora"},
     {"id":"","label":"LoRA add hash info to metadata","localized":"","hint":"Include LoRA file hashes in generated image metadata.<br>Useful for reproducibility and tracking which exact LoRA versions were used.","ui":"settings_lora"},