diff --git a/cli/lora-quant-fidelity.py b/cli/lora-quant-fidelity.py index df2c64126..1dc41bfe5 100644 --- a/cli/lora-quant-fidelity.py +++ b/cli/lora-quant-fidelity.py @@ -315,8 +315,8 @@ def analyze_module(W_dq, deq_params, mods, calib_rms=None): Dw = D * rms if rms is not None else D with torch.random.fork_rng(devices=[D.device] if D.device.type == 'cuda' else []): torch.manual_seed(0) - U, S, V = torch.svd_lowrank(Dw, q=q, niter=4) - Dk = (U * S) @ V.t() + U, S, V = torch.svd_lowrank(Dw, q=min(q + 64, *D.shape), niter=8) + Dk = (U[:, :q] * S[:q]) @ V[:, :q].t() if rms is not None: Dk = Dk / rms base16 = W_dq.to(torch.bfloat16).float() diff --git a/modules/lora/lora_factor_cache.py b/modules/lora/lora_factor_cache.py index a5ef4041a..6e7e46a0b 100644 --- a/modules/lora/lora_factor_cache.py +++ b/modules/lora/lora_factor_cache.py @@ -96,7 +96,7 @@ def begin_pass(wanted_names): from safetensors import safe_open with safe_open(path, framework='pt', device='cpu') as f: meta = f.metadata() or {} - if meta.get('sig') == sig and meta.get('fmt') == '2': + if meta.get('sig') == sig and meta.get('fmt') == '3': for k in f.keys(): entries[k] = f.get_tensor(k) os.utime(path, None) # freshness for LRU eviction @@ -190,7 +190,7 @@ def flush(): from safetensors.torch import save_file os.makedirs(cache_root, exist_ok=True) tmp = state['path'] + '.tmp' - save_file(state['store'], tmp, metadata={'sig': state['sig'], 'fmt': '2'}) + save_file(state['store'], tmp, metadata={'sig': state['sig'], 'fmt': '3'}) os.replace(tmp, state['path']) evict() except Exception as e: diff --git a/modules/lora/lora_sdnq.py b/modules/lora/lora_sdnq.py index 8d5330e09..b2a91de6c 100644 --- a/modules/lora/lora_sdnq.py +++ b/modules/lora/lora_sdnq.py @@ -274,7 +274,9 @@ def apply_hosted(self, network_layer_name, updown, wanted_names, use_previous=Fa # svd_lowrank draws random projections; fork so user generation seeds are untouched and re-applies are deterministic with torch.random.fork_rng(devices=[D.device] if D.device.type == 'cuda' else []): torch.manual_seed(0) - U, S, V = torch.svd_lowrank(D, q=q, niter=4) + # oversampled sketch with extra power iterations lands within noise of exact svd; only the top q columns are kept + U, S, V = torch.svd_lowrank(D, q=min(q + 64, *D.shape), niter=8) + U, S, V = U[:, :q], S[:q], V[:, :q] energy = float(S.square().sum() / D.square().sum().clamp(min=1e-30)) # captured fraction, in the weighted domain when calibrated up_h = (U * S).to(dtype=dtype) down_h = V.t()