From 2fcd99a4092729cb6655f65df8b4ddc453245559 Mon Sep 17 00:00:00 2001 From: CalamitousFelicitousness Date: Fri, 17 Jul 2026 03:04:58 +0100 Subject: [PATCH] test(lora): cover sdnq config matrix in factor path suite Checkpoints quantized without hadamard must attach factors unrotated; checkpoints carrying their own svd correction must keep it under apply and get the original factors back on remove. Both pinned in both svd layouts. --- test/test-sdnq-lora-factors.py | 51 +++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/test/test-sdnq-lora-factors.py b/test/test-sdnq-lora-factors.py index 7eb0c2943..6a91380a2 100644 --- a/test/test-sdnq-lora-factors.py +++ b/test/test-sdnq-lora-factors.py @@ -10,9 +10,11 @@ for the per-model analyzer): floor on uint4, while int8 retains most of the delta. Guards against the erasure law silently changing. - The factor path (modules/lora/lora_sdnq.py) applies plain LoRA deltas - through the svd side-channel exactly, in both svd layouts, with exact - stacking, multiplier scaling and bit-exact removal, wired through the real - networks.network_activate / network_deactivate control flow. + through the svd side-channel exactly, in both svd layouts and across + quantization configs (hadamard on/off, checkpoint svd correction present + or absent), with exact stacking, multiplier scaling and bit-exact + removal, wired through the real networks.network_activate / + network_deactivate control flow. - Multi-LoRA set transitions keep the base pristine: a layer that fell back to requantize (mixed factorable/non-factorable set) restores from backup before re-entering the factor path, layers targeted by only some of the @@ -94,13 +96,13 @@ def run_test(cat: str, fn): traceback.print_exc() -def build_layer(weights_dtype='uint4', use_quantized_matmul=False, seed=0): +def build_layer(weights_dtype='uint4', use_quantized_matmul=False, seed=0, use_hadamard=True, use_svd=False): torch.manual_seed(seed) lin = torch.nn.Linear(IN_F, OUT_F, bias=False, dtype=torch.bfloat16, device=DEVICE) with torch.no_grad(): lin.weight.copy_(torch.randn(OUT_F, IN_F, device=DEVICE) * 0.04) - cfg = SDNQConfig(weights_dtype=weights_dtype, group_size=0, hadamard_group_size=256, use_hadamard=True, - use_svd=False, use_quantized_matmul=use_quantized_matmul, dequantize_fp32=False, + cfg = SDNQConfig(weights_dtype=weights_dtype, group_size=0, hadamard_group_size=256, use_hadamard=use_hadamard, + use_svd=use_svd, svd_rank=32, use_quantized_matmul=use_quantized_matmul, dequantize_fp32=False, quantization_device=str(DEVICE), return_device=str(DEVICE)) layer, _ = sdnq_quantize_layer(lin, cfg, torch_dtype=torch.bfloat16, param_name='test.weight') layer.network_layer_name = 'lora_transformer_test' @@ -268,6 +270,41 @@ def test_dora_falls_back(): return True +def assert_factor_roundtrip(layer, tag): + """Apply-exact plus bit-exact removal on the given layer, whatever its quantization config.""" + A, B, D = make_delta() + Wdq0 = dq(layer) + orig_up = layer.svd_up + l_common.loaded_networks.clear() + l_common.loaded_networks.append(make_net('one', layer, A, B)) + wanted = (('one', 1.0, 1.0, None),) + assert lora_sdnq.factor_candidate(layer, layer.network_layer_name, wanted) is True, f'{tag}: not a factor candidate' + assert lora_sdnq.apply_factors(layer, layer.network_layer_name, wanted) is True, f'{tag}: apply failed' + E = dq(layer) - Wdq0 + rho = rho_of(E, D) + resid = float((E - D).norm() / D.norm()) + assert rho > 0.99 and resid < 0.2, f'{tag}: rho={rho:.4f} resid={resid:.4f}' + assert lora_sdnq.remove_factors(layer) and torch.equal(dq(layer), Wdq0), f'{tag}: remove not bit-exact' + assert layer.svd_up is orig_up, f'{tag}: original svd factors not restored' + l_common.loaded_networks.clear() + + +def test_no_hadamard_checkpoint(): + """Checkpoints quantized without hadamard: factors attach unrotated.""" + assert_factor_roundtrip(build_layer('uint4', use_hadamard=False), 'plain') + assert_factor_roundtrip(build_layer('uint4', use_hadamard=False, use_quantized_matmul=True), 'matmul') + return True + + +def test_checkpoint_svd_factors_preserved(): + """Checkpoints quantized with their own svd correction keep it under apply/remove.""" + layer = build_layer('uint4', use_svd=True) + assert layer.svd_up is not None, 'quantizer produced no svd correction' + assert_factor_roundtrip(layer, 'plain') + assert_factor_roundtrip(build_layer('uint4', use_svd=True, use_quantized_matmul=True), 'matmul') + return True + + # ============================================================ # Tests - memory accounting across apply modes # ============================================================ @@ -504,7 +541,7 @@ def run_tests(): for fn in [test_uint4_erases_substep_delta, test_int8_retains_delta]: run_test(CAT_LAW, fn) log.warning('=== Factor path ===') - for fn in [test_apply_exact_and_remove_bitexact, test_multiplier_and_alpha_scaling, test_stacking_two_networks, test_matmul_layout_transposed, test_dora_falls_back]: + for fn in [test_apply_exact_and_remove_bitexact, test_multiplier_and_alpha_scaling, test_stacking_two_networks, test_matmul_layout_transposed, test_dora_falls_back, test_no_hadamard_checkpoint, test_checkpoint_svd_factors_preserved]: run_test(CAT_FACTOR, fn) log.warning('=== Memory accounting ===') for fn in [test_factor_path_memory_is_factors_only, test_backup_mode_clones_full_quant_state, test_fuse_mode_marker_takes_no_memory]: