Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-19 01:04:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(sdnq): tighten attention quantization verdicts in the benchmark

Browse Source 
- on/off verdicts share one speed margin (recommend_speed_margin, 10%) across
  attention, dequant, compile, te and conv rows
- re-check the recommended toggle stack against unquantized; individually passing
  buybacks can eat a marginal qk gain
- judge smooth k and hadamard cost at block scope when measured: kernel rows hand
  the prep contiguous q/k/v, real models hand it strided views from the fused qkv
  projection; reasons cite both scopes
- give bare float8 qk its own shot at the margin before disabling, the verdict
  must not hinge on int8 alone
- compare triton flash against the recommended config, not always int8
- note self-attention shapes that disagree with the reference verdict
This commit is contained in:
CalamitousFelicitousness
2026-07-16 23:18:07 +01:00
parent 61a509a7af
commit f1ae4c2c1e
1 changed files with 102 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/sdnq-attention-benchmark.py
+102 -30
View File
@@ -235,6 +235,7 @@ conv_configs = [ # id, weights config (None = bf16 baseline), use conv quantized
all_dequant_sweeps = ["groups", "svd", "hgroups", "toggles", "conv"]
all_mm_backends = ["torch", "triton"]
recommend_error_cap = 2.0 # a faster config is not recommended when it multiplies measured output error beyond this
recommend_speed_margin = 0.90 # an on/off verdict needs new_ms <= old_ms * this; sub-margin wins sit inside 12-iter run noise and never justify added error
atten_settings = [
("sdnq_attention_matmul_type", "MatMul type"),
@@ -2383,7 +2384,7 @@ def best_config(results):
return min(near_fastest, key=lambda item: item[2])[0]
def build_recommendations(all_results, fp8_result, prep_status):
def build_recommendations(all_results, fp8_result, prep_status, block_results=None):
# prefer an image dit shape with the full config set as reference, then the video shapes
reference = None
for preset in recommendation_presets:
@@ -2408,15 +2409,20 @@ def build_recommendations(all_results, fp8_result, prep_status):
return str(getattr(shared.opts, key))
rows = []
# the enable checkbox is gone: disable/enable live on the MatMul type dropdown itself,
# so one row carries both the is-it-worth-it verdict and the dtype choice; compare
# against the unquantized sdnq row: quantization has to pay for its own prep
use_quantized = bool(int8_ms and noquant_ms and int8_ms < noquant_ms * 0.95)
# verdicts are computed before any row is built: the smooth k and hadamard buybacks
# feed a composition check on the qk verdict, since toggles that pass individually
# can still lose to unquantized as a stack. compare against the unquantized sdnq row:
# quantization has to pay for its own prep and clear the shared speed margin
use_quantized = bool(int8_ms and noquant_ms and int8_ms < noquant_ms * recommend_speed_margin)
if int8_ms and noquant_ms:
if use_quantized:
quant_reason = f"int8 qk measured x{noquant_ms / int8_ms:.2f} vs unquantized sdnq attention"
if int8_err and noquant_err:
quant_reason += f", error {int8_err:.5f} vs {noquant_err:.5f}; smooth k and hadamard below buy error back"
elif int8_ms < noquant_ms:
quant_reason = f"int8 qk measured only x{noquant_ms / int8_ms:.2f} vs unquantized sdnq attention, under the x{1 / recommend_speed_margin:.2f} margin the verdict requires"
if int8_err and noquant_err:
quant_reason += f"; unquantized keeps error {noquant_err:.5f} vs {int8_err:.5f}"
else:
quant_reason = f"unquantized sdnq measured x{int8_ms / noquant_ms:.2f} vs int8 qk with lower error; quantization prep outweighs the kernel gain on this gpu"
elif int8_ms and base_ms:
@@ -2426,10 +2432,70 @@ def build_recommendations(all_results, fp8_result, prep_status):
use_quantized = False
quant_reason = "int8 qk failed to run"
block_rows = block_results or {}
def block_ms(config_id):
return (block_rows.get(config_id) or {}).get("ms")
def buyback_cost(kernel_config_id, block_config_id):
# cost of a prep toggle, judged at the most representative measured scope: the
# kernel rows hand the prep contiguous q/k/v, real models hand it strided views
# from the fused qkv projection, which the block section measures
toggle_ms, toggle_err = measured(results, kernel_config_id)
if not (toggle_ms and int8_ms and int8_err and toggle_err):
return None, None, None
gain = int8_err / toggle_err if toggle_err > 0 else 1.0
kernel_cost = toggle_ms / int8_ms - 1.0
block_base_ms, block_toggle_ms = block_ms("int8-mm-atten"), block_ms(block_config_id)
if block_base_ms and block_toggle_ms:
cost = block_toggle_ms / block_base_ms - 1.0
return gain, cost, f"{cost:+.0%} at block scope with strided qkv ({kernel_cost:+.0%} on contiguous kernel tensors)"
return gain, kernel_cost, f"{kernel_cost:+.0%} time"
smooth_gain, smooth_cost, smooth_cost_note = buyback_cost("smooth", "int8-mm-smooth")
smooth_rec = smooth_reason = None
if smooth_gain is not None:
smooth_rec = smooth_gain >= 1.3 and smooth_cost <= 0.25 # attention-level cost is a few percent end to end
smooth_reason = f"int8 error x{smooth_gain:.1f} lower for {smooth_cost_note}"
hadamard_gain, hadamard_cost, hadamard_cost_note = buyback_cost("hadamard", "int8-mm-hadamard")
hadamard_rec = hadamard_reason = None
if hadamard_gain is not None:
hadamard_rec = hadamard_gain >= 1.3 and hadamard_cost <= 0.15
hadamard_reason = f"int8 error x{hadamard_gain:.1f} lower for {hadamard_cost_note}; hangs torch compile on non pow2 head dims (SD 1.5)"
# composition check: the toggles recommended above must still beat unquantized as a stack
if use_quantized and noquant_ms:
combo_id, combo_label = {
(True, True): ("smooth_hadamard", "int8 qk + smooth + hadamard"),
(True, False): ("smooth", "int8 qk + smooth k"),
(False, True): ("hadamard", "int8 qk + hadamard"),
(False, False): ("int8", "int8 qk"),
}[(bool(smooth_rec), bool(hadamard_rec))]
combo_ms, _combo_err = measured(results, combo_id)
if combo_ms and not combo_ms < noquant_ms * recommend_speed_margin:
use_quantized = False
quant_reason = f"the recommended stack ({combo_label}) measured x{noquant_ms / combo_ms:.2f} vs unquantized sdnq attention; the error buybacks eat the qk gain on this gpu"
# the verdict must not hinge on int8 alone: bare float8 qk can clear the margin on
# gpus where int8 falls short, so it gets its own shot before disabling
fp8qk_ms, fp8qk_err = measured(results, "fp8qk")
fp8_rescue = False
if not use_quantized and noquant_ms and fp8qk_ms and fp8qk_ms < noquant_ms * recommend_speed_margin:
if not (fp8qk_err and int8_err and fp8qk_err > int8_err * recommend_error_cap):
use_quantized = True
fp8_rescue = True
quant_reason = f"float8 qk measured x{noquant_ms / fp8qk_ms:.2f} vs unquantized sdnq attention where the int8 path fell short"
if fp8qk_err:
quant_reason += f", error {fp8qk_err:.5f}"
quant_reason += "; smooth k and hadamard buybacks were only measured on int8"
qk_choice = "enabled"
qk_reason = quant_reason + "; enabled resolves to int8, uint8 remaps to int8"
fp8qk_ms, fp8qk_err = measured(results, "fp8qk")
if fp8qk_ms and int8_ms:
if fp8_rescue:
qk_choice = "float8_e4m3fn"
qk_reason = quant_reason
elif fp8qk_ms and int8_ms:
qk_compare = f"float8 qk measured x{int8_ms / fp8qk_ms:.2f} vs int8"
if fp8qk_err and int8_err:
qk_compare += f", error {fp8qk_err:.5f} vs {int8_err:.5f}"
@@ -2451,7 +2517,7 @@ def build_recommendations(all_results, fp8_result, prep_status):
fp8pv_ms, fp8pv_err = measured(results, "fp8pv")
pv_measured = [(dtype, name, ms, err) for dtype, name, ms, err in [("float8_e4m3fn", "fp8", fp8pv_ms, fp8pv_err), ("int8", "int8", int8pv_ms, int8pv_err)] if ms]
best_pv = min(pv_measured, key=lambda item: item[2]) if pv_measured else None
if use_quantized and best_pv and int8_ms and best_pv[2] < int8_ms * 0.95:
if use_quantized and best_pv and int8_ms and best_pv[2] < int8_ms * recommend_speed_margin:
pv_dtype, pv_name, pv_ms, pv_err = best_pv
if pv_err and int8_err and pv_err > int8_err * recommend_error_cap:
rows.append(("PV MatMul type", current("sdnq_attention_pv_matmul_type"), "disabled", f"{pv_name} pv is x{int8_ms / pv_ms:.2f} faster but multiplies error x{pv_err / int8_err:.1f}; disabled keeps pv unquantized"))
@@ -2469,21 +2535,10 @@ def build_recommendations(all_results, fp8_result, prep_status):
pv_note = "disabled keeps pv unquantized"
rows.append(("PV MatMul type", current("sdnq_attention_pv_matmul_type"), "disabled", pv_note))
smooth_ms, smooth_err = measured(results, "smooth")
if smooth_ms and int8_ms and int8_err and smooth_err:
cost = smooth_ms / int8_ms - 1.0
gain = int8_err / smooth_err if smooth_err > 0 else 1.0
recommend = gain >= 1.3 and cost <= 0.25 # attention-level cost is a few percent end to end
rows.append(("Use Smooth K", current("sdnq_attention_smooth_k"), str(recommend), f"int8 error x{gain:.1f} lower for {cost:+.0%} time"))
hadamard_ms, hadamard_err = measured(results, "hadamard")
if hadamard_ms and int8_ms and int8_err and hadamard_err:
cost = hadamard_ms / int8_ms - 1.0
gain = int8_err / hadamard_err if hadamard_err > 0 else 1.0
if gain >= 1.3 and cost <= 0.15:
rows.append(("Use Hadamard", current("sdnq_attention_use_hadamard"), "True", f"int8 error x{gain:.1f} lower for {cost:+.0%} time; hangs torch compile on non pow2 head dims (SD 1.5)"))
else:
rows.append(("Use Hadamard", current("sdnq_attention_use_hadamard"), "False", f"error x{gain:.1f} lower but {cost:+.0%} time; consider for long-sequence sessions; hangs torch compile on non pow2 head dims (SD 1.5)"))
if smooth_rec is not None:
rows.append(("Use Smooth K", current("sdnq_attention_smooth_k"), str(smooth_rec), smooth_reason))
if hadamard_rec is not None:
rows.append(("Use Hadamard", current("sdnq_attention_use_hadamard"), str(hadamard_rec), hadamard_reason))
rows.append(("Hadamard Group Size", current("sdnq_attention_hadamard_group_size"), "256", "values above head dim are clamped; non pow2 values floor to the nearest power of 2"))
@@ -2505,6 +2560,19 @@ def build_recommendations(all_results, fp8_result, prep_status):
emit("[dim]qk quantization is not worth it here, so MatMul type recommends disabled; the smooth k and hadamard rows show what to pick if it is enabled anyway[/dim]")
notes = []
# the verdict above is judged at the reference shape; flag self-attention dit shapes
# that disagree (cross and te shapes are prep-dominated and always lose, skip them)
qk_winners, qk_losers = [], []
for shape_label in recommendation_presets:
shape_results = all_results.get(shape_label)
if not shape_results:
continue
shape_noquant_ms, _err = measured(shape_results, "noquant")
shape_int8_ms, _err = measured(shape_results, "int8")
if shape_noquant_ms and shape_int8_ms:
(qk_winners if shape_int8_ms < shape_noquant_ms * recommend_speed_margin else qk_losers).append(shape_label)
if qk_winners and qk_losers:
notes.append(f"[yellow]int8 qk beats unquantized by >=10% at {', '.join(qk_winners)} but not at {', '.join(qk_losers)}; the verdict follows the {reference} shape[/yellow]")
labels = {config_id: config_label(config_id, label) for config_id, label, _kwargs in bench_configs}
best_id = best_config(results)
if base_ms and best_id is not None:
@@ -2537,9 +2605,13 @@ def build_recommendations(all_results, fp8_result, prep_status):
sagefp16_ms, _sagefp16_err = measured(results, "sagefp16")
if sagefp16_ms:
notes.append("sage fp16 accum: benchmark-only, no sdnext setting reaches it (sage pins fp32 accum on sm86)")
# compare flash against the config the verdict above actually recommends
amdflash_ms, _amdflash_err = measured(results, "amdflash")
if amdflash_ms and int8_ms:
notes.append(f"vs triton flash at {reference}: sdnq int8 {int8_ms:.2f} ms, flash {amdflash_ms:.2f} ms; sdnq also covers masks, its gain tracks this gpu's int8 throughput")
if amdflash_ms:
if use_quantized and int8_ms:
notes.append(f"vs triton flash at {reference}: sdnq int8 {int8_ms:.2f} ms, flash {amdflash_ms:.2f} ms; sdnq also covers masks, its gain tracks this gpu's int8 throughput")
elif noquant_ms:
notes.append(f"vs triton flash at {reference}: unquantized sdnq {noquant_ms:.2f} ms, flash {amdflash_ms:.2f} ms; sdnq also covers masks")
# cross-reference the combined block section when it ran in this invocation: kernel-scope
# error buybacks can vanish at block output, and per-call costs differ inside a real block
block_results = report.get("block", {}).get("results", {})
@@ -2602,7 +2674,7 @@ def build_dequant_recommendations(dequant_results, weight_dequant_result, varian
rows = []
int8_eager, int8_compiled = entry("int8").get("eager_ms"), entry("int8").get("compiled_ms")
if int8_eager and int8_compiled:
recommend_compile = int8_compiled <= int8_eager * 0.90
recommend_compile = int8_compiled <= int8_eager * recommend_speed_margin
compile_reason = f"int8 dequant measured {ratio_text(int8_eager, int8_compiled)} compiled vs eager"
fwd_eager, fwd_compiled = entry("int8").get("fwd_eager_ms"), entry("int8").get("fwd_compiled_ms")
if fwd_eager and fwd_compiled:
@@ -2640,7 +2712,7 @@ def build_dequant_recommendations(dequant_results, weight_dequant_result, varian
best_sel, best_resolved, best_ms, best_err = min(near_fastest, key=lambda c: c[3] if c[3] is not None else float("inf"))
fwd_err = mm_entry.get("fwd_err")
err_ok = not (fwd_err and best_err) or best_err <= fwd_err * recommend_error_cap
faster = best_ms <= mm_entry["fwd_ms"] * 0.90
faster = best_ms <= mm_entry["fwd_ms"] * recommend_speed_margin
recommend_mm = faster and err_ok
mm_reason = f"{mm_id} quantized matmul ({best_resolved}) measured {ratio_text(mm_entry['fwd_ms'], best_ms)} vs the dequant path"
if fwd_err and best_err:
@@ -2672,7 +2744,7 @@ def build_dequant_recommendations(dequant_results, weight_dequant_result, varian
if te_entry.get("fwd_ms") and te_entry.get("mm_ms"):
te_fwd_err, te_mm_err = te_entry.get("fwd_err"), te_entry.get("mm_err")
te_err_ok = not (te_fwd_err and te_mm_err) or te_mm_err <= te_fwd_err * recommend_error_cap
te_faster = te_entry["mm_ms"] <= te_entry["fwd_ms"] * 0.90
te_faster = te_entry["mm_ms"] <= te_entry["fwd_ms"] * recommend_speed_margin
te_choice = "enabled" if te_faster and te_err_ok else "disabled"
te_reason = f"{te_mm_id} quantized matmul measured {ratio_text(te_entry['fwd_ms'], te_entry['mm_ms'])} vs the dequant path at {te_shape}"
if te_fwd_err and te_mm_err:
@@ -2756,7 +2828,7 @@ def build_dequant_recommendations(dequant_results, weight_dequant_result, varian
rows.append(("Quantize convolutional layers", current("sdnq_quantize_conv_layers"), str(conv_free), conv_reason))
if conv_int8.get("fwd_ms") and conv_mm_rows:
best_mm_id, best_mm = min(conv_mm_rows, key=lambda item: item[1]["fwd_ms"])
mm_faster = best_mm["fwd_ms"] <= conv_int8["fwd_ms"] * 0.90
mm_faster = best_mm["fwd_ms"] <= conv_int8["fwd_ms"] * recommend_speed_margin
mm_err_ok = not (best_mm.get("out_err") and conv_int8.get("out_err")) or best_mm["out_err"] <= conv_int8["out_err"] * recommend_error_cap
conv_mm_reason = f"{best_mm_id} measured {best_mm['fwd_ms']:.3f} vs {conv_int8['fwd_ms']:.3f} ms for the int8 conv dequant path at {conv_shapes[0][0]}, output error {best_mm.get('out_err', 0):.5f} vs {conv_int8.get('out_err', 0):.5f}"
other_base, _other_int8, other_mm_rows = conv_per_shape.get(conv_shapes[1][0], ({}, {}, [])) if len(conv_shapes) > 1 else ({}, {}, [])
@@ -3078,7 +3150,7 @@ def main():
emit(f"[yellow]skipping {preset}: needs about {needed:.0f} gb free vram, {free_vram_gb():.1f} gb available[/yellow]")
continue
all_results[preset] = bench_shape(preset, args.iters, args.warmup, position=(index, len(selected)), config_timeout=args.config_timeout, fp8_result=fp8_result)
build_recommendations(all_results, fp8_result, prep_status)
build_recommendations(all_results, fp8_result, prep_status, block_results=(report.get("block") or {}).get("results"))
flush_outputs()