From 6c65b7d13bb1dbeccaafbae95e436f4771137055 Mon Sep 17 00:00:00 2001 From: forforever73 <690105611@qq.com> Date: Fri, 31 Jul 2026 19:37:12 +0800 Subject: [PATCH] test-backend-ops : replace the FA vec tune mode with a bounded (Q,NE) slice --- tests/test-backend-ops.cpp | 502 +++---------------------------------- 1 file changed, 35 insertions(+), 467 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index c6a27aa617..f26863a57a 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -478,7 +478,6 @@ enum test_mode { MODE_PERF, MODE_GRAD, MODE_SUPPORT, - MODE_TUNE, }; // Output format support similar to llama-bench @@ -10574,12 +10573,10 @@ static std::vector> make_test_cases_from_file(const c return test_cases; } -// ---- FA vec (Q,NE) tuning: numerical correctness check + drift guard ---- +// ---- FA vec (Q,NE): forced-config numerical slice (Metal only) ---- // metal proc_address bridges (resolved by string, not symbol linkage) using set_fa_vec_override_t = void (*)(int, int); using clear_fa_vec_override_t = void (*)(void); -using fa_vec_bucket_t = int (*)(int64_t); // runtime ne11/ne01 bucketers, shared via proc bridge -using fa_vec_baseline_ne_t = int (*)(int, int); // runtime (dk,dv) -> baseline NE // legal NE for a (dk,dv): NL = 32/NE, require (dk/4)%NL==0 && (dv/4)%NL==0 static std::vector fa_vec_legal_ne(int dk, int dv) { @@ -10593,67 +10590,45 @@ static std::vector fa_vec_legal_ne(int dk, int dv) { return r; } -// Baseline-path smoke test: with no override, fa_vec_pick returns (1, baseline_ne) and -// dispatches the no-suffix baseline kernel. Checks baseline correctness across all 10 shapes -// (NE is numerically transparent; the (Q,NE) kernels are covered by run_fa_vec_tune_check). -static bool run_fa_vec_drift_guard(ggml_backend_t backend_metal, ggml_backend_t backend_cpu) { - struct shape_t { int dk, dv; }; - const shape_t shapes[] = { {32,32},{64,64},{96,96},{128,128},{192,192}, - {192,128},{256,256},{320,256},{512,512},{576,512} }; - bool ok = true; - for (auto s : shapes) { - // no override + short KV (ne11 < FA_VEC_NE11_BUCKETS[0]) -> fa_vec_pick returns baseline; - // compare metal vs CPU-ref - test_flash_attn_ext tc(s.dk, s.dv, /*nh=*/4, {1,1}, /*kv=*/512, /*nb=*/8, - /*mask=*/true, /*sinks=*/false, 0.0f, 0.0f, GGML_PREC_F32, - GGML_TYPE_F16, GGML_TYPE_F16); - auto st = tc.eval(backend_metal, backend_cpu, "FLASH_ATTN_EXT", nullptr); - if (st == test_status_t::FAIL) { - printf("DRIFT/baseline FAIL dk=%d dv=%d\n", s.dk, s.dv); - ok = false; - } - } - return ok; -} +// Forces every legal (Q,NE) on two representative shapes and compares against the CPU +// reference. Metal-only: the override is a backend-global switch, so it cannot be expressed +// per test case in the backend-agnostic case list. Covers padded rows (ne01 % Q != 0), +// per-qq sinks, kvpad, the nsg-dependent shmem offsets / parallel-reduce stride +// (ne11 -> nsg 1/2/4) and the quantized dequant-once path. +static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu) { + auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend)); -// Numerical gate: for each (dk,dv) x legal (Q,NE), force the override and compare metal -// vs CPU-ref. The ne01/ne11/mask/sinks points exercise the rebased body's padded rows -// (ne01 not a multiple of Q), per-qq sinks, skip-INF, kvpad, and the nsg-dependent shmem -// offsets / parallel-reduce stride (ne11 -> nsg 1/2/4). -static bool run_fa_vec_tune_check(ggml_backend_t backend_metal, ggml_backend_t backend_cpu) { - auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_metal)); auto set_ov = (set_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override"); auto clear_ov = (clear_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_clear_fa_vec_override"); - if (!set_ov || !clear_ov) { printf("metal fa_vec override proc unavailable\n"); return false; } + if (!set_ov || !clear_ov) { + return true; // not the Metal backend: nothing to force + } struct shape_t { int dk, dv; }; - const shape_t shapes[] = { {32,32},{64,64},{96,96},{128,128},{192,192}, - {192,128},{256,256},{320,256},{512,512},{576,512} }; - const int ne01_pts[] = { 1, 2, 3, 8, 17 }; // 1, Q+/-1, 2Q-1, prime; vec upper bound < 20 - const int ne11_pts[] = { 512, 4096, 8192 }; // drives adaptive nsg 1/2/4 - const int ne11_kvpad[] = { 4097, 8193 }; // has_kvpad (non-32-multiple kv) - const bool mask_pts[] = { true, false }; - const bool sinks_pts[] = { false, true }; + const shape_t shapes[] = { { 128, 128 }, { 576, 512 } }; // mainstream head size + MLA shared K/V view + const int ne01_pts[] = { 1, 3 }; // decode, and padded rows for Q=2 and Q=4 + const int ne11_pts[] = { 512, 4097 }; // nsg=1, and nsg>=2 together with kvpad + const ggml_type types[] = { GGML_TYPE_F16, GGML_TYPE_Q4_0 }; - bool ok = true; int n_run = 0, n_fail = 0; for (auto s : shapes) { for (int ne : fa_vec_legal_ne(s.dk, s.dv)) { - for (int Q : {1, 2, 4}) { - for (bool mask : mask_pts) { - for (bool sinks : sinks_pts) { + for (int Q : { 1, 2, 4 }) { + for (ggml_type type_kv : types) { + for (bool sinks : { false, true }) { for (int ne01 : ne01_pts) { for (int ne11 : ne11_pts) { set_ov(Q, ne); - test_flash_attn_ext tc(s.dk, s.dv, /*nh=*/4, {1,1}, /*kv=*/ne11, /*nb=*/ne01, - mask, sinks, 0.0f, 0.0f, GGML_PREC_F32, - GGML_TYPE_F16, GGML_TYPE_F16); - auto st = tc.eval(backend_metal, backend_cpu, "FLASH_ATTN_EXT", nullptr); + test_flash_attn_ext tc(s.dk, s.dv, /*nh=*/4, { 1, 1 }, /*kv=*/ne11, /*nb=*/ne01, + /*mask=*/true, sinks, 0.0f, 0.0f, GGML_PREC_F32, + type_kv, type_kv); + auto st = tc.eval(backend, backend_cpu, "FLASH_ATTN_EXT", nullptr); clear_ov(); + if (st == test_status_t::FAIL) { - printf("FAIL dk=%d dv=%d Q=%d ne=%d ne01=%d ne11=%d mask=%d sinks=%d\n", - s.dk, s.dv, Q, ne, ne01, ne11, (int) mask, (int) sinks); - ok = false; n_fail++; + printf(" FAIL fa_vec slice: dk=%d dv=%d Q=%d ne=%d type=%s ne01=%d ne11=%d sinks=%d\n", + s.dk, s.dv, Q, ne, ggml_type_name(type_kv), ne01, ne11, (int) sinks); + n_fail++; } n_run++; } @@ -10663,393 +10638,14 @@ static bool run_fa_vec_tune_check(ggml_backend_t backend_metal, ggml_backend_t b } } } - // kvpad pass (non-32-multiple kv); dk128/256 only to keep the case count down - for (auto s : shapes) { - if (!((s.dk == 128 && s.dv == 128) || (s.dk == 256 && s.dv == 256))) { - continue; - } - for (int ne : fa_vec_legal_ne(s.dk, s.dv)) { - for (int Q : {1, 2, 4}) { - for (int ne11 : ne11_kvpad) { - set_ov(Q, ne); - test_flash_attn_ext tc(s.dk, s.dv, /*nh=*/4, {1,1}, /*kv=*/ne11, /*nb=*/8, - /*mask=*/true, /*sinks=*/false, 0.0f, 0.0f, GGML_PREC_F32, - GGML_TYPE_F16, GGML_TYPE_F16); - auto st = tc.eval(backend_metal, backend_cpu, "FLASH_ATTN_EXT", nullptr); - clear_ov(); - if (st == test_status_t::FAIL) { - printf("FAIL(kvpad) dk=%d dv=%d Q=%d ne=%d ne11=%d\n", s.dk, s.dv, Q, ne, ne11); - ok = false; n_fail++; - } - n_run++; - } - } - } - } - // quantized K/V numerical check: the rebased body's dequant path is Q-generic - // (dequant once, reuse across Q rows); confirm it stays correct for every quant precision. - const ggml_type qtypes[] = { GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0 }; - const int q_ne01[] = { 1, 2, 3, 8 }; - const int q_ne11[] = { 512, 4096, 8192 }; - for (ggml_type qt : qtypes) { - for (auto s : shapes) { - for (int ne : fa_vec_legal_ne(s.dk, s.dv)) { - for (int Q : { 1, 2, 4 }) { - for (bool sinks : { false, true }) { - for (int ne01 : q_ne01) { - for (int ne11 : q_ne11) { - set_ov(Q, ne); - test_flash_attn_ext tc(s.dk, s.dv, /*nh=*/4, {1, 1}, /*kv=*/ne11, /*nb=*/ne01, - /*mask=*/true, sinks, 0.0f, 0.0f, GGML_PREC_F32, qt, qt); - auto st = tc.eval(backend_metal, backend_cpu, "FLASH_ATTN_EXT", nullptr); - clear_ov(); - if (st == test_status_t::FAIL) { - printf("FAIL(quant) type=%s dk=%d dv=%d Q=%d ne=%d ne01=%d ne11=%d sinks=%d\n", - ggml_type_name(qt), s.dk, s.dv, Q, ne, ne01, ne11, (int) sinks); - ok = false; n_fail++; - } - n_run++; - } - } - } - } - } - } - } - printf("fa_vec tune-check: %d cases run, %d failed\n", n_run, n_fail); - return ok; -} -// A prebuilt FA op graph for one (dk,dv,ne01,ne11) cell. The op is replicated n_runs -// times so a single graph_compute amortizes dispatch/sync overhead. The graph is reused -// across (Q,NE) overrides (the override only changes the pipeline picked at encode time), -// which avoids re-allocating/re-initializing the (large) K/V tensors per candidate. -struct fa_perf_cell { - ggml_context_ptr ctx; - ggml_backend_buffer_ptr buf; - ggml_cgraph * gf = nullptr; - int n_runs = 0; - bool ok = false; -}; + printf(" fa_vec (Q,NE) slice: %d cases run, %d failed\n", n_run, n_fail); -static fa_perf_cell fa_build_perf_cell(ggml_backend_t backend, int dk, int dv, int ne01, int ne11, - ggml_type type_kv = GGML_TYPE_F16) { - fa_perf_cell cell; - - // GQA shape (nr23=[8,1]) matching real spec-decode / verify workloads: enough query - // heads to keep the GPU busy so the Q>1 K/V-reuse benefit is visible (and comparable - // to the documented #23114 numbers). nh here is the number of KV heads. - test_flash_attn_ext tc(dk, dv, /*nh=*/4, { 8, 1 }, /*kv=*/ne11, /*nb=*/ne01, - /*mask=*/true, /*sinks=*/false, 0.0f, 0.0f, GGML_PREC_F32, - type_kv, type_kv); - - const size_t graph_nodes = 1024; - ggml_init_params params = { - /* .mem_size = */ ggml_tensor_overhead() * 128 + ggml_graph_overhead_custom(graph_nodes, false), - /* .mem_base = */ NULL, - /* .no_alloc = */ true, - }; - cell.ctx.reset(ggml_init(params)); - GGML_ASSERT(cell.ctx); - - ggml_tensor * out = tc.build_graph(cell.ctx.get()); - if (!ggml_backend_supports_op(backend, out)) { - return cell; - } - - cell.buf.reset(ggml_backend_alloc_ctx_tensors(cell.ctx.get(), backend)); - if (cell.buf == NULL) { - return cell; - } - - tc.initialize_tensors(cell.ctx.get()); - - cell.gf = ggml_new_graph_custom(cell.ctx.get(), graph_nodes, false); - ggml_build_forward_expand(cell.gf, out); - - // replicate the op to amortize overhead (target ~50 GFLOP/compute, capped to bound graph size) - cell.n_runs = 1; - if (tc.op_flops(out) > 0) { - const uint64_t target_flops = 50ULL * 1000 * 1000 * 1000; - const int cap = 512; - const int by_flops = (int) std::min(cap, (int64_t) (target_flops / tc.op_flops(out))); - cell.n_runs = std::max(1, std::min(by_flops, (int) (ggml_graph_size(cell.gf) - ggml_graph_n_nodes(cell.gf)))); - } - for (int i = 1; i < cell.n_runs; ++i) { - ggml_graph_add_node(cell.gf, out); - } - cell.ok = true; - return cell; -} - -// Median per-op GPU time (us) for the currently-set override over the prebuilt cell graph. -static double time_fa_cell_median(ggml_backend_t backend, const fa_perf_cell & cell, int reps) { - if (!cell.ok) { - return -1.0; - } - ggml_backend_graph_compute(backend, cell.gf); // warmup (compiles the pipeline for the override) - ggml_backend_synchronize(backend); - - std::vector samples; - samples.reserve(reps); - for (int r = 0; r < reps; ++r) { - const int64_t t0 = ggml_time_us(); - ggml_backend_graph_compute(backend, cell.gf); - ggml_backend_synchronize(backend); - samples.push_back((double) (ggml_time_us() - t0)); - } - std::nth_element(samples.begin(), samples.begin() + samples.size() / 2, samples.end()); - return samples[samples.size() / 2] / cell.n_runs; -} - -// Perf sweep over the (Q,NE) grid, emitting pasteable fa_vec_tuned_table rows. -// nsg/nwg are left to the ops.cpp adaptive heuristic (not part of the table). -static bool run_fa_vec_tune_perf(ggml_backend_t backend_metal) { - auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_metal)); - auto set_ov = (set_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override"); - auto clr_ov = (clear_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_clear_fa_vec_override"); - // Share the runtime's bucketers + baseline NE via read-only proc bridges, so the emitted - // (ne11_b, ne01_b) keys match fa_vec_pick and can't silently desync from FA_VEC_*_BUCKETS. - auto ne11_bucket = (fa_vec_bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne11_bucket"); - auto ne01_bucket = (fa_vec_bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne01_bucket"); - auto baseline_ne = (fa_vec_baseline_ne_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_baseline_ne"); - if (!set_ov || !clr_ov || !ne11_bucket || !ne01_bucket || !baseline_ne) { - printf("metal fa_vec tuning procs unavailable\n"); - return false; - } - - struct shape_t { int dk, dv; }; - const shape_t shapes[] = { { 32, 32 }, { 64, 64 }, { 96, 96 }, { 128, 128 }, { 192, 192 }, - { 192, 128 }, { 256, 256 }, { 320, 256 }, { 512, 512 }, { 576, 512 } }; - const int ne11_rep[] = { 512, 2048, 8192, 32768 }; // ne11 bucket representatives - const int ne01_rep[] = { 1, 2, 3, 4, 5, 6, 7, 8, 16 }; // PR grid: point-bucket reps (1-4) + tail mod-4 cycle (5-8) + large anchor (16); vec serves ne01<20 - const int REPS = 7; // odd -> exact median - - printf("# fa_vec perf sweep — replace GGML_METAL_DEVICE_M4_MAX with this machine's device\n"); - printf("# [1] per-bucket timings (us, winner *): '=> cfg Nx' beats baseline, else '=> baseline'\n"); - printf("# [2] a pasteable fa_vec_tuned_table block (domain defaults + exceptions) is printed after [1]\n"); - - struct cand_t { int Q, NE; double t; }; // one timed (Q,NE) candidate - struct pt_t { int dk, dv, ne11, ne01; // one swept grid point with its candidate times - std::vector cs; double base_t; }; - - // group_split compression knobs (see ggml-metal-tuning.h for the row / lookup semantics) - const double TUNE_TAU = 0.05; // max POINTWISE regret (worst point in a bucket) to ride a domain - // default instead of emitting the bucket's own exception row - const double TUNE_THETA = 1.05; // min AGGREGATE bucket speedup vs baseline to tune the bucket at all - - auto type_token = [](ggml_type t) -> const char * { - switch (t) { - case GGML_TYPE_Q4_0: return "GGML_TYPE_Q4_0"; - case GGML_TYPE_Q4_1: return "GGML_TYPE_Q4_1"; - case GGML_TYPE_Q5_0: return "GGML_TYPE_Q5_0"; - case GGML_TYPE_Q5_1: return "GGML_TYPE_Q5_1"; - case GGML_TYPE_Q8_0: return "GGML_TYPE_Q8_0"; - default: return "GGML_TYPE_F16"; - } - }; - - const ggml_type types[] = { GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, - GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0 }; - for (ggml_type type_kv : types) { - printf("\n### dtype=%s\n", ggml_type_name(type_kv)); - std::vector pts; // one per swept (shape, ne11, ne01); bucketed + compressed below - - for (auto s : shapes) { - const int base_ne = baseline_ne(s.dk, s.dv); - const std::vector legal = fa_vec_legal_ne(s.dk, s.dv); - for (int ne11 : ne11_rep) { - for (int ne01 : ne01_rep) { - fa_perf_cell cell = fa_build_perf_cell(backend_metal, s.dk, s.dv, ne01, ne11, type_kv); - if (!cell.ok) { - continue; - } - - std::vector cs; - for (int ne : legal) { - for (int Q : { 1, 2, 4 }) { - cs.push_back({ Q, ne, 0.0 }); - } - } - // randomize config order to decorrelate thermal throttling across the cell - std::shuffle(cs.begin(), cs.end(), std::mt19937(1234)); - - double anchor = 0.0; // periodic baseline re-measure -> throttling detection - for (size_t i = 0; i < cs.size(); ++i) { - set_ov(cs[i].Q, cs[i].NE); - cs[i].t = time_fa_cell_median(backend_metal, cell, REPS); - clr_ov(); - - if (i % 4 == 0) { - const double a = time_fa_cell_median(backend_metal, cell, REPS); - if (anchor > 0.0) { - double drift = (a - anchor) / anchor; - if (drift < 0) { - drift = -drift; - } - if (drift > 0.10) { - printf("# WARN throttling? anchor drift %.1f%% dk=%d ne11=%d\n", 100.0 * drift, s.dk, ne11); - } - } - anchor = (anchor > 0.0) ? std::min(anchor, a) : a; - } - } - - std::sort(cs.begin(), cs.end(), [](const cand_t & a, const cand_t & b) { - return a.Q != b.Q ? a.Q < b.Q : a.NE < b.NE; - }); - cand_t best = cs[0]; - for (const auto & c : cs) { - if (c.t > 0.0 && (best.t <= 0.0 || c.t < best.t)) { - best = c; - } - } - double base_t = 0.0; // the swept (Q=1, base_ne) candidate == baseline kernel - for (const auto & c : cs) { - if (c.Q == 1 && c.NE == base_ne) { base_t = c.t; break; } - } - const bool keep = best.t > 0.0 && base_t > 0.0 && best.t < base_t * 0.98; - - printf("# dtype=%s dk=%d dv=%d ne11=%d ne01=%d:", ggml_type_name(type_kv), s.dk, s.dv, ne11, ne01); - for (const auto & c : cs) { - printf(" Q%dNE%d=%.1f%s", c.Q, c.NE, c.t, (c.Q == best.Q && c.NE == best.NE) ? "*" : ""); - } - if (keep) { - printf(" => Q%d,NE%d %.2fx\n", best.Q, best.NE, base_t / best.t); - } else { - printf(" => baseline\n"); - } - pts.push_back({ s.dk, s.dv, ne11, ne01, cs, base_t }); - } - } - } - - // [2] Compress into pasteable rows. Per (dk,dv) and ne01 domain {decode==1, batch>=2}, emit - // one ne11-collapsed default cfg (fewest rows, ne11_b=-1) plus a per-bucket exception wherever - // the default's pointwise regret vs the bucket's min-max-regret target, or its aggregate - // slowdown vs baseline, exceeds TUNE_TAU. The target still carries an intrinsic per-point regret - // from lumping ne01 into one tail bucket — TUNE_TAU bounds resolved-vs-target, not vs-oracle. - std::vector rows_out; - char rbuf[192]; - for (auto s : shapes) { - const int base_ne = baseline_ne(s.dk, s.dv); - std::vector cfgs; // candidate list (identical for every grid point of this shape) - int base_i = 0; - for (int ne : fa_vec_legal_ne(s.dk, s.dv)) { - for (int Q : { 1, 2, 4 }) { - if (Q == 1 && ne == base_ne) { base_i = (int) cfgs.size(); } - cfgs.push_back({ Q, ne, 0.0 }); - } - } - auto cfg_time = [&](const pt_t & p, int i) { - for (const auto & c : p.cs) { if (c.Q == cfgs[i].Q && c.NE == cfgs[i].NE) { return c.t; } } - return 0.0; - }; - - // Bucket the grid points using the runtime's bucketers (via proc), so keys match fa_vec_pick. - // Short-KV points (ne11 bucket 0) are dropped — the runtime serves those from baseline. Each - // kept bucket picks a min-max-regret target (gated by the aggregate TUNE_THETA benefit gate). - struct bkt_t { int b11, b01, Ti; std::vector agg; double base_agg; - std::vector bp; }; // bp kept for the pointwise ride test below - std::set> seen; - for (const auto & p : pts) { - if (p.dk != s.dk || p.dv != s.dv) { continue; } - const int b11 = ne11_bucket(p.ne11); - if (b11 == 0) { continue; } - seen.insert({ b11, ne01_bucket(p.ne01) }); - } - std::vector bks; - for (const auto & bb : seen) { - const int b11 = bb.first, b01 = bb.second; - std::vector bp; - for (const auto & p : pts) { - if (p.dk == s.dk && p.dv == s.dv && ne11_bucket(p.ne11) == b11 && ne01_bucket(p.ne01) == b01) { - bp.push_back(&p); - } - } - std::vector agg(cfgs.size(), 0.0), worst(cfgs.size(), 0.0); - for (const auto * p : bp) { - double bestt = 0.0; - for (size_t i = 0; i < cfgs.size(); ++i) { - double t = cfg_time(*p, (int) i); - if (t > 0.0 && (bestt == 0.0 || t < bestt)) { bestt = t; } - } - for (size_t i = 0; i < cfgs.size(); ++i) { - double t = cfg_time(*p, (int) i); - agg[i] += t; - if (t > 0.0 && bestt > 0.0) { worst[i] = std::max(worst[i], t / bestt); } - } - } - int robust = 0; - for (size_t i = 1; i < cfgs.size(); ++i) { - if (worst[i] < worst[robust] || - (worst[i] == worst[robust] && (cfgs[i].Q < cfgs[robust].Q || - (cfgs[i].Q == cfgs[robust].Q && cfgs[i].NE < cfgs[robust].NE)))) { - robust = (int) i; - } - } - const bool tune = robust != base_i && agg[base_i] / agg[robust] >= TUNE_THETA; - bks.push_back({ b11, b01, tune ? robust : base_i, agg, agg[base_i], bp }); - } - - // Pointwise regret of default cfg d vs the bucket target Ti. Per-point (not aggregate): - // a ratio-of-sums lets a default that wins on aligned ne01 (8, 16) hide a large penalty - // on a misaligned point (ne01=5), so the ride/exception decision must be per point. - auto reg_pointwise = [&](const bkt_t * b, int d) { - double r = 0.0; - for (const auto * p : b->bp) { - const double td = cfg_time(*p, d), tT = cfg_time(*p, b->Ti); - if (td > 0.0 && tT > 0.0) { r = std::max(r, td / tT - 1.0); } - } - return r; - }; - - for (int dom = 0; dom <= 1; ++dom) { // 0 = decode (ne01==1), 1 = batch (ne01>=2) - std::vector db; - for (const auto & b : bks) { if ((dom == 0) == (b.b01 == 0)) { db.push_back(&b); } } - if (db.empty()) { continue; } - - // default cfg = the one minimizing (#rows, total achieved time, Q, NE) - int bestD = -1, bestRows = 1 << 30; double bestTot = 0.0; - for (size_t d = 0; d < cfgs.size(); ++d) { - int rows = ((int) d != base_i) ? 1 : 0; double tot = 0.0; - for (const auto * b : db) { - const double reg = reg_pointwise(b, (int) d); // pointwise vs bucket target - const double slow = b->agg[d] / b->base_agg - 1.0; // aggregate vs baseline (safety net) - if (reg > TUNE_TAU || slow > TUNE_TAU) { rows++; tot += b->agg[b->Ti]; } - else { tot += b->agg[d]; } - } - const bool better = bestD < 0 || rows < bestRows || - (rows == bestRows && (tot < bestTot || - (tot == bestTot && (cfgs[d].Q < cfgs[bestD].Q || - (cfgs[d].Q == cfgs[bestD].Q && cfgs[d].NE < cfgs[bestD].NE))))); - if (better) { bestD = (int) d; bestRows = rows; bestTot = tot; } - } - - const int dom_id = (dom == 0) ? 0 : 1; // FA_VEC_DOMAIN_DECODE / FA_VEC_DOMAIN_BATCH - if (bestD != base_i) { - snprintf(rbuf, sizeof(rbuf), " { { GGML_METAL_DEVICE_M4_MAX, %s, %d, %d, -1, %d }, { %d, %d } },", - type_token(type_kv), s.dk, s.dv, dom_id, cfgs[bestD].Q, cfgs[bestD].NE); - rows_out.emplace_back(rbuf); - } - for (const auto * b : db) { - const double reg = reg_pointwise(b, bestD); // pointwise vs bucket target - const double slow = b->agg[bestD] / b->base_agg - 1.0; // aggregate vs baseline - if (reg <= TUNE_TAU && slow <= TUNE_TAU) { continue; } // rides the default / baseline - snprintf(rbuf, sizeof(rbuf), " { { GGML_METAL_DEVICE_M4_MAX, %s, %d, %d, %d, %d }, { %d, %d } },", - type_token(type_kv), s.dk, s.dv, b->b11, b->b01, cfgs[b->Ti].Q, cfgs[b->Ti].NE); - rows_out.emplace_back(rbuf); - } - } - } - printf("\n // ---- %s: %zu rows ----\n", ggml_type_name(type_kv), rows_out.size()); - for (const auto & r : rows_out) { printf("%s\n", r.c_str()); } - } // for type_kv - return true; + return n_fail == 0; } static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mode mode, const char * op_names_filter, const char * params_filter, - printer * output_printer, const char * test_file_path, int parallel_workers, bool tune_perf = false) { + printer * output_printer, const char * test_file_path, int parallel_workers) { auto filter_test_cases = [](std::vector> & test_cases, const char * params_filter) { if (params_filter == nullptr) { return; @@ -11079,9 +10675,6 @@ static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mo case MODE_PERF: test_cases = make_test_cases_perf(); break; - case MODE_TUNE: - // MODE_TUNE routes to its own dispatch below; no generic test_cases needed - break; } } else { test_cases = make_test_cases_from_file(test_file_path); @@ -11188,7 +10781,11 @@ static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mo output_printer->print_summary(test_summary_info(n_ok, tests_run, false)); output_printer->print_failed_tests(failed_tests); - return n_ok == tests_run; + // Metal-only: force every legal (Q,NE) on a bounded slice of shapes. Reuses the + // reference CPU backend above; a no-op on backends without the override proc. + const bool slice_ok = run_fa_vec_slice(backend, backend_cpu.get()); + + return n_ok == tests_run && slice_ok; } if (mode == MODE_GRAD) { @@ -11217,29 +10814,6 @@ static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mo return true; } - if (mode == MODE_TUNE) { - // self-create a CPU backend with the reference implementation as golden. - // (backend_cpu in the MODE_TEST block above is out of scope here.) - ggml_backend_t backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, NULL); - GGML_ASSERT(backend_cpu != NULL); - { - using ggml_backend_cpu_set_use_ref_t = void (*)(ggml_backend_t, bool); - auto * cpu_reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); - auto * set_use_ref = (ggml_backend_cpu_set_use_ref_t) ggml_backend_reg_get_proc_address(cpu_reg, "ggml_backend_cpu_set_use_ref"); - if (set_use_ref) { - set_use_ref(backend_cpu, true); - } - } - - // backend here is the MODE_TUNE-selected metal backend (-b MTL0) - const bool ok = tune_perf - ? run_fa_vec_tune_perf(backend) - : (run_fa_vec_drift_guard(backend, backend_cpu) && run_fa_vec_tune_check(backend, backend_cpu)); - - ggml_backend_free(backend_cpu); - return ok; - } - if (mode == MODE_SUPPORT) { // Filter out fusion cases test_cases.erase( @@ -11365,7 +10939,6 @@ static void usage(char ** argv) { printf(" - grad (compare gradients from backpropagation with method of finite differences)\n"); printf(" - perf (performance evaluation)\n"); printf(" - support (probe backend operation support)\n"); - printf(" - tune (FA vec (Q,NE) numerical correctness check vs CPU reference; --tune-perf for the perf sweep)\n"); printf(" op names for -o are as given by ggml_op_desc() (e.g. ADD, MUL_MAT, etc),\n"); printf(" optionally including the full test case string (e.g. \"ADD(type=f16,ne=[1,1,8,1],nr=[1,1,1,1],nf=1)\")\n"); printf(" --output specifies output format (default: console, options: console, sql, csv)\n"); @@ -11383,7 +10956,6 @@ int main(int argc, char ** argv) { const char * params_filter = nullptr; const char * test_file_path = nullptr; int parallel_workers = 1; - bool tune_perf = false; for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "test") == 0) { @@ -11394,10 +10966,6 @@ int main(int argc, char ** argv) { mode = MODE_GRAD; } else if (strcmp(argv[i], "support") == 0) { mode = MODE_SUPPORT; - } else if (strcmp(argv[i], "tune") == 0) { - mode = MODE_TUNE; - } else if (strcmp(argv[i], "--tune-perf") == 0) { - tune_perf = true; } else if (strcmp(argv[i], "-o") == 0) { if (i + 1 < argc) { op_names_filter = argv[++i]; @@ -11505,7 +11073,7 @@ int main(int argc, char ** argv) { false, "", ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024, true)); - bool ok = test_backend(backend.get(), dev, mode, op_names_filter, params_filter, output_printer.get(), test_file_path, parallel_workers, tune_perf); + bool ok = test_backend(backend.get(), dev, mode, op_names_filter, params_filter, output_printer.get(), test_file_path, parallel_workers); if (ok) { n_ok++;