Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-18 16:55:05 +02:00
Code Issues Packages Projects Releases Wiki Activity

test-backend-ops : replace the FA vec tune mode with a bounded (Q,NE) slice

Browse Source
This commit is contained in:
forforever73
2026-07-31 19:37:12 +08:00
committed by Georgi Gerganov
parent 48c64c6abf
commit 6c65b7d13b
1 changed files with 35 additions and 467 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-backend-ops.cpp
+35 -467
View File
@@ -478,7 +478,6 @@ enum test_mode {
MODE_PERF,
MODE_GRAD,
MODE_SUPPORT,
MODE_TUNE,
};
// Output format support similar to llama-bench
@@ -10574,12 +10573,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_from_file(const c
return test_cases;
}
// ---- FA vec (Q,NE) tuning: numerical correctness check + drift guard ----
// ---- FA vec (Q,NE): forced-config numerical slice (Metal only) ----
// metal proc_address bridges (resolved by string, not symbol linkage)
using set_fa_vec_override_t = void (*)(int, int);
using clear_fa_vec_override_t = void (*)(void);
using fa_vec_bucket_t = int (*)(int64_t); // runtime ne11/ne01 bucketers, shared via proc bridge
using fa_vec_baseline_ne_t = int (*)(int, int); // runtime (dk,dv) -> baseline NE
// legal NE for a (dk,dv): NL = 32/NE, require (dk/4)%NL==0 && (dv/4)%NL==0
static std::vector<int> fa_vec_legal_ne(int dk, int dv) {
@@ -10593,67 +10590,45 @@ static std::vector<int> fa_vec_legal_ne(int dk, int dv) {
return r;
}
// Baseline-path smoke test: with no override, fa_vec_pick returns (1, baseline_ne) and
// dispatches the no-suffix baseline kernel. Checks baseline correctness across all 10 shapes
// (NE is numerically transparent; the (Q,NE) kernels are covered by run_fa_vec_tune_check).
static bool run_fa_vec_drift_guard(ggml_backend_t backend_metal, ggml_backend_t backend_cpu) {
struct shape_t { int dk, dv; };
const shape_t shapes[] = { {32,32},{64,64},{96,96},{128,128},{192,192},
{192,128},{256,256},{320,256},{512,512},{576,512} };
bool ok = true;
for (auto s : shapes) {
// no override + short KV (ne11 < FA_VEC_NE11_BUCKETS[0]) -> fa_vec_pick returns baseline;
// compare metal vs CPU-ref
test_flash_attn_ext tc(s.dk, s.dv, /*nh=*/4, {1,1}, /*kv=*/512, /*nb=*/8,
/*mask=*/true, /*sinks=*/false, 0.0f, 0.0f, GGML_PREC_F32,
GGML_TYPE_F16, GGML_TYPE_F16);
auto st = tc.eval(backend_metal, backend_cpu, "FLASH_ATTN_EXT", nullptr);
if (st == test_status_t::FAIL) {
printf("DRIFT/baseline FAIL dk=%d dv=%d\n", s.dk, s.dv);
ok = false;
}
}
return ok;
}
// Forces every legal (Q,NE) on two representative shapes and compares against the CPU
// reference. Metal-only: the override is a backend-global switch, so it cannot be expressed
// per test case in the backend-agnostic case list. Covers padded rows (ne01 % Q != 0),
// per-qq sinks, kvpad, the nsg-dependent shmem offsets / parallel-reduce stride
// (ne11 -> nsg 1/2/4) and the quantized dequant-once path.
static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu) {
auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend));
// Numerical gate: for each (dk,dv) x legal (Q,NE), force the override and compare metal
// vs CPU-ref. The ne01/ne11/mask/sinks points exercise the rebased body's padded rows
// (ne01 not a multiple of Q), per-qq sinks, skip-INF, kvpad, and the nsg-dependent shmem
// offsets / parallel-reduce stride (ne11 -> nsg 1/2/4).
static bool run_fa_vec_tune_check(ggml_backend_t backend_metal, ggml_backend_t backend_cpu) {
auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_metal));
auto set_ov = (set_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override");
auto clear_ov = (clear_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_clear_fa_vec_override");
if (!set_ov || !clear_ov) { printf("metal fa_vec override proc unavailable\n"); return false; }
if (!set_ov || !clear_ov) {
return true; // not the Metal backend: nothing to force
}
struct shape_t { int dk, dv; };
const shape_t shapes[] = { {32,32},{64,64},{96,96},{128,128},{192,192},
{192,128},{256,256},{320,256},{512,512},{576,512} };
const int ne01_pts[] = { 1, 2, 3, 8, 17 }; // 1, Q+/-1, 2Q-1, prime; vec upper bound < 20
const int ne11_pts[] = { 512, 4096, 8192 }; // drives adaptive nsg 1/2/4
const int ne11_kvpad[] = { 4097, 8193 }; // has_kvpad (non-32-multiple kv)
const bool mask_pts[] = { true, false };
const bool sinks_pts[] = { false, true };
const shape_t shapes[] = { { 128, 128 }, { 576, 512 } }; // mainstream head size + MLA shared K/V view
const int ne01_pts[] = { 1, 3 }; // decode, and padded rows for Q=2 and Q=4
const int ne11_pts[] = { 512, 4097 }; // nsg=1, and nsg>=2 together with kvpad
const ggml_type types[] = { GGML_TYPE_F16, GGML_TYPE_Q4_0 };
bool ok = true;
int n_run = 0, n_fail = 0;
for (auto s : shapes) {
for (int ne : fa_vec_legal_ne(s.dk, s.dv)) {
for (int Q : {1, 2, 4}) {
for (bool mask : mask_pts) {
for (bool sinks : sinks_pts) {
for (int Q : { 1, 2, 4 }) {
for (ggml_type type_kv : types) {
for (bool sinks : { false, true }) {
for (int ne01 : ne01_pts) {
for (int ne11 : ne11_pts) {
set_ov(Q, ne);
test_flash_attn_ext tc(s.dk, s.dv, /*nh=*/4, {1,1}, /*kv=*/ne11, /*nb=*/ne01,
mask, sinks, 0.0f, 0.0f, GGML_PREC_F32,
GGML_TYPE_F16, GGML_TYPE_F16);
auto st = tc.eval(backend_metal, backend_cpu, "FLASH_ATTN_EXT", nullptr);
test_flash_attn_ext tc(s.dk, s.dv, /*nh=*/4, { 1, 1 }, /*kv=*/ne11, /*nb=*/ne01,
/*mask=*/true, sinks, 0.0f, 0.0f, GGML_PREC_F32,
type_kv, type_kv);
auto st = tc.eval(backend, backend_cpu, "FLASH_ATTN_EXT", nullptr);
clear_ov();
if (st == test_status_t::FAIL) {
printf("FAIL dk=%d dv=%d Q=%d ne=%d ne01=%d ne11=%d mask=%d sinks=%d\n",
s.dk, s.dv, Q, ne, ne01, ne11, (int) mask, (int) sinks);
ok = false; n_fail++;
printf(" FAIL fa_vec slice: dk=%d dv=%d Q=%d ne=%d type=%s ne01=%d ne11=%d sinks=%d\n",
s.dk, s.dv, Q, ne, ggml_type_name(type_kv), ne01, ne11, (int) sinks);
n_fail++;
}
n_run++;
}
@@ -10663,393 +10638,14 @@ static bool run_fa_vec_tune_check(ggml_backend_t backend_metal, ggml_backend_t b
}
}
}
// kvpad pass (non-32-multiple kv); dk128/256 only to keep the case count down
for (auto s : shapes) {
if (!((s.dk == 128 && s.dv == 128) || (s.dk == 256 && s.dv == 256))) {
continue;
}
for (int ne : fa_vec_legal_ne(s.dk, s.dv)) {
for (int Q : {1, 2, 4}) {
for (int ne11 : ne11_kvpad) {
set_ov(Q, ne);
test_flash_attn_ext tc(s.dk, s.dv, /*nh=*/4, {1,1}, /*kv=*/ne11, /*nb=*/8,
/*mask=*/true, /*sinks=*/false, 0.0f, 0.0f, GGML_PREC_F32,
GGML_TYPE_F16, GGML_TYPE_F16);
auto st = tc.eval(backend_metal, backend_cpu, "FLASH_ATTN_EXT", nullptr);
clear_ov();
if (st == test_status_t::FAIL) {
printf("FAIL(kvpad) dk=%d dv=%d Q=%d ne=%d ne11=%d\n", s.dk, s.dv, Q, ne, ne11);
ok = false; n_fail++;
}
n_run++;
}
}
}
}
// quantized K/V numerical check: the rebased body's dequant path is Q-generic
// (dequant once, reuse across Q rows); confirm it stays correct for every quant precision.
const ggml_type qtypes[] = { GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0 };
const int q_ne01[] = { 1, 2, 3, 8 };
const int q_ne11[] = { 512, 4096, 8192 };
for (ggml_type qt : qtypes) {
for (auto s : shapes) {
for (int ne : fa_vec_legal_ne(s.dk, s.dv)) {
for (int Q : { 1, 2, 4 }) {
for (bool sinks : { false, true }) {
for (int ne01 : q_ne01) {
for (int ne11 : q_ne11) {
set_ov(Q, ne);
test_flash_attn_ext tc(s.dk, s.dv, /*nh=*/4, {1, 1}, /*kv=*/ne11, /*nb=*/ne01,
/*mask=*/true, sinks, 0.0f, 0.0f, GGML_PREC_F32, qt, qt);
auto st = tc.eval(backend_metal, backend_cpu, "FLASH_ATTN_EXT", nullptr);
clear_ov();
if (st == test_status_t::FAIL) {
printf("FAIL(quant) type=%s dk=%d dv=%d Q=%d ne=%d ne01=%d ne11=%d sinks=%d\n",
ggml_type_name(qt), s.dk, s.dv, Q, ne, ne01, ne11, (int) sinks);
ok = false; n_fail++;
}
n_run++;
}
}
}
}
}
}
}
printf("fa_vec tune-check: %d cases run, %d failed\n", n_run, n_fail);
return ok;
}
// A prebuilt FA op graph for one (dk,dv,ne01,ne11) cell. The op is replicated n_runs
// times so a single graph_compute amortizes dispatch/sync overhead. The graph is reused
// across (Q,NE) overrides (the override only changes the pipeline picked at encode time),
// which avoids re-allocating/re-initializing the (large) K/V tensors per candidate.
struct fa_perf_cell {
ggml_context_ptr ctx;
ggml_backend_buffer_ptr buf;
ggml_cgraph * gf = nullptr;
int n_runs = 0;
bool ok = false;
};
printf(" fa_vec (Q,NE) slice: %d cases run, %d failed\n", n_run, n_fail);
static fa_perf_cell fa_build_perf_cell(ggml_backend_t backend, int dk, int dv, int ne01, int ne11,
ggml_type type_kv = GGML_TYPE_F16) {
fa_perf_cell cell;
// GQA shape (nr23=[8,1]) matching real spec-decode / verify workloads: enough query
// heads to keep the GPU busy so the Q>1 K/V-reuse benefit is visible (and comparable
// to the documented #23114 numbers). nh here is the number of KV heads.
test_flash_attn_ext tc(dk, dv, /*nh=*/4, { 8, 1 }, /*kv=*/ne11, /*nb=*/ne01,
/*mask=*/true, /*sinks=*/false, 0.0f, 0.0f, GGML_PREC_F32,
type_kv, type_kv);
const size_t graph_nodes = 1024;
ggml_init_params params = {
/* .mem_size = */ ggml_tensor_overhead() * 128 + ggml_graph_overhead_custom(graph_nodes, false),
/* .mem_base = */ NULL,
/* .no_alloc = */ true,
};
cell.ctx.reset(ggml_init(params));
GGML_ASSERT(cell.ctx);
ggml_tensor * out = tc.build_graph(cell.ctx.get());
if (!ggml_backend_supports_op(backend, out)) {
return cell;
}
cell.buf.reset(ggml_backend_alloc_ctx_tensors(cell.ctx.get(), backend));
if (cell.buf == NULL) {
return cell;
}
tc.initialize_tensors(cell.ctx.get());
cell.gf = ggml_new_graph_custom(cell.ctx.get(), graph_nodes, false);
ggml_build_forward_expand(cell.gf, out);
// replicate the op to amortize overhead (target ~50 GFLOP/compute, capped to bound graph size)
cell.n_runs = 1;
if (tc.op_flops(out) > 0) {
const uint64_t target_flops = 50ULL * 1000 * 1000 * 1000;
const int cap = 512;
const int by_flops = (int) std::min<int64_t>(cap, (int64_t) (target_flops / tc.op_flops(out)));
cell.n_runs = std::max(1, std::min<int>(by_flops, (int) (ggml_graph_size(cell.gf) - ggml_graph_n_nodes(cell.gf))));
}
for (int i = 1; i < cell.n_runs; ++i) {
ggml_graph_add_node(cell.gf, out);
}
cell.ok = true;
return cell;
}
// Median per-op GPU time (us) for the currently-set override over the prebuilt cell graph.
static double time_fa_cell_median(ggml_backend_t backend, const fa_perf_cell & cell, int reps) {
if (!cell.ok) {
return -1.0;
}
ggml_backend_graph_compute(backend, cell.gf); // warmup (compiles the pipeline for the override)
ggml_backend_synchronize(backend);
std::vector<double> samples;
samples.reserve(reps);
for (int r = 0; r < reps; ++r) {
const int64_t t0 = ggml_time_us();
ggml_backend_graph_compute(backend, cell.gf);
ggml_backend_synchronize(backend);
samples.push_back((double) (ggml_time_us() - t0));
}
std::nth_element(samples.begin(), samples.begin() + samples.size() / 2, samples.end());
return samples[samples.size() / 2] / cell.n_runs;
}
// Perf sweep over the (Q,NE) grid, emitting pasteable fa_vec_tuned_table rows.
// nsg/nwg are left to the ops.cpp adaptive heuristic (not part of the table).
static bool run_fa_vec_tune_perf(ggml_backend_t backend_metal) {
auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_metal));
auto set_ov = (set_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override");
auto clr_ov = (clear_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_clear_fa_vec_override");
// Share the runtime's bucketers + baseline NE via read-only proc bridges, so the emitted
// (ne11_b, ne01_b) keys match fa_vec_pick and can't silently desync from FA_VEC_*_BUCKETS.
auto ne11_bucket = (fa_vec_bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne11_bucket");
auto ne01_bucket = (fa_vec_bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne01_bucket");
auto baseline_ne = (fa_vec_baseline_ne_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_baseline_ne");
if (!set_ov || !clr_ov || !ne11_bucket || !ne01_bucket || !baseline_ne) {
printf("metal fa_vec tuning procs unavailable\n");
return false;
}
struct shape_t { int dk, dv; };
const shape_t shapes[] = { { 32, 32 }, { 64, 64 }, { 96, 96 }, { 128, 128 }, { 192, 192 },
{ 192, 128 }, { 256, 256 }, { 320, 256 }, { 512, 512 }, { 576, 512 } };
const int ne11_rep[] = { 512, 2048, 8192, 32768 }; // ne11 bucket representatives
const int ne01_rep[] = { 1, 2, 3, 4, 5, 6, 7, 8, 16 }; // PR grid: point-bucket reps (1-4) + tail mod-4 cycle (5-8) + large anchor (16); vec serves ne01<20
const int REPS = 7; // odd -> exact median
printf("# fa_vec perf sweep — replace GGML_METAL_DEVICE_M4_MAX with this machine's device\n");
printf("# [1] per-bucket timings (us, winner *): '=> cfg Nx' beats baseline, else '=> baseline'\n");
printf("# [2] a pasteable fa_vec_tuned_table block (domain defaults + exceptions) is printed after [1]\n");
struct cand_t { int Q, NE; double t; }; // one timed (Q,NE) candidate
struct pt_t { int dk, dv, ne11, ne01; // one swept grid point with its candidate times
std::vector<cand_t> cs; double base_t; };
// group_split compression knobs (see ggml-metal-tuning.h for the row / lookup semantics)
const double TUNE_TAU = 0.05; // max POINTWISE regret (worst point in a bucket) to ride a domain
// default instead of emitting the bucket's own exception row
const double TUNE_THETA = 1.05; // min AGGREGATE bucket speedup vs baseline to tune the bucket at all
auto type_token = [](ggml_type t) -> const char * {
switch (t) {
case GGML_TYPE_Q4_0: return "GGML_TYPE_Q4_0";
case GGML_TYPE_Q4_1: return "GGML_TYPE_Q4_1";
case GGML_TYPE_Q5_0: return "GGML_TYPE_Q5_0";
case GGML_TYPE_Q5_1: return "GGML_TYPE_Q5_1";
case GGML_TYPE_Q8_0: return "GGML_TYPE_Q8_0";
default: return "GGML_TYPE_F16";
}
};
const ggml_type types[] = { GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1,
GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0 };
for (ggml_type type_kv : types) {
printf("\n### dtype=%s\n", ggml_type_name(type_kv));
std::vector<pt_t> pts; // one per swept (shape, ne11, ne01); bucketed + compressed below
for (auto s : shapes) {
const int base_ne = baseline_ne(s.dk, s.dv);
const std::vector<int> legal = fa_vec_legal_ne(s.dk, s.dv);
for (int ne11 : ne11_rep) {
for (int ne01 : ne01_rep) {
fa_perf_cell cell = fa_build_perf_cell(backend_metal, s.dk, s.dv, ne01, ne11, type_kv);
if (!cell.ok) {
continue;
}
std::vector<cand_t> cs;
for (int ne : legal) {
for (int Q : { 1, 2, 4 }) {
cs.push_back({ Q, ne, 0.0 });
}
}
// randomize config order to decorrelate thermal throttling across the cell
std::shuffle(cs.begin(), cs.end(), std::mt19937(1234));
double anchor = 0.0; // periodic baseline re-measure -> throttling detection
for (size_t i = 0; i < cs.size(); ++i) {
set_ov(cs[i].Q, cs[i].NE);
cs[i].t = time_fa_cell_median(backend_metal, cell, REPS);
clr_ov();
if (i % 4 == 0) {
const double a = time_fa_cell_median(backend_metal, cell, REPS);
if (anchor > 0.0) {
double drift = (a - anchor) / anchor;
if (drift < 0) {
drift = -drift;
}
if (drift > 0.10) {
printf("# WARN throttling? anchor drift %.1f%% dk=%d ne11=%d\n", 100.0 * drift, s.dk, ne11);
}
}
anchor = (anchor > 0.0) ? std::min(anchor, a) : a;
}
}
std::sort(cs.begin(), cs.end(), [](const cand_t & a, const cand_t & b) {
return a.Q != b.Q ? a.Q < b.Q : a.NE < b.NE;
});
cand_t best = cs[0];
for (const auto & c : cs) {
if (c.t > 0.0 && (best.t <= 0.0 || c.t < best.t)) {
best = c;
}
}
double base_t = 0.0; // the swept (Q=1, base_ne) candidate == baseline kernel
for (const auto & c : cs) {
if (c.Q == 1 && c.NE == base_ne) { base_t = c.t; break; }
}
const bool keep = best.t > 0.0 && base_t > 0.0 && best.t < base_t * 0.98;
printf("# dtype=%s dk=%d dv=%d ne11=%d ne01=%d:", ggml_type_name(type_kv), s.dk, s.dv, ne11, ne01);
for (const auto & c : cs) {
printf(" Q%dNE%d=%.1f%s", c.Q, c.NE, c.t, (c.Q == best.Q && c.NE == best.NE) ? "*" : "");
}
if (keep) {
printf(" => Q%d,NE%d %.2fx\n", best.Q, best.NE, base_t / best.t);
} else {
printf(" => baseline\n");
}
pts.push_back({ s.dk, s.dv, ne11, ne01, cs, base_t });
}
}
}
// [2] Compress into pasteable rows. Per (dk,dv) and ne01 domain {decode==1, batch>=2}, emit
// one ne11-collapsed default cfg (fewest rows, ne11_b=-1) plus a per-bucket exception wherever
// the default's pointwise regret vs the bucket's min-max-regret target, or its aggregate
// slowdown vs baseline, exceeds TUNE_TAU. The target still carries an intrinsic per-point regret
// from lumping ne01 into one tail bucket — TUNE_TAU bounds resolved-vs-target, not vs-oracle.
std::vector<std::string> rows_out;
char rbuf[192];
for (auto s : shapes) {
const int base_ne = baseline_ne(s.dk, s.dv);
std::vector<cand_t> cfgs; // candidate list (identical for every grid point of this shape)
int base_i = 0;
for (int ne : fa_vec_legal_ne(s.dk, s.dv)) {
for (int Q : { 1, 2, 4 }) {
if (Q == 1 && ne == base_ne) { base_i = (int) cfgs.size(); }
cfgs.push_back({ Q, ne, 0.0 });
}
}
auto cfg_time = [&](const pt_t & p, int i) {
for (const auto & c : p.cs) { if (c.Q == cfgs[i].Q && c.NE == cfgs[i].NE) { return c.t; } }
return 0.0;
};
// Bucket the grid points using the runtime's bucketers (via proc), so keys match fa_vec_pick.
// Short-KV points (ne11 bucket 0) are dropped — the runtime serves those from baseline. Each
// kept bucket picks a min-max-regret target (gated by the aggregate TUNE_THETA benefit gate).
struct bkt_t { int b11, b01, Ti; std::vector<double> agg; double base_agg;
std::vector<const pt_t *> bp; }; // bp kept for the pointwise ride test below
std::set<std::pair<int, int>> seen;
for (const auto & p : pts) {
if (p.dk != s.dk || p.dv != s.dv) { continue; }
const int b11 = ne11_bucket(p.ne11);
if (b11 == 0) { continue; }
seen.insert({ b11, ne01_bucket(p.ne01) });
}
std::vector<bkt_t> bks;
for (const auto & bb : seen) {
const int b11 = bb.first, b01 = bb.second;
std::vector<const pt_t *> bp;
for (const auto & p : pts) {
if (p.dk == s.dk && p.dv == s.dv && ne11_bucket(p.ne11) == b11 && ne01_bucket(p.ne01) == b01) {
bp.push_back(&p);
}
}
std::vector<double> agg(cfgs.size(), 0.0), worst(cfgs.size(), 0.0);
for (const auto * p : bp) {
double bestt = 0.0;
for (size_t i = 0; i < cfgs.size(); ++i) {
double t = cfg_time(*p, (int) i);
if (t > 0.0 && (bestt == 0.0 || t < bestt)) { bestt = t; }
}
for (size_t i = 0; i < cfgs.size(); ++i) {
double t = cfg_time(*p, (int) i);
agg[i] += t;
if (t > 0.0 && bestt > 0.0) { worst[i] = std::max(worst[i], t / bestt); }
}
}
int robust = 0;
for (size_t i = 1; i < cfgs.size(); ++i) {
if (worst[i] < worst[robust] ||
(worst[i] == worst[robust] && (cfgs[i].Q < cfgs[robust].Q ||
(cfgs[i].Q == cfgs[robust].Q && cfgs[i].NE < cfgs[robust].NE)))) {
robust = (int) i;
}
}
const bool tune = robust != base_i && agg[base_i] / agg[robust] >= TUNE_THETA;
bks.push_back({ b11, b01, tune ? robust : base_i, agg, agg[base_i], bp });
}
// Pointwise regret of default cfg d vs the bucket target Ti. Per-point (not aggregate):
// a ratio-of-sums lets a default that wins on aligned ne01 (8, 16) hide a large penalty
// on a misaligned point (ne01=5), so the ride/exception decision must be per point.
auto reg_pointwise = [&](const bkt_t * b, int d) {
double r = 0.0;
for (const auto * p : b->bp) {
const double td = cfg_time(*p, d), tT = cfg_time(*p, b->Ti);
if (td > 0.0 && tT > 0.0) { r = std::max(r, td / tT - 1.0); }
}
return r;
};
for (int dom = 0; dom <= 1; ++dom) { // 0 = decode (ne01==1), 1 = batch (ne01>=2)
std::vector<const bkt_t *> db;
for (const auto & b : bks) { if ((dom == 0) == (b.b01 == 0)) { db.push_back(&b); } }
if (db.empty()) { continue; }
// default cfg = the one minimizing (#rows, total achieved time, Q, NE)
int bestD = -1, bestRows = 1 << 30; double bestTot = 0.0;
for (size_t d = 0; d < cfgs.size(); ++d) {
int rows = ((int) d != base_i) ? 1 : 0; double tot = 0.0;
for (const auto * b : db) {
const double reg = reg_pointwise(b, (int) d); // pointwise vs bucket target
const double slow = b->agg[d] / b->base_agg - 1.0; // aggregate vs baseline (safety net)
if (reg > TUNE_TAU || slow > TUNE_TAU) { rows++; tot += b->agg[b->Ti]; }
else { tot += b->agg[d]; }
}
const bool better = bestD < 0 || rows < bestRows ||
(rows == bestRows && (tot < bestTot ||
(tot == bestTot && (cfgs[d].Q < cfgs[bestD].Q ||
(cfgs[d].Q == cfgs[bestD].Q && cfgs[d].NE < cfgs[bestD].NE)))));
if (better) { bestD = (int) d; bestRows = rows; bestTot = tot; }
}
const int dom_id = (dom == 0) ? 0 : 1; // FA_VEC_DOMAIN_DECODE / FA_VEC_DOMAIN_BATCH
if (bestD != base_i) {
snprintf(rbuf, sizeof(rbuf), " { { GGML_METAL_DEVICE_M4_MAX, %s, %d, %d, -1, %d }, { %d, %d } },",
type_token(type_kv), s.dk, s.dv, dom_id, cfgs[bestD].Q, cfgs[bestD].NE);
rows_out.emplace_back(rbuf);
}
for (const auto * b : db) {
const double reg = reg_pointwise(b, bestD); // pointwise vs bucket target
const double slow = b->agg[bestD] / b->base_agg - 1.0; // aggregate vs baseline
if (reg <= TUNE_TAU && slow <= TUNE_TAU) { continue; } // rides the default / baseline
snprintf(rbuf, sizeof(rbuf), " { { GGML_METAL_DEVICE_M4_MAX, %s, %d, %d, %d, %d }, { %d, %d } },",
type_token(type_kv), s.dk, s.dv, b->b11, b->b01, cfgs[b->Ti].Q, cfgs[b->Ti].NE);
rows_out.emplace_back(rbuf);
}
}
}
printf("\n // ---- %s: %zu rows ----\n", ggml_type_name(type_kv), rows_out.size());
for (const auto & r : rows_out) { printf("%s\n", r.c_str()); }
} // for type_kv
return true;
return n_fail == 0;
}
static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mode mode, const char * op_names_filter, const char * params_filter,
printer * output_printer, const char * test_file_path, int parallel_workers, bool tune_perf = false) {
printer * output_printer, const char * test_file_path, int parallel_workers) {
auto filter_test_cases = [](std::vector<std::unique_ptr<test_case>> & test_cases, const char * params_filter) {
if (params_filter == nullptr) {
return;
@@ -11079,9 +10675,6 @@ static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mo
case MODE_PERF:
test_cases = make_test_cases_perf();
break;
case MODE_TUNE:
// MODE_TUNE routes to its own dispatch below; no generic test_cases needed
break;
}
} else {
test_cases = make_test_cases_from_file(test_file_path);
@@ -11188,7 +10781,11 @@ static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mo
output_printer->print_summary(test_summary_info(n_ok, tests_run, false));
output_printer->print_failed_tests(failed_tests);
return n_ok == tests_run;
// Metal-only: force every legal (Q,NE) on a bounded slice of shapes. Reuses the
// reference CPU backend above; a no-op on backends without the override proc.
const bool slice_ok = run_fa_vec_slice(backend, backend_cpu.get());
return n_ok == tests_run && slice_ok;
}
if (mode == MODE_GRAD) {
@@ -11217,29 +10814,6 @@ static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mo
return true;
}
if (mode == MODE_TUNE) {
// self-create a CPU backend with the reference implementation as golden.
// (backend_cpu in the MODE_TEST block above is out of scope here.)
ggml_backend_t backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, NULL);
GGML_ASSERT(backend_cpu != NULL);
{
using ggml_backend_cpu_set_use_ref_t = void (*)(ggml_backend_t, bool);
auto * cpu_reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu));
auto * set_use_ref = (ggml_backend_cpu_set_use_ref_t) ggml_backend_reg_get_proc_address(cpu_reg, "ggml_backend_cpu_set_use_ref");
if (set_use_ref) {
set_use_ref(backend_cpu, true);
}
}
// backend here is the MODE_TUNE-selected metal backend (-b MTL0)
const bool ok = tune_perf
? run_fa_vec_tune_perf(backend)
: (run_fa_vec_drift_guard(backend, backend_cpu) && run_fa_vec_tune_check(backend, backend_cpu));
ggml_backend_free(backend_cpu);
return ok;
}
if (mode == MODE_SUPPORT) {
// Filter out fusion cases
test_cases.erase(
@@ -11365,7 +10939,6 @@ static void usage(char ** argv) {
printf(" - grad (compare gradients from backpropagation with method of finite differences)\n");
printf(" - perf (performance evaluation)\n");
printf(" - support (probe backend operation support)\n");
printf(" - tune (FA vec (Q,NE) numerical correctness check vs CPU reference; --tune-perf for the perf sweep)\n");
printf(" op names for -o are as given by ggml_op_desc() (e.g. ADD, MUL_MAT, etc),\n");
printf(" optionally including the full test case string (e.g. \"ADD(type=f16,ne=[1,1,8,1],nr=[1,1,1,1],nf=1)\")\n");
printf(" --output specifies output format (default: console, options: console, sql, csv)\n");
@@ -11383,7 +10956,6 @@ int main(int argc, char ** argv) {
const char * params_filter = nullptr;
const char * test_file_path = nullptr;
int parallel_workers = 1;
bool tune_perf = false;
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "test") == 0) {
@@ -11394,10 +10966,6 @@ int main(int argc, char ** argv) {
mode = MODE_GRAD;
} else if (strcmp(argv[i], "support") == 0) {
mode = MODE_SUPPORT;
} else if (strcmp(argv[i], "tune") == 0) {
mode = MODE_TUNE;
} else if (strcmp(argv[i], "--tune-perf") == 0) {
tune_perf = true;
} else if (strcmp(argv[i], "-o") == 0) {
if (i + 1 < argc) {
op_names_filter = argv[++i];
@@ -11505,7 +11073,7 @@ int main(int argc, char ** argv) {
false, "", ggml_backend_dev_description(dev),
total / 1024 / 1024, free / 1024 / 1024, true));
bool ok = test_backend(backend.get(), dev, mode, op_names_filter, params_filter, output_printer.get(), test_file_path, parallel_workers, tune_perf);
bool ok = test_backend(backend.get(), dev, mode, op_names_filter, params_filter, output_printer.get(), test_file_path, parallel_workers);
if (ok) {
n_ok++;