From ea337d0cda6114cc40cb776d4261d0c979ad058d Mon Sep 17 00:00:00 2001
From: forforever73 <690105611@qq.com>
Date: Sun, 2 Aug 2026 00:15:20 +0800
Subject: [PATCH] tools : document the Metal tuner, point the table comment at
 it

---
 ggml/src/ggml-metal/ggml-metal-tuning.cpp |  2 +-
 ggml/src/ggml-metal/ggml-metal-tuning.h   |  5 +-
 tools/tuning/README.md                    | 61 +++++++++++++++++++++++
 3 files changed, 65 insertions(+), 3 deletions(-)
 create mode 100644 tools/tuning/README.md

diff --git a/ggml/src/ggml-metal/ggml-metal-tuning.cpp b/ggml/src/ggml-metal/ggml-metal-tuning.cpp
index 2bf9bb5b48..6eb56a7ef2 100644
--- a/ggml/src/ggml-metal/ggml-metal-tuning.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-tuning.cpp
@@ -62,7 +62,7 @@ fa_vec_cfg_t fa_vec_baseline_cfg(int dk, int dv) {
     return { 1, (int8_t) fa_vec_baseline_ne(dk, dv) };
 }
 
-// Generated by `test-backend-ops tune --tune-perf`; do not hand-edit.
+// Generated by `ggml-metal-tuning fa-vec`; do not hand-edit.
 // One row per kept bucket, plus per-(dtype,dk,dv) ne11-collapsed domain defaults
 // (ne11_b = FA_VEC_NE11_DEFAULT, ne01_b = domain). To retune or add a device, re-run the
 // sweep and paste its block. See ggml-metal-tuning.h for the row/lookup semantics.
diff --git a/ggml/src/ggml-metal/ggml-metal-tuning.h b/ggml/src/ggml-metal/ggml-metal-tuning.h
index 6ea948b407..8f1c2b238c 100644
--- a/ggml/src/ggml-metal/ggml-metal-tuning.h
+++ b/ggml/src/ggml-metal/ggml-metal-tuning.h
@@ -18,8 +18,9 @@ int fa_vec_ne11_bucket(int64_t ne11);
 int fa_vec_ne01_bucket(int64_t ne01);
 
 // NE baked into each (dk,dv) baseline instantiation in kernels/fa.metal.
-// Hand-maintained mirror; keep in sync with those instantiations (run_fa_vec_tune_check
-// exercises every (Q,NE), so a missing instantiation surfaces there).
+// Hand-maintained mirror; keep in sync with those instantiations. test-backend-ops forces
+// every legal (Q,NE) on dk=128 and dk=576 under Metal, so a missing instantiation for
+// those two surfaces there; the other head sizes are only covered by the offline tuner.
 int fa_vec_baseline_ne(int dk, int dv);
 
 // Tuned table has two row kinds. Exact rows key a (ne11_b, ne01_b) bucket. Default rows
diff --git a/tools/tuning/README.md b/tools/tuning/README.md
new file mode 100644
index 0000000000..e22f2842bf
--- /dev/null
+++ b/tools/tuning/README.md
@@ -0,0 +1,61 @@
+# ggml-metal-tuning
+
+Offline kernel tuner for the Metal backend.
+It sweeps a kernel's config grid on the machine it runs on and prints pasteable table rows for `ggml/src/ggml-metal/ggml-metal-tuning.cpp`.
+
+This is not a test: it never reports pass/fail on performance.
+A non-zero exit code means bad arguments or a wrong environment (no Metal device, missing proc bridges), never a perf result.
+
+| tuner | tunes | table |
+|---|---|---|
+| `fa-vec` | flash-attn vec `(Q, NE)` per `(dtype, head size, KV depth, batch width)` | `fa_vec_tuned_table` |
+
+## Adding a device to the FA-vec table
+
+Build on the target machine:
+
+```bash
+cmake -B build -DGGML_METAL=ON
+cmake --build build --target ggml-metal-tuning -j
+cmake --build build --target test-backend-ops -j
+```
+
+Sweep the grid (6 dtypes x 10 head sizes x 4 KV depths x 9 batch widths; a few hours):
+
+```bash
+./build/bin/ggml-metal-tuning fa-vec > fa_vec_rows.txt 2> fa_vec_sweep.log
+```
+
+`fa_vec_rows.txt` is the finished table block, ready to paste: the min-max-regret target, the aggregate benefit gate, the short-KV drop and the pointwise compression are already applied.
+`fa_vec_sweep.log` holds the per-cell timings, bucket coverage, noise floor and any cooldown activity.
+Post both: the log is what makes the rows reviewable.
+
+Long sweeps can be split.
+`--dtype f16,q4_0` and `--dk 128,192` restrict the grid, and the emitted rows for one `(dtype, head size)` do not depend on the others.
+
+Then validate the numerics, where Metal is compared against the CPU reference:
+
+```bash
+./build/bin/test-backend-ops test -o FLASH_ATTN_EXT -b MTL0
+```
+
+This forces every legal `(Q, NE)` on `dk=128` and `dk=576`.
+The tuner itself does no numerical checks, so the other head sizes have no automated numerical coverage.
+
+If the device is not in `enum ggml_metal_device_id` yet, register it in `ggml/src/ggml-metal/ggml-metal-device.{h,m}` first.
+The tuner emits whatever token the runtime reports for the machine, so an unregistered device emits `GGML_METAL_DEVICE_GENERIC` and its rows would apply to every unknown device.
+
+## Thermal throttling
+
+Long sweeps heat the GPU, and a throttled measurement is indistinguishable from a slow kernel.
+The tuner re-measures a fixed baseline config every four candidates as an anchor.
+When the anchor drifts more than `--cool-drift` (10% by default) from the coolest anchor seen in that cell, the tuner:
+
+1. discards every candidate measured since the last clean anchor,
+2. sleeps with exponential backoff until the anchor comes back within `--cool-eps` (3%),
+3. re-measures the discarded candidates.
+
+If it cannot cool down within `--cool-max-wait` seconds, or a cell needs more than `--cool-max-retry` rounds, that cell is dropped from the table and reported on stderr.
+
+`--no-cooldown` only warns on drift and keeps the measurement.
+Use it to reproduce a sweep taken without cooling.