diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 37561563f6..c8ad1db436 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -38,5 +38,8 @@ else() add_subdirectory(export-lora) endif() add_subdirectory(fit-params) + if (GGML_METAL) + add_subdirectory(tuning) + endif() add_subdirectory(results) endif() diff --git a/tools/tuning/CMakeLists.txt b/tools/tuning/CMakeLists.txt new file mode 100644 index 0000000000..a6909c2307 --- /dev/null +++ b/tools/tuning/CMakeLists.txt @@ -0,0 +1,9 @@ +set(TARGET ggml-metal-tuning) + +add_executable(${TARGET} main.cpp bench.cpp fa-vec.cpp) +target_link_libraries(${TARGET} PRIVATE ggml ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_17) + +if(LLAMA_TOOLS_INSTALL) + install(TARGETS ${TARGET} RUNTIME) +endif() diff --git a/tools/tuning/bench.cpp b/tools/tuning/bench.cpp new file mode 100644 index 0000000000..9adf305c65 --- /dev/null +++ b/tools/tuning/bench.cpp @@ -0,0 +1 @@ +#include "bench.h" diff --git a/tools/tuning/bench.h b/tools/tuning/bench.h new file mode 100644 index 0000000000..6f70f09bee --- /dev/null +++ b/tools/tuning/bench.h @@ -0,0 +1 @@ +#pragma once diff --git a/tools/tuning/fa-vec.cpp b/tools/tuning/fa-vec.cpp new file mode 100644 index 0000000000..e21e17a4d0 --- /dev/null +++ b/tools/tuning/fa-vec.cpp @@ -0,0 +1,13 @@ +#include "fa-vec.h" + +#include + +bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tuner_opts & opts) { + fprintf(stderr, "fa-vec tuner: not implemented yet\n"); + + (void) backend; + (void) dev; + (void) opts; + + return true; +} diff --git a/tools/tuning/fa-vec.h b/tools/tuning/fa-vec.h new file mode 100644 index 0000000000..b015a8051f --- /dev/null +++ b/tools/tuning/fa-vec.h @@ -0,0 +1,20 @@ +#pragma once + +#include "ggml-backend.h" + +// options shared by all tuners; parsed in main.cpp +struct tuner_opts { + const char * dtype_filter = nullptr; // comma-separated, e.g. "f16,q4_0"; null = all + const char * dk_filter = nullptr; // comma-separated dk values, e.g. "128,192"; null = all + int reps = 7; + unsigned seed = 1234; + bool cooldown = true; + double cool_drift = 0.10; + double cool_eps = 0.03; + int cool_max_wait = 120; + int cool_max_retry = 2; +}; + +// runs the FA-vec (Q,NE) sweep and prints a pasteable table block on stdout. +// returns false only on environment failure (missing procs), never on perf results. +bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tuner_opts & opts); diff --git a/tools/tuning/main.cpp b/tools/tuning/main.cpp new file mode 100644 index 0000000000..ce9eb0a993 --- /dev/null +++ b/tools/tuning/main.cpp @@ -0,0 +1,140 @@ +#include "fa-vec.h" + +#include "ggml.h" +#include "ggml-backend.h" + +#include +#include +#include + +struct tuner_def { + const char * name; + bool (*run)(ggml_backend_t, ggml_backend_dev_t, const tuner_opts &); +}; + +static const tuner_def k_tuners[] = { + { "fa-vec", tuner_fa_vec_run }, +}; + +static void usage(const char * argv0) { + printf("usage: %s [options]\n", argv0); + printf("\n"); + printf(" offline kernel tuner for the Metal backend: sweeps a kernel's config grid and\n"); + printf(" prints pasteable table rows for the machine it runs on. never a pass/fail test.\n"); + printf("\n"); + printf(" tuners:\n"); + printf(" fa-vec flash-attn vec (Q,NE) for ggml-metal-tuning.cpp\n"); + printf("\n"); + printf(" options:\n"); + printf(" -b backend device (default: first Metal device)\n"); + printf(" --dtype restrict KV dtypes, e.g. f16,q4_0 (default: all)\n"); + printf(" --dk restrict head sizes, e.g. 128,192 (default: all)\n"); + printf(" --reps timed reps per candidate, odd for an exact median (default: 7)\n"); + printf(" --seed RNG seed; per-cell seeds mix it with the shape (default: 1234)\n"); + printf(" --no-cooldown do not pause/re-measure on thermal drift, only warn\n"); + printf(" --cool-drift anchor drift that triggers a cooldown (default: 0.10)\n"); + printf(" --cool-eps anchor tolerance to consider the GPU cool again (default: 0.03)\n"); + printf(" --cool-max-wait give up cooling a cell after this many seconds (default: 120)\n"); + printf(" --cool-max-retry re-measure rounds per cell before giving up (default: 2)\n"); + printf("\n"); + printf(" the table goes to stdout, all diagnostics to stderr:\n"); + printf(" %s fa-vec > rows.txt 2> sweep.log\n", argv0); +} + +int main(int argc, char ** argv) { + const char * tuner = nullptr; + const char * bname = nullptr; + tuner_opts opts; + + for (int i = 1; i < argc; i++) { + const char * a = argv[i]; + if (strcmp(a, "-h") == 0 || strcmp(a, "--help") == 0) { + usage(argv[0]); + return 0; + } else if (strcmp(a, "-b") == 0 && i + 1 < argc) { + bname = argv[++i]; + } else if (strcmp(a, "--dtype") == 0 && i + 1 < argc) { + opts.dtype_filter = argv[++i]; + } else if (strcmp(a, "--dk") == 0 && i + 1 < argc) { + opts.dk_filter = argv[++i]; + } else if (strcmp(a, "--reps") == 0 && i + 1 < argc) { + opts.reps = atoi(argv[++i]); + } else if (strcmp(a, "--seed") == 0 && i + 1 < argc) { + opts.seed = (unsigned) strtoul(argv[++i], nullptr, 10); + } else if (strcmp(a, "--no-cooldown") == 0) { + opts.cooldown = false; + } else if (strcmp(a, "--cool-drift") == 0 && i + 1 < argc) { + opts.cool_drift = atof(argv[++i]); + } else if (strcmp(a, "--cool-eps") == 0 && i + 1 < argc) { + opts.cool_eps = atof(argv[++i]); + } else if (strcmp(a, "--cool-max-wait") == 0 && i + 1 < argc) { + opts.cool_max_wait = atoi(argv[++i]); + } else if (strcmp(a, "--cool-max-retry") == 0 && i + 1 < argc) { + opts.cool_max_retry = atoi(argv[++i]); + } else if (a[0] != '-' && tuner == nullptr) { + tuner = a; + } else { + fprintf(stderr, "error: unrecognized or incomplete argument: %s\n\n", a); + usage(argv[0]); + return 1; + } + } + + if (tuner == nullptr) { + usage(argv[0]); + return 1; + } + if (opts.reps < 1) { + fprintf(stderr, "error: --reps must be >= 1\n"); + return 1; + } + + const tuner_def * t = nullptr; + for (const auto & cand : k_tuners) { + if (strcmp(tuner, cand.name) == 0) { + t = &cand; + break; + } + } + if (t == nullptr) { + fprintf(stderr, "error: unknown tuner: %s\n\n", tuner); + usage(argv[0]); + return 1; + } + + ggml_backend_load_all(); + + ggml_backend_dev_t dev = nullptr; + for (size_t i = 0; i < ggml_backend_dev_count(); i++) { + ggml_backend_dev_t d = ggml_backend_dev_get(i); + if (bname) { + if (strcmp(ggml_backend_dev_name(d), bname) == 0) { + dev = d; + break; + } + } else if (strncmp(ggml_backend_dev_name(d), "MTL", 3) == 0) { + dev = d; + break; + } + } + + if (dev == nullptr) { + fprintf(stderr, "error: no %s device found\n", bname ? bname : "Metal"); + return 1; + } + + ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr); + if (backend == nullptr) { + fprintf(stderr, "error: failed to init backend %s\n", ggml_backend_dev_name(dev)); + return 1; + } + + fprintf(stderr, "device: %s (%s)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev)); + + const bool ok = t->run(backend, dev, opts); + + ggml_backend_free(backend); + ggml_quantize_free(); + + return ok ? 0 : 1; +}