Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 01:04:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

add op-agnostic perf cell + median timing for the tuner

Browse Source
This commit is contained in:
forforever73
2026-07-31 14:56:55 +08:00
committed by Georgi Gerganov
parent 1f9a360b00
commit 0a540a5e89
2 changed files with 181 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/tuning/bench.cpp
+118
View File
@@ -1 +1,119 @@
#include "bench.h"
#include <algorithm>
#include <cmath>
#include <cstdio>
perf_cell build_perf_cell(ggml_backend_t backend, const build_graph_fn & build,
const init_tensors_fn & init, const op_flops_fn & flops) {
perf_cell cell;
const size_t graph_nodes = 1024;
ggml_init_params params = {
/* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead_custom(graph_nodes, false),
/* .mem_base = */ NULL,
/* .no_alloc = */ true,
};
cell.ctx.reset(ggml_init(params));
GGML_ASSERT(cell.ctx);
ggml_tensor * out = build(cell.ctx.get());
if (!out || !ggml_backend_supports_op(backend, out)) {
return cell;
}
cell.buf.reset(ggml_backend_alloc_ctx_tensors(cell.ctx.get(), backend));
if (cell.buf == NULL) {
return cell;
}
init(cell.ctx.get());
cell.gf = ggml_new_graph_custom(cell.ctx.get(), graph_nodes, false);
ggml_build_forward_expand(cell.gf, out);
// replicate the op to amortize overhead (target ~50 GFLOP/compute, capped to bound graph size)
cell.n_runs = 1;
if (flops(out) > 0) {
const uint64_t target_flops = 50ULL * 1000 * 1000 * 1000;
const int cap = 512;
const int by_flops = (int) std::min<int64_t>(cap, (int64_t) (target_flops / flops(out)));
cell.n_runs = std::max(1, std::min<int>(by_flops, (int) (ggml_graph_size(cell.gf) - ggml_graph_n_nodes(cell.gf))));
}
for (int i = 1; i < cell.n_runs; ++i) {
ggml_graph_add_node(cell.gf, out);
}
cell.ok = true;
return cell;
}
double time_cell_median(ggml_backend_t backend, const perf_cell & cell, int reps) {
if (!cell.ok) {
return -1.0;
}
ggml_backend_graph_compute(backend, cell.gf); // warmup (compiles the pipeline for this config)
ggml_backend_synchronize(backend);
std::vector<double> samples;
samples.reserve(reps);
for (int r = 0; r < reps; ++r) {
const int64_t t0 = ggml_time_us();
ggml_backend_graph_compute(backend, cell.gf);
ggml_backend_synchronize(backend);
samples.push_back((double) (ggml_time_us() - t0));
}
std::nth_element(samples.begin(), samples.begin() + samples.size()/2, samples.end());
return samples[samples.size()/2] / cell.n_runs;
}
cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int reps,
int n_cands, const std::vector<int> & order,
const set_candidate_fn & set_cand,
const clear_candidate_fn & clear_cand,
int baseline_cand, const cooldown_opts & cool,
const char * cell_label) {
cell_result res;
res.t.assign(n_cands, 0.0);
double anchor_ref = 0.0;
for (size_t i = 0; i < order.size(); ++i) {
set_cand(order[i]);
res.t[order[i]] = time_cell_median(backend, cell, reps);
clear_cand();
if (i % 4 != 0) {
continue;
}
// re-measure the baseline config as an anchor: same config every time, so any
// change is the machine, not the kernel
set_cand(baseline_cand);
const double a = time_cell_median(backend, cell, reps);
clear_cand();
if (a <= 0.0) {
continue;
}
res.anchor_min = res.anchor_min > 0.0 ? std::min(res.anchor_min, a) : a;
res.anchor_max = std::max(res.anchor_max, a);
if (anchor_ref > 0.0) {
const double drift = std::fabs(a - anchor_ref) / anchor_ref;
if (drift > cool.drift) {
fprintf(stderr, "# WARN throttling? anchor drift %.1f%% %s\n", 100.0*drift, cell_label);
}
}
anchor_ref = anchor_ref > 0.0 ? std::min(anchor_ref, a) : a;
}
return res;
}
tools/tuning/bench.h
+63
View File
@@ -1 +1,64 @@
#pragma once
#include "ggml.h"
#include "ggml-backend.h"
#include "ggml-cpp.h"
#include <cstdint>
#include <functional>
#include <vector>
// one prebuilt op graph, replicated n_runs times so a single graph_compute amortizes
// dispatch/sync overhead. reused across candidates: an override only changes which
// pipeline is picked at encode time, so the (large) input tensors stay allocated.
struct perf_cell {
ggml_context_ptr ctx;
ggml_backend_buffer_ptr buf;
ggml_cgraph * gf = nullptr;
int n_runs = 0;
bool ok = false;
};
// builds the op graph for one shape. returns the output tensor, or null if unsupported.
using build_graph_fn = std::function<ggml_tensor *(ggml_context *)>;
// fills the allocated tensors of ctx with input data
using init_tensors_fn = std::function<void(ggml_context *)>;
// flops of one op instance, used to size n_runs
using op_flops_fn = std::function<uint64_t(ggml_tensor *)>;
perf_cell build_perf_cell(ggml_backend_t backend, const build_graph_fn & build,
const init_tensors_fn & init, const op_flops_fn & flops);
// median per-op time (us) over the prebuilt cell for whatever config is currently set
double time_cell_median(ggml_backend_t backend, const perf_cell & cell, int reps);
struct cooldown_opts {
bool enabled = true;
double drift = 0.10; // anchor drift that triggers a cooldown
double eps = 0.03; // anchor tolerance to call the GPU cool again
int max_wait = 120; // seconds of cooling per cell before giving up
int max_retry = 2; // re-measure rounds per cell before giving up
};
// applies candidate i (an index into the tuner's own candidate list)
using set_candidate_fn = std::function<void(int)>;
// undoes the last set_candidate
using clear_candidate_fn = std::function<void()>;
struct cell_result {
std::vector<double> t; // time (us) per candidate index, <= 0 if not measured
bool trusted = true; // false -> caller must drop this cell
double anchor_min = 0.0;
double anchor_max = 0.0;
int n_cooldowns = 0;
};
// times every candidate over the prebuilt cell, re-measuring a periodic baseline anchor
// to watch for thermal drift. order[] gives the (shuffled) visiting order; baseline_cand is
// the candidate the anchor forces, so drift is measured against a config the tuner controls.
cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int reps,
int n_cands, const std::vector<int> & order,
const set_candidate_fn & set_cand,
const clear_candidate_fn & clear_cand,
int baseline_cand, const cooldown_opts & cool,
const char * cell_label);