Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 09:15:04 +02:00
Code Issues Packages Projects Releases Wiki Activity

common: add enum common_speculative_type

Browse Source
This commit is contained in:
Sascha Rogmann
2026-01-18 18:45:10 +01:00
parent 456268fa7f
commit b38eb5907c
8 changed files with 359 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/speculative.cpp
+243 -32
View File
@@ -14,6 +14,26 @@
#define SPEC_VOCAB_MAX_SIZE_DIFFERENCE 128
#define SPEC_VOCAB_CHECK_START_TOKEN_ID 5
const std::vector<enum common_speculative_type> common_speculative_types = {
COMMON_SPECULATIVE_TYPE_NONE,
COMMON_SPECULATIVE_TYPE_DRAFT,
COMMON_SPECULATIVE_TYPE_EAGLE3,
COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE,
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K,
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V,
COMMON_SPECULATIVE_TYPE_NGRAM_CACHE
};
const std::map<std::string, enum common_speculative_type> common_speculative_type_from_name_map = {
{"none", COMMON_SPECULATIVE_TYPE_NONE},
{"draft", COMMON_SPECULATIVE_TYPE_DRAFT},
{"eagle3", COMMON_SPECULATIVE_TYPE_EAGLE3},
{"ngram_simple", COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE},
{"ngram_map_k", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K},
{"ngram_map_k4v", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V},
{"ngram_cache", COMMON_SPECULATIVE_TYPE_NGRAM_CACHE}
};
struct common_speculative_self {
uint16_t size_ngram = 12; // size of n-grams to lookup in self-mode
uint16_t size_mgram = 48; // size of m-grams to draft in self-mode
@@ -21,6 +41,16 @@ struct common_speculative_self {
size_t idx_last_check = 0; // index of last check in context history
};
struct common_speculative_impl {
const enum common_speculative_type type;
size_t drafts_call_count = 0; // number of times this implementation was called.
size_t drafts_generated_count = 0; // number of times a draft or part was generated by this implementation.
size_t drafts_accepted_count = 0; // number of times a draft or part was accepted by the target model.
size_t drafts_generated_tokens = 0; // number of tokens generated by this implementation.
size_t drafts_accepted_tokens = 0; // number of tokens accepted by the target model.
};
struct common_speculative {
struct llama_context * ctx_tgt; // only used for retokenizing from ctx_dft
struct llama_context * ctx_dft;
@@ -31,29 +61,133 @@ struct common_speculative {
bool vocab_dft_compatible = true; // whether retokenization is needed
std::map<std::string, std::string> tgt_dft_replacements = {};
const uint16_t self_mode = 0; // 0: off, 1: self speculative, 2: n-grams (keys) only, 3: n-grams/m-grams (key-values)
common_ngram_map map; // draft ngram map for speculative decoding without draft model
common_speculative_self self_state; // state of self-speculation (simple implementation, not ngram-map)
std::vector<common_speculative_impl> impls; // list of implementations to use and their statistics
common_speculative_impl * curr_impl = nullptr; // current implementation in use (for stats)
};
common_ngram_map get_common_ngram_map(std::vector<common_speculative_config> configs);
common_ngram_map get_common_ngram_map(std::vector<common_speculative_config> configs) {
for (const auto & config : configs) {
switch (config.type) {
case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE:
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K:
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: {
// create common_ngram_map from config.config
// compute size_key, size_value, key_only, check_rate, min_hits from config.config
uint16_t size_key = 12;
uint16_t size_value = 48;
bool key_only = false;
uint16_t check_rate = 3;
uint16_t min_hits = 1;
const std::map<std::string, std::string> & cfg = config.config;
// check for key "size_ngram" in cfg
if (cfg.find("size_ngram") != cfg.end()) {
size_key = std::stoi(cfg.at("size_ngram"));
if (size_key < 1 || size_key > 1024) {
throw std::invalid_argument("size_ngram must be between 1 and 1024");
}
}
// check for key "size_mgram" in cfg
if (cfg.find("size_mgram") != cfg.end()) {
size_value = std::stoi(cfg.at("size_mgram"));
if (size_value < 1 || size_value > 1024) {
throw std::invalid_argument("size_mgram must be between 1 and 1024");
}
}
// check for key "key_only" in cfg
if (cfg.find("key_only") != cfg.end()) {
// key_onle == true, if cfg.at("key_only") has value "true".
key_only = (cfg.at("key_only") == "true");
}
// check for key "check_rate" in cfg
if (cfg.find("check_rate") != cfg.end()) {
check_rate = std::stoi(cfg.at("check_rate"));
if (check_rate < 1 || check_rate > 1024) {
throw std::invalid_argument("check_rate must be between 1 and 1024");
}
}
// check for key "min_hits" in cfg
if (cfg.find("min_hits") != cfg.end()) {
min_hits = std::stoi(cfg.at("min_hits"));
if (min_hits < 1 || min_hits > 1024) {
throw std::invalid_argument("min_hits must be between 1 and 1024");
}
}
return common_ngram_map(size_key, size_value, key_only, check_rate, min_hits);
break;
}
case COMMON_SPECULATIVE_TYPE_NONE:
case COMMON_SPECULATIVE_TYPE_DRAFT:
case COMMON_SPECULATIVE_TYPE_EAGLE3:
case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE:
break;
case COMMON_SPECULATIVE_TYPE_COUNT:
break;
}
}
return common_ngram_map(12, 48, false, 3, 1); // default fallback
}
std::string common_speculative_type_name_str() {
std::string result = "";
for (size_t i = 0; i < common_speculative_types.size(); i++) {
if (i > 0) {
result += ", ";
}
result += common_speculative_type_to_str(common_speculative_types[i]);
}
return result;
}
std::string common_speculative_type_to_str(enum common_speculative_type type) {
switch (type) {
case COMMON_SPECULATIVE_TYPE_NONE: return "none";
case COMMON_SPECULATIVE_TYPE_DRAFT: return "draft";
case COMMON_SPECULATIVE_TYPE_EAGLE3: return "eagle3";
case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: return "ngram_simple";
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: return "ngram_map_k";
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: return "ngram_map_k4v";
case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE: return "ngram_cache";
default: return "unknown";
}
}
enum common_speculative_type common_speculative_type_from_name(const std::string & name) {
const auto it = common_speculative_type_from_name_map.find(name);
if (it == common_speculative_type_from_name_map.end()) {
return COMMON_SPECULATIVE_TYPE_COUNT;
}
return it->second;
}
struct common_speculative * common_speculative_init(
struct llama_context * ctx_tgt,
struct llama_context * ctx_dft,
uint16_t self_mode, // 0: off, 1: self speculative, 2: n-grams (keys) only, 3: n-grams/m-grams (key-values)
const std::vector<uint16_t> self_cfg // ngram size, mgram size, keys only (0|1), min hits
const std::vector<common_speculative_config> configs
) {
uint16_t ngram_size_key = self_cfg.size() >= 1 ? self_cfg[0] : 12;
uint16_t mgram_size_value = self_cfg.size() >= 2 ? self_cfg[1] : 48;
uint16_t check_rate = self_cfg.size() >= 3 ? self_cfg[2] : 3;
bool key_only = (self_mode != 3);
uint16_t min_hits = self_cfg.size() >= 4 ? self_cfg[3] : 1;
common_ngram_map ngram_map = common_ngram_map(ngram_size_key, mgram_size_value, key_only, check_rate, min_hits);
common_ngram_map ngram_map = get_common_ngram_map(configs);
uint16_t ngram_size_key = ngram_map.size_key;
uint16_t mgram_size_value = ngram_map.size_value;
uint16_t check_rate = ngram_map.check_rate;
common_speculative_self self_state = common_speculative_self{
/* .size_ngram = */ ngram_size_key,
/* .size_mgram = */ mgram_size_value,
/* .check_rate = */ check_rate,
/* .idx_last_check = */ 0,
};
std::vector<common_speculative_impl> implementations = {};
LOG_INF("common_speculative_init: configs.size = %zu\n", configs.size());
for (const auto & config : configs) {
LOG_INF("common_speculative_init: adding implementation %s\n", common_speculative_type_to_str(config.type).c_str());
implementations.push_back(common_speculative_impl{config.type});
}
auto * result = new common_speculative {
/* .ctx_tgt = */ ctx_tgt,
/* .ctx_dft = */ ctx_dft,
@@ -62,9 +196,9 @@ struct common_speculative * common_speculative_init(
/* .prompt_dft = */ {},
/* .vocab_dft_compatible = */ false,
/* .tgt_dft_replacements = */ {},
/* .self_mode = */ self_mode,
/* .map = */ ngram_map,
/* .self_state = */ self_state
/* .self_state = */ self_state,
/* .impls = */ implementations
};
LOG_INF("common_speculative_init: created speculative decoder, map.n = %d\n", result->map.size_key);
@@ -218,6 +352,12 @@ static std::string replace_to_tgt(
return result;
}
llama_tokens common_speculative_use_draft_model(
struct common_speculative * spec,
struct common_speculative_params params,
const llama_tokens & prompt_tgt_main_model, // specified in target model vocab
llama_token id_last);
llama_tokens common_speculative_gen_self_draft(
common_speculative * spec,
const llama_tokens & tokens, llama_token sampled);
@@ -227,17 +367,78 @@ llama_tokens common_speculative_gen_draft(
struct common_speculative_params params,
const llama_tokens & prompt_tgt_main_model, // specified in target model vocab
llama_token id_last) {
if (spec->self_mode) {
// Look in the current context for a n-gram and return the following tokens as the draft.
llama_tokens draft_self = common_speculative_gen_self_draft(spec,
prompt_tgt_main_model, id_last);
if (!draft_self.empty()) {
return draft_self;
llama_tokens result = {};
spec->curr_impl = nullptr; // reset current implementation
for (auto & impl : spec->impls) {
impl.drafts_call_count++;
// LOG name and call_count
switch (impl.type) {
case COMMON_SPECULATIVE_TYPE_NONE:
{
break;
}
case COMMON_SPECULATIVE_TYPE_DRAFT:
{
// Create a draft using a draft model.
result = common_speculative_use_draft_model(spec, params, prompt_tgt_main_model, id_last);
break;
}
case COMMON_SPECULATIVE_TYPE_EAGLE3:
{
// Work in progress: https://github.com/ggml-org/llama.cpp/pull/18039
break;
}
case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE:
{
// Use common_ngram_map_draft to generate a draft from the current context.
result = common_speculative_gen_self_draft(spec, prompt_tgt_main_model, id_last);
break;
}
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K:
{
// Use common_ngram_map_draft to generate a draft from the current context.
common_ngram_map_draft(spec->map, prompt_tgt_main_model, id_last, result);
break;
}
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V:
{
// Use common_ngram_map_draft to generate a draft from the current context.
common_ngram_map_draft(spec->map, prompt_tgt_main_model, id_last, result);
break;
}
case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE:
{
// TODO call common/ngram-cache.cpp
break;
}
case COMMON_SPECULATIVE_TYPE_COUNT:
{
GGML_ABORT("invalid speculative type COUNT");
break;
}
}
if (!result.empty()) {
LOG_DBG("%s: called impl %s, hist size = %zu, call_count = %zu, gen = %zu\n", __func__,
common_speculative_type_to_str(impl.type).c_str(),
prompt_tgt_main_model.size(),
impl.drafts_call_count, result.size());
spec->curr_impl = &impl; // set current implementation for stats
impl.drafts_generated_count++;
impl.drafts_generated_tokens += result.size();
break; // We have a draft, so break out of the loop and return it.
}
}
if (spec == nullptr || spec->ctx_dft == nullptr) {
return {}; // no draft model, return
}
return result;
}
llama_tokens common_speculative_use_draft_model(
struct common_speculative * spec,
struct common_speculative_params params,
const llama_tokens & prompt_tgt_main_model, // specified in target model vocab
llama_token id_last) {
auto & batch = spec->batch;
auto & ctx_tgt = spec->ctx_tgt;
@@ -413,8 +614,17 @@ llama_tokens common_speculative_gen_draft(
}
void common_speculative_send_accepted(struct common_speculative * spec, const uint16_t n_accepted) {
// use new function to update the ngram map statistics.
common_ngram_map_send_accepted(spec->map, n_accepted);
common_speculative_impl * impl = spec->curr_impl;
if (impl != nullptr) {
if (n_accepted > 0) {
impl->drafts_accepted_count++;
impl->drafts_accepted_tokens += n_accepted;
}
if (impl->type == COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K ||
impl->type == COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V) {
common_ngram_map_send_accepted(spec->map, n_accepted);
}
}
}
// self-speculative decoding
@@ -433,14 +643,6 @@ llama_tokens common_speculative_gen_self_draft(
common_speculative * spec,
const llama_tokens & tokens, llama_token sampled) {
common_ngram_map & map = spec->map;
if (spec->self_mode != 1) {
// Use common_ngram_map_draft to generate a draft from the current context.
llama_tokens draft_tokens;
common_ngram_map_draft(map, tokens, sampled, draft_tokens);
return draft_tokens;
}
// Simple implementation of self-speculative decoding without draft model, without ngram-map.
//
common_speculative_self & self_state = spec->self_state;
@@ -513,7 +715,16 @@ llama_tokens common_speculative_gen_self_draft(
}
void common_speculative_print_stats(const struct common_speculative * spec) {
if (spec->map.drafts_generated_tokens > 0) { // only print if we have some stats
common_ngram_map_print_stats(spec->map);
if (spec == nullptr) {
return;
}
for (const auto & impl : spec->impls) {
LOG_INF("statistics %s: #calls = %zu, #gen drafts = %zu, #acc drafts = %zu, #gen tokens = %zu, #acc tokens = %zu\n",
common_speculative_type_to_str(impl.type).c_str(),
impl.drafts_call_count,
impl.drafts_generated_count,
impl.drafts_accepted_count,
impl.drafts_generated_tokens,
impl.drafts_accepted_tokens);
}
}