Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 01:04:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

common: add enum common_speculative_type

Browse Source
This commit is contained in:
Sascha Rogmann
2026-01-18 18:45:10 +01:00
parent 456268fa7f
commit b38eb5907c
8 changed files with 359 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+45 -35
View File
@@ -6,6 +6,7 @@
#include "json-schema-to-grammar.h"
#include "log.h"
#include "sampling.h"
#include "speculative.h"
#include "preset.h"
// fix problem with std::min and std::max
@@ -625,6 +626,26 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
if (!params.speculative.tensor_buft_overrides.empty()) {
params.speculative.tensor_buft_overrides.push_back({nullptr, nullptr});
}
if (!params.speculative.model.path.empty()) {
bool found_draft = false;
bool found_eagle3 = false;
for (const auto & config : params.speculative.configs) {
if (config.type == COMMON_SPECULATIVE_TYPE_DRAFT) {
found_draft = true;
}
if (config.type == COMMON_SPECULATIVE_TYPE_EAGLE3) {
found_eagle3 = true;
break;
}
}
if (!found_draft) {
params.speculative.configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT));
}
// TODO PR-18039: if params.speculative.eagle3
//if (!found_eagle3) {
// params.speculative.configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT));
//}
}
if (!params.chat_template.empty() && !common_chat_verify_template(params.chat_template, params.use_jinja)) {
throw std::runtime_error(string_format(
@@ -3393,45 +3414,34 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
}
).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
add_opt(common_arg(
{"--spec-self"}, "N",
"mode of self-speculation without a draft model: disabled(0), fixed(1), keys-only(2), key-values(3) (default: %d)\n",
[](common_params & params, int value) {
if (value < 0 || value > 3) {
throw std::invalid_argument("invalid value");
}
params.speculative.self_mode = value;
}
).set_examples({LLAMA_EXAMPLE_SERVER}));
add_opt(common_arg(
{"--spec-self-config"}, "N0,N1,N2,...",
"speculative self decoding config: ngram size (key), mgram size (value), check rate, min hits (default: %d,%d,%d,%d)",
{"--spec-config"}, "SPECULATIVE_CONFIG",
string_format("list of speculative decoding types, separated by ';', optionally followed by a colon and a comma-separated list of key=value pairs\n(types: %s)\n", common_speculative_type_name_str().c_str()),
[](common_params & params, const std::string & value) {
std::string arg_next = value;
// split string by , and /
const std::regex regex{ R"([,/]+)" };
std::sregex_token_iterator it{ arg_next.begin(), arg_next.end(), regex, -1 };
std::vector<std::string> split_arg{ it, {} };
if (split_arg.size() > 4) {
throw std::invalid_argument(
string_format("got %d input configs, but self-speculative decoding config require at most 4 values", (int)split_arg.size())
);
}
for (size_t i = 0; i < split_arg.size(); ++i) {
int val = std::stoi(split_arg[i]);
if (i == 0 && (val < 1 || val > 255)) {
throw std::invalid_argument("ngram size must be between 1 and 255");
const auto config_strings = string_split<std::string>(value, ';');
for (const auto & config_string : config_strings) {
const auto parts = string_split<std::string>(config_string, ':');
if (parts.size() < 1 || parts.size() > 2) {
throw std::invalid_argument("invalid speculative decoding config");
}
if (i == 1 && (val < 1 || val > 255)) {
throw std::invalid_argument("mgram size must be between 1 and 255");
const auto type_str = parts[0];
const auto type = common_speculative_type_from_name(type_str);
if (type == COMMON_SPECULATIVE_TYPE_COUNT) {
throw std::invalid_argument(string_format("unknown speculative decoding type: %s", type_str.c_str()));
}
if (i == 2 && val == 0) {
throw std::invalid_argument("check rate must be greater than 0");
common_speculative_config spec_config = {type};
if (parts.size() == 2) {
const auto key_value_pairs = string_split<std::string>(parts[1], ',');
for (const auto & key_value_pair : key_value_pairs) {
const auto key_value = string_split<std::string>(key_value_pair, '=');
if (key_value.size() != 2) {
throw std::invalid_argument("invalid key=value pair");
}
const auto & key = key_value[0];
const auto & value = key_value[1];
spec_config.config[key] = value;
}
}
if (i == 3 && (val < 1 || val > 255)) {
throw std::invalid_argument("min hits must be between 1 and 255");
}
params.speculative.self_cfg[i] = (uint16_t) val;
params.speculative.configs.push_back(spec_config);
}
}
).set_examples({LLAMA_EXAMPLE_SERVER}));
common/common.h
+21 -2
View File
@@ -164,6 +164,17 @@ enum common_params_sampling_config : uint64_t {
COMMON_PARAMS_SAMPLING_CONFIG_MIROSTAT_ETA = 1 << 11,
};
enum common_speculative_type {
COMMON_SPECULATIVE_TYPE_NONE, // no speculative decoding
COMMON_SPECULATIVE_TYPE_DRAFT, // draft model
COMMON_SPECULATIVE_TYPE_EAGLE3, // eagle draft model
COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys only
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values
COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, // self-speculative decoding with 3-level n-gram cache
COMMON_SPECULATIVE_TYPE_COUNT // number of types, unknown type
};
// sampling parameters
struct common_params_sampling {
@@ -242,6 +253,14 @@ struct common_params_model {
std::string name = ""; // in format <user>/<model>[:<tag>] (tag is optional) // NOLINT
};
struct common_speculative_config {
common_speculative_type type;
std::map<std::string, std::string> config; // map of incubative options (not yet in common_params)
common_speculative_config(common_speculative_type t,
const std::map<std::string, std::string>& c = {}) : type(t), config(c) {}
};
struct common_params_speculative {
std::vector<ggml_backend_dev_t> devices; // devices to use for offloading
@@ -251,8 +270,6 @@ struct common_params_speculative {
int32_t n_gpu_layers = -1; // number of layers to store in VRAM for the draft model (-1 - use default)
float p_split = 0.1f; // speculative decoding split probability
float p_min = 0.75f; // minimum speculative decoding probability (greedy)
int32_t self_mode = 0; // mode of self-speculative decoding without draft model (default: 0 = off)
std::vector<uint16_t> self_cfg = {12, 48, 2, 1}; // self-speculative decoding config (n-gram size, m-gram size, check rate, min hits)
std::vector<std::pair<std::string, std::string>> replacements; // main to speculative model replacements
std::vector<llama_model_tensor_buft_override> tensor_buft_overrides;
@@ -263,6 +280,8 @@ struct common_params_speculative {
struct cpu_params cpuparams_batch;
struct common_params_model model;
std::vector<common_speculative_config> configs = {}; // list of speculative configs to try
};
struct common_params_vocoder {
common/ngram-map.cpp
-21
View File
@@ -117,7 +117,6 @@ void common_ngram_map_draft(common_ngram_map & map,
map.last_draft_created = false;
map.last_draft_key_idx = key_offset;
map.last_draft_value_idx = 0; // value 0 is used for simple mode
map.drafts_generated_tokens += draft.size();
return;
}
@@ -236,7 +235,6 @@ void common_ngram_map_draft(common_ngram_map & map,
map.last_draft_created = true;
map.last_draft_key_idx = key_offset;
map.last_draft_value_idx = slot_max; // value used for draft generation.
map.drafts_generated_tokens += draft.size();
}
void common_ngram_map_send_accepted(common_ngram_map & map, uint16_t n_accepted) {
@@ -257,25 +255,6 @@ void common_ngram_map_send_accepted(common_ngram_map & map, uint16_t n_accepted)
LOG_INF("common_ngram_map_send_accepted: n_accepted = %d, prev value_num = %d\n",
n_accepted, curr_value.n_accepted);
curr_value.n_accepted = n_accepted;
// draft statistics update
if (n_accepted > 0) {
map.drafts_accepted_count++;
} else {
map.drafts_rejected_count++;
}
map.drafts_accepted_tokens += n_accepted;
}
// Display statistics of the ngram map.
void common_ngram_map_print_stats(const common_ngram_map & map) {
LOG_INF("ngram map: size_key = %d, size_value = %d, key_only = %s, min_hits = %d\n",
map.size_key, map.size_value,
map.key_only ? "true" : "false",
map.min_hits);
LOG_INF("drafts_accepted_count = %zu, drafts_rejected_count = %zu, drafts_generated_tokens = %zu, drafts_accepted_tokens = %zu\n",
map.drafts_accepted_count, map.drafts_rejected_count,
map.drafts_generated_tokens, map.drafts_accepted_tokens);
}
// Helper functions.
common/ngram-map.h
-8
View File
@@ -45,11 +45,6 @@ struct common_ngram_map {
: size_key(sz_key), size_value(sz_value), key_only(only_keys), keys(std::vector<common_ngram_map_key>{}),
check_rate(check_rate), min_hits(min_hits) {}
size_t drafts_accepted_count = 0; // number of drafts accepted by the target model.
size_t drafts_rejected_count = 0; // number of drafts rejected by the target model.
size_t drafts_generated_tokens = 0; // number of tokens generated by this ngram map.
size_t drafts_accepted_tokens = 0; // number of tokens accepted by the target model.
bool last_draft_created = false; // true if a draft was created at last call.
size_t last_draft_key_idx = 0; // index of last key used for draft generation.
uint16_t last_draft_value_idx = 0; // index of last value used for draft generation.
@@ -69,6 +64,3 @@ void common_ngram_map_draft(
// Update the statistics of a value after a draft was accepted.
void common_ngram_map_send_accepted(common_ngram_map & map, uint16_t n_accepted);
// Display statistics of the ngram map.
void common_ngram_map_print_stats(const common_ngram_map & map);
common/speculative.cpp
+243 -32
View File
@@ -14,6 +14,26 @@
#define SPEC_VOCAB_MAX_SIZE_DIFFERENCE 128
#define SPEC_VOCAB_CHECK_START_TOKEN_ID 5
const std::vector<enum common_speculative_type> common_speculative_types = {
COMMON_SPECULATIVE_TYPE_NONE,
COMMON_SPECULATIVE_TYPE_DRAFT,
COMMON_SPECULATIVE_TYPE_EAGLE3,
COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE,
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K,
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V,
COMMON_SPECULATIVE_TYPE_NGRAM_CACHE
};
const std::map<std::string, enum common_speculative_type> common_speculative_type_from_name_map = {
{"none", COMMON_SPECULATIVE_TYPE_NONE},
{"draft", COMMON_SPECULATIVE_TYPE_DRAFT},
{"eagle3", COMMON_SPECULATIVE_TYPE_EAGLE3},
{"ngram_simple", COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE},
{"ngram_map_k", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K},
{"ngram_map_k4v", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V},
{"ngram_cache", COMMON_SPECULATIVE_TYPE_NGRAM_CACHE}
};
struct common_speculative_self {
uint16_t size_ngram = 12; // size of n-grams to lookup in self-mode
uint16_t size_mgram = 48; // size of m-grams to draft in self-mode
@@ -21,6 +41,16 @@ struct common_speculative_self {
size_t idx_last_check = 0; // index of last check in context history
};
struct common_speculative_impl {
const enum common_speculative_type type;
size_t drafts_call_count = 0; // number of times this implementation was called.
size_t drafts_generated_count = 0; // number of times a draft or part was generated by this implementation.
size_t drafts_accepted_count = 0; // number of times a draft or part was accepted by the target model.
size_t drafts_generated_tokens = 0; // number of tokens generated by this implementation.
size_t drafts_accepted_tokens = 0; // number of tokens accepted by the target model.
};
struct common_speculative {
struct llama_context * ctx_tgt; // only used for retokenizing from ctx_dft
struct llama_context * ctx_dft;
@@ -31,29 +61,133 @@ struct common_speculative {
bool vocab_dft_compatible = true; // whether retokenization is needed
std::map<std::string, std::string> tgt_dft_replacements = {};
const uint16_t self_mode = 0; // 0: off, 1: self speculative, 2: n-grams (keys) only, 3: n-grams/m-grams (key-values)
common_ngram_map map; // draft ngram map for speculative decoding without draft model
common_speculative_self self_state; // state of self-speculation (simple implementation, not ngram-map)
std::vector<common_speculative_impl> impls; // list of implementations to use and their statistics
common_speculative_impl * curr_impl = nullptr; // current implementation in use (for stats)
};
common_ngram_map get_common_ngram_map(std::vector<common_speculative_config> configs);
common_ngram_map get_common_ngram_map(std::vector<common_speculative_config> configs) {
for (const auto & config : configs) {
switch (config.type) {
case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE:
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K:
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: {
// create common_ngram_map from config.config
// compute size_key, size_value, key_only, check_rate, min_hits from config.config
uint16_t size_key = 12;
uint16_t size_value = 48;
bool key_only = false;
uint16_t check_rate = 3;
uint16_t min_hits = 1;
const std::map<std::string, std::string> & cfg = config.config;
// check for key "size_ngram" in cfg
if (cfg.find("size_ngram") != cfg.end()) {
size_key = std::stoi(cfg.at("size_ngram"));
if (size_key < 1 || size_key > 1024) {
throw std::invalid_argument("size_ngram must be between 1 and 1024");
}
}
// check for key "size_mgram" in cfg
if (cfg.find("size_mgram") != cfg.end()) {
size_value = std::stoi(cfg.at("size_mgram"));
if (size_value < 1 || size_value > 1024) {
throw std::invalid_argument("size_mgram must be between 1 and 1024");
}
}
// check for key "key_only" in cfg
if (cfg.find("key_only") != cfg.end()) {
// key_onle == true, if cfg.at("key_only") has value "true".
key_only = (cfg.at("key_only") == "true");
}
// check for key "check_rate" in cfg
if (cfg.find("check_rate") != cfg.end()) {
check_rate = std::stoi(cfg.at("check_rate"));
if (check_rate < 1 || check_rate > 1024) {
throw std::invalid_argument("check_rate must be between 1 and 1024");
}
}
// check for key "min_hits" in cfg
if (cfg.find("min_hits") != cfg.end()) {
min_hits = std::stoi(cfg.at("min_hits"));
if (min_hits < 1 || min_hits > 1024) {
throw std::invalid_argument("min_hits must be between 1 and 1024");
}
}
return common_ngram_map(size_key, size_value, key_only, check_rate, min_hits);
break;
}
case COMMON_SPECULATIVE_TYPE_NONE:
case COMMON_SPECULATIVE_TYPE_DRAFT:
case COMMON_SPECULATIVE_TYPE_EAGLE3:
case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE:
break;
case COMMON_SPECULATIVE_TYPE_COUNT:
break;
}
}
return common_ngram_map(12, 48, false, 3, 1); // default fallback
}
std::string common_speculative_type_name_str() {
std::string result = "";
for (size_t i = 0; i < common_speculative_types.size(); i++) {
if (i > 0) {
result += ", ";
}
result += common_speculative_type_to_str(common_speculative_types[i]);
}
return result;
}
std::string common_speculative_type_to_str(enum common_speculative_type type) {
switch (type) {
case COMMON_SPECULATIVE_TYPE_NONE: return "none";
case COMMON_SPECULATIVE_TYPE_DRAFT: return "draft";
case COMMON_SPECULATIVE_TYPE_EAGLE3: return "eagle3";
case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: return "ngram_simple";
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: return "ngram_map_k";
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: return "ngram_map_k4v";
case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE: return "ngram_cache";
default: return "unknown";
}
}
enum common_speculative_type common_speculative_type_from_name(const std::string & name) {
const auto it = common_speculative_type_from_name_map.find(name);
if (it == common_speculative_type_from_name_map.end()) {
return COMMON_SPECULATIVE_TYPE_COUNT;
}
return it->second;
}
struct common_speculative * common_speculative_init(
struct llama_context * ctx_tgt,
struct llama_context * ctx_dft,
uint16_t self_mode, // 0: off, 1: self speculative, 2: n-grams (keys) only, 3: n-grams/m-grams (key-values)
const std::vector<uint16_t> self_cfg // ngram size, mgram size, keys only (0|1), min hits
const std::vector<common_speculative_config> configs
) {
uint16_t ngram_size_key = self_cfg.size() >= 1 ? self_cfg[0] : 12;
uint16_t mgram_size_value = self_cfg.size() >= 2 ? self_cfg[1] : 48;
uint16_t check_rate = self_cfg.size() >= 3 ? self_cfg[2] : 3;
bool key_only = (self_mode != 3);
uint16_t min_hits = self_cfg.size() >= 4 ? self_cfg[3] : 1;
common_ngram_map ngram_map = common_ngram_map(ngram_size_key, mgram_size_value, key_only, check_rate, min_hits);
common_ngram_map ngram_map = get_common_ngram_map(configs);
uint16_t ngram_size_key = ngram_map.size_key;
uint16_t mgram_size_value = ngram_map.size_value;
uint16_t check_rate = ngram_map.check_rate;
common_speculative_self self_state = common_speculative_self{
/* .size_ngram = */ ngram_size_key,
/* .size_mgram = */ mgram_size_value,
/* .check_rate = */ check_rate,
/* .idx_last_check = */ 0,
};
std::vector<common_speculative_impl> implementations = {};
LOG_INF("common_speculative_init: configs.size = %zu\n", configs.size());
for (const auto & config : configs) {
LOG_INF("common_speculative_init: adding implementation %s\n", common_speculative_type_to_str(config.type).c_str());
implementations.push_back(common_speculative_impl{config.type});
}
auto * result = new common_speculative {
/* .ctx_tgt = */ ctx_tgt,
/* .ctx_dft = */ ctx_dft,
@@ -62,9 +196,9 @@ struct common_speculative * common_speculative_init(
/* .prompt_dft = */ {},
/* .vocab_dft_compatible = */ false,
/* .tgt_dft_replacements = */ {},
/* .self_mode = */ self_mode,
/* .map = */ ngram_map,
/* .self_state = */ self_state
/* .self_state = */ self_state,
/* .impls = */ implementations
};
LOG_INF("common_speculative_init: created speculative decoder, map.n = %d\n", result->map.size_key);
@@ -218,6 +352,12 @@ static std::string replace_to_tgt(
return result;
}
llama_tokens common_speculative_use_draft_model(
struct common_speculative * spec,
struct common_speculative_params params,
const llama_tokens & prompt_tgt_main_model, // specified in target model vocab
llama_token id_last);
llama_tokens common_speculative_gen_self_draft(
common_speculative * spec,
const llama_tokens & tokens, llama_token sampled);
@@ -227,17 +367,78 @@ llama_tokens common_speculative_gen_draft(
struct common_speculative_params params,
const llama_tokens & prompt_tgt_main_model, // specified in target model vocab
llama_token id_last) {
if (spec->self_mode) {
// Look in the current context for a n-gram and return the following tokens as the draft.
llama_tokens draft_self = common_speculative_gen_self_draft(spec,
prompt_tgt_main_model, id_last);
if (!draft_self.empty()) {
return draft_self;
llama_tokens result = {};
spec->curr_impl = nullptr; // reset current implementation
for (auto & impl : spec->impls) {
impl.drafts_call_count++;
// LOG name and call_count
switch (impl.type) {
case COMMON_SPECULATIVE_TYPE_NONE:
{
break;
}
case COMMON_SPECULATIVE_TYPE_DRAFT:
{
// Create a draft using a draft model.
result = common_speculative_use_draft_model(spec, params, prompt_tgt_main_model, id_last);
break;
}
case COMMON_SPECULATIVE_TYPE_EAGLE3:
{
// Work in progress: https://github.com/ggml-org/llama.cpp/pull/18039
break;
}
case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE:
{
// Use common_ngram_map_draft to generate a draft from the current context.
result = common_speculative_gen_self_draft(spec, prompt_tgt_main_model, id_last);
break;
}
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K:
{
// Use common_ngram_map_draft to generate a draft from the current context.
common_ngram_map_draft(spec->map, prompt_tgt_main_model, id_last, result);
break;
}
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V:
{
// Use common_ngram_map_draft to generate a draft from the current context.
common_ngram_map_draft(spec->map, prompt_tgt_main_model, id_last, result);
break;
}
case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE:
{
// TODO call common/ngram-cache.cpp
break;
}
case COMMON_SPECULATIVE_TYPE_COUNT:
{
GGML_ABORT("invalid speculative type COUNT");
break;
}
}
if (!result.empty()) {
LOG_DBG("%s: called impl %s, hist size = %zu, call_count = %zu, gen = %zu\n", __func__,
common_speculative_type_to_str(impl.type).c_str(),
prompt_tgt_main_model.size(),
impl.drafts_call_count, result.size());
spec->curr_impl = &impl; // set current implementation for stats
impl.drafts_generated_count++;
impl.drafts_generated_tokens += result.size();
break; // We have a draft, so break out of the loop and return it.
}
}
if (spec == nullptr || spec->ctx_dft == nullptr) {
return {}; // no draft model, return
}
return result;
}
llama_tokens common_speculative_use_draft_model(
struct common_speculative * spec,
struct common_speculative_params params,
const llama_tokens & prompt_tgt_main_model, // specified in target model vocab
llama_token id_last) {
auto & batch = spec->batch;
auto & ctx_tgt = spec->ctx_tgt;
@@ -413,8 +614,17 @@ llama_tokens common_speculative_gen_draft(
}
void common_speculative_send_accepted(struct common_speculative * spec, const uint16_t n_accepted) {
// use new function to update the ngram map statistics.
common_ngram_map_send_accepted(spec->map, n_accepted);
common_speculative_impl * impl = spec->curr_impl;
if (impl != nullptr) {
if (n_accepted > 0) {
impl->drafts_accepted_count++;
impl->drafts_accepted_tokens += n_accepted;
}
if (impl->type == COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K ||
impl->type == COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V) {
common_ngram_map_send_accepted(spec->map, n_accepted);
}
}
}
// self-speculative decoding
@@ -433,14 +643,6 @@ llama_tokens common_speculative_gen_self_draft(
common_speculative * spec,
const llama_tokens & tokens, llama_token sampled) {
common_ngram_map & map = spec->map;
if (spec->self_mode != 1) {
// Use common_ngram_map_draft to generate a draft from the current context.
llama_tokens draft_tokens;
common_ngram_map_draft(map, tokens, sampled, draft_tokens);
return draft_tokens;
}
// Simple implementation of self-speculative decoding without draft model, without ngram-map.
//
common_speculative_self & self_state = spec->self_state;
@@ -513,7 +715,16 @@ llama_tokens common_speculative_gen_self_draft(
}
void common_speculative_print_stats(const struct common_speculative * spec) {
if (spec->map.drafts_generated_tokens > 0) { // only print if we have some stats
common_ngram_map_print_stats(spec->map);
if (spec == nullptr) {
return;
}
for (const auto & impl : spec->impls) {
LOG_INF("statistics %s: #calls = %zu, #gen drafts = %zu, #acc drafts = %zu, #gen tokens = %zu, #acc tokens = %zu\n",
common_speculative_type_to_str(impl.type).c_str(),
impl.drafts_call_count,
impl.drafts_generated_count,
impl.drafts_accepted_count,
impl.drafts_generated_tokens,
impl.drafts_accepted_tokens);
}
}
common/speculative.h
+10 -2
View File
@@ -12,11 +12,19 @@ struct common_speculative_params {
float p_min = 0.75f; // min probability required to accept a token in the draft
};
// comma separated list of all types
std::string common_speculative_type_name_str();
// convert string to type
enum common_speculative_type common_speculative_type_from_name(const std::string & name);
// convert type to string
std::string common_speculative_type_to_str(enum common_speculative_type type);
struct common_speculative * common_speculative_init(
struct llama_context * ctx_tgt,
struct llama_context * ctx_dft,
const uint16_t self_mode = 0, // 0: off, 1: self speculative, 2: n-grams (keys) only, 3: n-grams/m-grams (key-values)
const std::vector<uint16_t> self_cfg = { 12, 48, 3, 1 } // ngram size, mgram size, check rate, min hits
const std::vector<common_speculative_config> configs = {} // incubator config (options not yet in common_params)
);
void common_speculative_free(struct common_speculative * spec);
tools/server/server-context.cpp
+6 -8
View File
@@ -260,7 +260,7 @@ struct server_slot {
// Checks if a draft model is active or self-speculation using context-tokens
bool can_speculate() const {
return ctx_dft || task->params.speculative.self_mode;
return task->params.speculative.configs.size() > 0;
}
void add_token(const completion_token_output & token) {
@@ -397,8 +397,8 @@ struct server_slot {
"draft acceptance rate = %0.5f (%5d accepted / %5d generated)\n",
draft_ratio, n_draft_accepted, n_draft_total
);
common_speculative_print_stats(spec);
}
common_speculative_print_stats(spec);
}
json to_json(bool only_metrics = false) const {
@@ -776,8 +776,7 @@ private:
}
slot.spec = common_speculative_init(slot.ctx, slot.ctx_dft,
params_base.speculative.self_mode,
params_base.speculative.self_cfg);
params_base.speculative.configs);
if (slot.spec == nullptr) {
SRV_ERR("%s", "failed to create speculator\n");
return false;
@@ -785,11 +784,10 @@ private:
for (auto & pair : params_base.speculative.replacements) {
common_speculative_add_replacement_tgt_dft(slot.spec, pair.first.c_str(), pair.second.c_str());
}
} else if (params_base.speculative.self_mode) {
SLT_INF(slot, "init spec for self-speculative decoding, slot %d\n", i);
} else if (params_base.speculative.configs.size() > 0) {
SLT_INF(slot, "init spec for speculative decoding without draft model, slot %d\n", i);
slot.spec = common_speculative_init(nullptr, nullptr,
params_base.speculative.self_mode,
params_base.speculative.self_cfg);
params_base.speculative.configs);
}
SLT_INF(slot, "new slot, n_ctx = %d\n", slot.n_ctx);
tools/server/server-task.cpp
+34 -2
View File
@@ -5,6 +5,7 @@
#include "llama.h"
#include "chat.h"
#include "sampling.h"
#include "speculative.h"
#include "json-schema-to-grammar.h"
using json = nlohmann::ordered_json;
@@ -237,8 +238,39 @@ task_params server_task::params_from_json_cmpl(
params.speculative.n_min = json_value(data, "speculative.n_min", defaults.speculative.n_min);
params.speculative.n_max = json_value(data, "speculative.n_max", defaults.speculative.n_max);
params.speculative.p_min = json_value(data, "speculative.p_min", defaults.speculative.p_min);
params.speculative.self_mode = json_value(data, "speculative.self_mode", defaults.speculative.self_mode);
params.speculative.self_cfg = json_value(data, "speculative.self_cfg", defaults.speculative.self_cfg);
//params.speculative.self_mode = json_value(data, "speculative.self_mode", defaults.speculative.self_mode);
//params.speculative.self_cfg = json_value(data, "speculative.self_cfg", defaults.speculative.self_cfg);
// Set params.speculative.configs. Use json-array "speculative.configs" if provided in data, otherwise use {}
{
params.speculative.configs = defaults.speculative.configs;
const auto & configs = data.find("speculative.configs");
if (configs != data.end() && configs->is_array()) {
params.speculative.configs.clear();
for (const auto & config : *configs) {
if (config.is_object()) {
// config should have keys "type" and "config" (optional)
const auto & type = config.find("type");
if (type != config.end() && type->is_string()) {
const auto type_name = type->get<std::string>();
const auto type_enum = common_speculative_type_from_name(type_name);
if (type_enum != COMMON_SPECULATIVE_TYPE_COUNT) {
common_speculative_config cfg(type_enum);
const auto & cfg_map = config.find("config");
if (cfg_map != config.end() && cfg_map->is_object()) {
for (const auto & [key, value] : cfg_map->items()) {
cfg.config[key] = value.get<std::string>();
}
}
params.speculative.configs.push_back(cfg);
} else {
SRV_WRN("Unknown speculative type: %s\n", type_name.c_str());
}
}
}
}
}
}
params.speculative.n_min = std::min(params.speculative.n_max, params.speculative.n_min);
params.speculative.n_min = std::max(params.speculative.n_min, 0);