diff --git a/common/arg.cpp b/common/arg.cpp index 8fc15261a5..a258d1a494 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -6,6 +6,7 @@ #include "json-schema-to-grammar.h" #include "log.h" #include "sampling.h" +#include "speculative.h" #include "preset.h" // fix problem with std::min and std::max @@ -625,6 +626,26 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context if (!params.speculative.tensor_buft_overrides.empty()) { params.speculative.tensor_buft_overrides.push_back({nullptr, nullptr}); } + if (!params.speculative.model.path.empty()) { + bool found_draft = false; + bool found_eagle3 = false; + for (const auto & config : params.speculative.configs) { + if (config.type == COMMON_SPECULATIVE_TYPE_DRAFT) { + found_draft = true; + } + if (config.type == COMMON_SPECULATIVE_TYPE_EAGLE3) { + found_eagle3 = true; + break; + } + } + if (!found_draft) { + params.speculative.configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT)); + } + // TODO PR-18039: if params.speculative.eagle3 + //if (!found_eagle3) { + // params.speculative.configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT)); + //} + } if (!params.chat_template.empty() && !common_chat_verify_template(params.chat_template, params.use_jinja)) { throw std::runtime_error(string_format( @@ -3393,45 +3414,34 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( - {"--spec-self"}, "N", - "mode of self-speculation without a draft model: disabled(0), fixed(1), keys-only(2), key-values(3) (default: %d)\n", - [](common_params & params, int value) { - if (value < 0 || value > 3) { - throw std::invalid_argument("invalid value"); - } - params.speculative.self_mode = value; - } - ).set_examples({LLAMA_EXAMPLE_SERVER})); - add_opt(common_arg( - {"--spec-self-config"}, "N0,N1,N2,...", - "speculative self decoding config: ngram size (key), mgram size (value), check rate, min hits (default: %d,%d,%d,%d)", + {"--spec-config"}, "SPECULATIVE_CONFIG", + string_format("list of speculative decoding types, separated by ';', optionally followed by a colon and a comma-separated list of key=value pairs\n(types: %s)\n", common_speculative_type_name_str().c_str()), [](common_params & params, const std::string & value) { - std::string arg_next = value; - - // split string by , and / - const std::regex regex{ R"([,/]+)" }; - std::sregex_token_iterator it{ arg_next.begin(), arg_next.end(), regex, -1 }; - std::vector split_arg{ it, {} }; - if (split_arg.size() > 4) { - throw std::invalid_argument( - string_format("got %d input configs, but self-speculative decoding config require at most 4 values", (int)split_arg.size()) - ); - } - for (size_t i = 0; i < split_arg.size(); ++i) { - int val = std::stoi(split_arg[i]); - if (i == 0 && (val < 1 || val > 255)) { - throw std::invalid_argument("ngram size must be between 1 and 255"); + const auto config_strings = string_split(value, ';'); + for (const auto & config_string : config_strings) { + const auto parts = string_split(config_string, ':'); + if (parts.size() < 1 || parts.size() > 2) { + throw std::invalid_argument("invalid speculative decoding config"); } - if (i == 1 && (val < 1 || val > 255)) { - throw std::invalid_argument("mgram size must be between 1 and 255"); + const auto type_str = parts[0]; + const auto type = common_speculative_type_from_name(type_str); + if (type == COMMON_SPECULATIVE_TYPE_COUNT) { + throw std::invalid_argument(string_format("unknown speculative decoding type: %s", type_str.c_str())); } - if (i == 2 && val == 0) { - throw std::invalid_argument("check rate must be greater than 0"); + common_speculative_config spec_config = {type}; + if (parts.size() == 2) { + const auto key_value_pairs = string_split(parts[1], ','); + for (const auto & key_value_pair : key_value_pairs) { + const auto key_value = string_split(key_value_pair, '='); + if (key_value.size() != 2) { + throw std::invalid_argument("invalid key=value pair"); + } + const auto & key = key_value[0]; + const auto & value = key_value[1]; + spec_config.config[key] = value; + } } - if (i == 3 && (val < 1 || val > 255)) { - throw std::invalid_argument("min hits must be between 1 and 255"); - } - params.speculative.self_cfg[i] = (uint16_t) val; + params.speculative.configs.push_back(spec_config); } } ).set_examples({LLAMA_EXAMPLE_SERVER})); diff --git a/common/common.h b/common/common.h index be38ef1185..8910a1c525 100644 --- a/common/common.h +++ b/common/common.h @@ -164,6 +164,17 @@ enum common_params_sampling_config : uint64_t { COMMON_PARAMS_SAMPLING_CONFIG_MIROSTAT_ETA = 1 << 11, }; +enum common_speculative_type { + COMMON_SPECULATIVE_TYPE_NONE, // no speculative decoding + COMMON_SPECULATIVE_TYPE_DRAFT, // draft model + COMMON_SPECULATIVE_TYPE_EAGLE3, // eagle draft model + COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding + COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys only + COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values + COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, // self-speculative decoding with 3-level n-gram cache + COMMON_SPECULATIVE_TYPE_COUNT // number of types, unknown type +}; + // sampling parameters struct common_params_sampling { @@ -242,6 +253,14 @@ struct common_params_model { std::string name = ""; // in format /[:] (tag is optional) // NOLINT }; +struct common_speculative_config { + common_speculative_type type; + std::map config; // map of incubative options (not yet in common_params) + + common_speculative_config(common_speculative_type t, + const std::map& c = {}) : type(t), config(c) {} +}; + struct common_params_speculative { std::vector devices; // devices to use for offloading @@ -251,8 +270,6 @@ struct common_params_speculative { int32_t n_gpu_layers = -1; // number of layers to store in VRAM for the draft model (-1 - use default) float p_split = 0.1f; // speculative decoding split probability float p_min = 0.75f; // minimum speculative decoding probability (greedy) - int32_t self_mode = 0; // mode of self-speculative decoding without draft model (default: 0 = off) - std::vector self_cfg = {12, 48, 2, 1}; // self-speculative decoding config (n-gram size, m-gram size, check rate, min hits) std::vector> replacements; // main to speculative model replacements std::vector tensor_buft_overrides; @@ -263,6 +280,8 @@ struct common_params_speculative { struct cpu_params cpuparams_batch; struct common_params_model model; + + std::vector configs = {}; // list of speculative configs to try }; struct common_params_vocoder { diff --git a/common/ngram-map.cpp b/common/ngram-map.cpp index 487f48362f..f79066cae0 100644 --- a/common/ngram-map.cpp +++ b/common/ngram-map.cpp @@ -117,7 +117,6 @@ void common_ngram_map_draft(common_ngram_map & map, map.last_draft_created = false; map.last_draft_key_idx = key_offset; map.last_draft_value_idx = 0; // value 0 is used for simple mode - map.drafts_generated_tokens += draft.size(); return; } @@ -236,7 +235,6 @@ void common_ngram_map_draft(common_ngram_map & map, map.last_draft_created = true; map.last_draft_key_idx = key_offset; map.last_draft_value_idx = slot_max; // value used for draft generation. - map.drafts_generated_tokens += draft.size(); } void common_ngram_map_send_accepted(common_ngram_map & map, uint16_t n_accepted) { @@ -257,25 +255,6 @@ void common_ngram_map_send_accepted(common_ngram_map & map, uint16_t n_accepted) LOG_INF("common_ngram_map_send_accepted: n_accepted = %d, prev value_num = %d\n", n_accepted, curr_value.n_accepted); curr_value.n_accepted = n_accepted; - - // draft statistics update - if (n_accepted > 0) { - map.drafts_accepted_count++; - } else { - map.drafts_rejected_count++; - } - map.drafts_accepted_tokens += n_accepted; -} - -// Display statistics of the ngram map. -void common_ngram_map_print_stats(const common_ngram_map & map) { - LOG_INF("ngram map: size_key = %d, size_value = %d, key_only = %s, min_hits = %d\n", - map.size_key, map.size_value, - map.key_only ? "true" : "false", - map.min_hits); - LOG_INF("drafts_accepted_count = %zu, drafts_rejected_count = %zu, drafts_generated_tokens = %zu, drafts_accepted_tokens = %zu\n", - map.drafts_accepted_count, map.drafts_rejected_count, - map.drafts_generated_tokens, map.drafts_accepted_tokens); } // Helper functions. diff --git a/common/ngram-map.h b/common/ngram-map.h index 478cc6d54b..6485eba186 100644 --- a/common/ngram-map.h +++ b/common/ngram-map.h @@ -45,11 +45,6 @@ struct common_ngram_map { : size_key(sz_key), size_value(sz_value), key_only(only_keys), keys(std::vector{}), check_rate(check_rate), min_hits(min_hits) {} - size_t drafts_accepted_count = 0; // number of drafts accepted by the target model. - size_t drafts_rejected_count = 0; // number of drafts rejected by the target model. - size_t drafts_generated_tokens = 0; // number of tokens generated by this ngram map. - size_t drafts_accepted_tokens = 0; // number of tokens accepted by the target model. - bool last_draft_created = false; // true if a draft was created at last call. size_t last_draft_key_idx = 0; // index of last key used for draft generation. uint16_t last_draft_value_idx = 0; // index of last value used for draft generation. @@ -69,6 +64,3 @@ void common_ngram_map_draft( // Update the statistics of a value after a draft was accepted. void common_ngram_map_send_accepted(common_ngram_map & map, uint16_t n_accepted); - -// Display statistics of the ngram map. -void common_ngram_map_print_stats(const common_ngram_map & map); diff --git a/common/speculative.cpp b/common/speculative.cpp index 6f35edf6be..f52121df59 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -14,6 +14,26 @@ #define SPEC_VOCAB_MAX_SIZE_DIFFERENCE 128 #define SPEC_VOCAB_CHECK_START_TOKEN_ID 5 +const std::vector common_speculative_types = { + COMMON_SPECULATIVE_TYPE_NONE, + COMMON_SPECULATIVE_TYPE_DRAFT, + COMMON_SPECULATIVE_TYPE_EAGLE3, + COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, + COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, + COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, + COMMON_SPECULATIVE_TYPE_NGRAM_CACHE +}; + +const std::map common_speculative_type_from_name_map = { + {"none", COMMON_SPECULATIVE_TYPE_NONE}, + {"draft", COMMON_SPECULATIVE_TYPE_DRAFT}, + {"eagle3", COMMON_SPECULATIVE_TYPE_EAGLE3}, + {"ngram_simple", COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE}, + {"ngram_map_k", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K}, + {"ngram_map_k4v", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V}, + {"ngram_cache", COMMON_SPECULATIVE_TYPE_NGRAM_CACHE} +}; + struct common_speculative_self { uint16_t size_ngram = 12; // size of n-grams to lookup in self-mode uint16_t size_mgram = 48; // size of m-grams to draft in self-mode @@ -21,6 +41,16 @@ struct common_speculative_self { size_t idx_last_check = 0; // index of last check in context history }; +struct common_speculative_impl { + const enum common_speculative_type type; + + size_t drafts_call_count = 0; // number of times this implementation was called. + size_t drafts_generated_count = 0; // number of times a draft or part was generated by this implementation. + size_t drafts_accepted_count = 0; // number of times a draft or part was accepted by the target model. + size_t drafts_generated_tokens = 0; // number of tokens generated by this implementation. + size_t drafts_accepted_tokens = 0; // number of tokens accepted by the target model. +}; + struct common_speculative { struct llama_context * ctx_tgt; // only used for retokenizing from ctx_dft struct llama_context * ctx_dft; @@ -31,29 +61,133 @@ struct common_speculative { bool vocab_dft_compatible = true; // whether retokenization is needed std::map tgt_dft_replacements = {}; - const uint16_t self_mode = 0; // 0: off, 1: self speculative, 2: n-grams (keys) only, 3: n-grams/m-grams (key-values) common_ngram_map map; // draft ngram map for speculative decoding without draft model common_speculative_self self_state; // state of self-speculation (simple implementation, not ngram-map) + + std::vector impls; // list of implementations to use and their statistics + common_speculative_impl * curr_impl = nullptr; // current implementation in use (for stats) }; +common_ngram_map get_common_ngram_map(std::vector configs); + +common_ngram_map get_common_ngram_map(std::vector configs) { + for (const auto & config : configs) { + switch (config.type) { + case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: + case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: + case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: { + // create common_ngram_map from config.config + // compute size_key, size_value, key_only, check_rate, min_hits from config.config + uint16_t size_key = 12; + uint16_t size_value = 48; + bool key_only = false; + uint16_t check_rate = 3; + uint16_t min_hits = 1; + const std::map & cfg = config.config; + // check for key "size_ngram" in cfg + if (cfg.find("size_ngram") != cfg.end()) { + size_key = std::stoi(cfg.at("size_ngram")); + if (size_key < 1 || size_key > 1024) { + throw std::invalid_argument("size_ngram must be between 1 and 1024"); + } + } + // check for key "size_mgram" in cfg + if (cfg.find("size_mgram") != cfg.end()) { + size_value = std::stoi(cfg.at("size_mgram")); + if (size_value < 1 || size_value > 1024) { + throw std::invalid_argument("size_mgram must be between 1 and 1024"); + } + } + // check for key "key_only" in cfg + if (cfg.find("key_only") != cfg.end()) { + // key_onle == true, if cfg.at("key_only") has value "true". + key_only = (cfg.at("key_only") == "true"); + } + // check for key "check_rate" in cfg + if (cfg.find("check_rate") != cfg.end()) { + check_rate = std::stoi(cfg.at("check_rate")); + if (check_rate < 1 || check_rate > 1024) { + throw std::invalid_argument("check_rate must be between 1 and 1024"); + } + } + // check for key "min_hits" in cfg + if (cfg.find("min_hits") != cfg.end()) { + min_hits = std::stoi(cfg.at("min_hits")); + if (min_hits < 1 || min_hits > 1024) { + throw std::invalid_argument("min_hits must be between 1 and 1024"); + } + } + return common_ngram_map(size_key, size_value, key_only, check_rate, min_hits); + break; + } + case COMMON_SPECULATIVE_TYPE_NONE: + case COMMON_SPECULATIVE_TYPE_DRAFT: + case COMMON_SPECULATIVE_TYPE_EAGLE3: + case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE: + break; + case COMMON_SPECULATIVE_TYPE_COUNT: + break; + } + } + + return common_ngram_map(12, 48, false, 3, 1); // default fallback +} + +std::string common_speculative_type_name_str() { + std::string result = ""; + for (size_t i = 0; i < common_speculative_types.size(); i++) { + if (i > 0) { + result += ", "; + } + result += common_speculative_type_to_str(common_speculative_types[i]); + } + return result; +} + +std::string common_speculative_type_to_str(enum common_speculative_type type) { + switch (type) { + case COMMON_SPECULATIVE_TYPE_NONE: return "none"; + case COMMON_SPECULATIVE_TYPE_DRAFT: return "draft"; + case COMMON_SPECULATIVE_TYPE_EAGLE3: return "eagle3"; + case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: return "ngram_simple"; + case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: return "ngram_map_k"; + case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: return "ngram_map_k4v"; + case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE: return "ngram_cache"; + default: return "unknown"; + } +} + +enum common_speculative_type common_speculative_type_from_name(const std::string & name) { + const auto it = common_speculative_type_from_name_map.find(name); + if (it == common_speculative_type_from_name_map.end()) { + return COMMON_SPECULATIVE_TYPE_COUNT; + } + return it->second; +} + + + struct common_speculative * common_speculative_init( struct llama_context * ctx_tgt, struct llama_context * ctx_dft, - uint16_t self_mode, // 0: off, 1: self speculative, 2: n-grams (keys) only, 3: n-grams/m-grams (key-values) - const std::vector self_cfg // ngram size, mgram size, keys only (0|1), min hits + const std::vector configs ) { - uint16_t ngram_size_key = self_cfg.size() >= 1 ? self_cfg[0] : 12; - uint16_t mgram_size_value = self_cfg.size() >= 2 ? self_cfg[1] : 48; - uint16_t check_rate = self_cfg.size() >= 3 ? self_cfg[2] : 3; - bool key_only = (self_mode != 3); - uint16_t min_hits = self_cfg.size() >= 4 ? self_cfg[3] : 1; - common_ngram_map ngram_map = common_ngram_map(ngram_size_key, mgram_size_value, key_only, check_rate, min_hits); + common_ngram_map ngram_map = get_common_ngram_map(configs); + uint16_t ngram_size_key = ngram_map.size_key; + uint16_t mgram_size_value = ngram_map.size_value; + uint16_t check_rate = ngram_map.check_rate; common_speculative_self self_state = common_speculative_self{ /* .size_ngram = */ ngram_size_key, /* .size_mgram = */ mgram_size_value, /* .check_rate = */ check_rate, /* .idx_last_check = */ 0, }; + std::vector implementations = {}; + LOG_INF("common_speculative_init: configs.size = %zu\n", configs.size()); + for (const auto & config : configs) { + LOG_INF("common_speculative_init: adding implementation %s\n", common_speculative_type_to_str(config.type).c_str()); + implementations.push_back(common_speculative_impl{config.type}); + } auto * result = new common_speculative { /* .ctx_tgt = */ ctx_tgt, /* .ctx_dft = */ ctx_dft, @@ -62,9 +196,9 @@ struct common_speculative * common_speculative_init( /* .prompt_dft = */ {}, /* .vocab_dft_compatible = */ false, /* .tgt_dft_replacements = */ {}, - /* .self_mode = */ self_mode, /* .map = */ ngram_map, - /* .self_state = */ self_state + /* .self_state = */ self_state, + /* .impls = */ implementations }; LOG_INF("common_speculative_init: created speculative decoder, map.n = %d\n", result->map.size_key); @@ -218,6 +352,12 @@ static std::string replace_to_tgt( return result; } +llama_tokens common_speculative_use_draft_model( + struct common_speculative * spec, + struct common_speculative_params params, + const llama_tokens & prompt_tgt_main_model, // specified in target model vocab + llama_token id_last); + llama_tokens common_speculative_gen_self_draft( common_speculative * spec, const llama_tokens & tokens, llama_token sampled); @@ -227,17 +367,78 @@ llama_tokens common_speculative_gen_draft( struct common_speculative_params params, const llama_tokens & prompt_tgt_main_model, // specified in target model vocab llama_token id_last) { - if (spec->self_mode) { - // Look in the current context for a n-gram and return the following tokens as the draft. - llama_tokens draft_self = common_speculative_gen_self_draft(spec, - prompt_tgt_main_model, id_last); - if (!draft_self.empty()) { - return draft_self; + llama_tokens result = {}; + + spec->curr_impl = nullptr; // reset current implementation + for (auto & impl : spec->impls) { + impl.drafts_call_count++; + // LOG name and call_count + switch (impl.type) { + case COMMON_SPECULATIVE_TYPE_NONE: + { + break; + } + case COMMON_SPECULATIVE_TYPE_DRAFT: + { + // Create a draft using a draft model. + result = common_speculative_use_draft_model(spec, params, prompt_tgt_main_model, id_last); + break; + } + case COMMON_SPECULATIVE_TYPE_EAGLE3: + { + // Work in progress: https://github.com/ggml-org/llama.cpp/pull/18039 + break; + } + case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: + { + // Use common_ngram_map_draft to generate a draft from the current context. + result = common_speculative_gen_self_draft(spec, prompt_tgt_main_model, id_last); + break; + } + case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: + { + // Use common_ngram_map_draft to generate a draft from the current context. + common_ngram_map_draft(spec->map, prompt_tgt_main_model, id_last, result); + break; + } + case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: + { + // Use common_ngram_map_draft to generate a draft from the current context. + common_ngram_map_draft(spec->map, prompt_tgt_main_model, id_last, result); + break; + } + case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE: + { + // TODO call common/ngram-cache.cpp + break; + } + case COMMON_SPECULATIVE_TYPE_COUNT: + { + GGML_ABORT("invalid speculative type COUNT"); + break; + } + } + if (!result.empty()) { + LOG_DBG("%s: called impl %s, hist size = %zu, call_count = %zu, gen = %zu\n", __func__, + common_speculative_type_to_str(impl.type).c_str(), + prompt_tgt_main_model.size(), + impl.drafts_call_count, result.size()); + spec->curr_impl = &impl; // set current implementation for stats + impl.drafts_generated_count++; + impl.drafts_generated_tokens += result.size(); + + break; // We have a draft, so break out of the loop and return it. } } - if (spec == nullptr || spec->ctx_dft == nullptr) { - return {}; // no draft model, return - } + + return result; +} + +llama_tokens common_speculative_use_draft_model( + struct common_speculative * spec, + struct common_speculative_params params, + const llama_tokens & prompt_tgt_main_model, // specified in target model vocab + llama_token id_last) { auto & batch = spec->batch; auto & ctx_tgt = spec->ctx_tgt; @@ -413,8 +614,17 @@ llama_tokens common_speculative_gen_draft( } void common_speculative_send_accepted(struct common_speculative * spec, const uint16_t n_accepted) { - // use new function to update the ngram map statistics. - common_ngram_map_send_accepted(spec->map, n_accepted); + common_speculative_impl * impl = spec->curr_impl; + if (impl != nullptr) { + if (n_accepted > 0) { + impl->drafts_accepted_count++; + impl->drafts_accepted_tokens += n_accepted; + } + if (impl->type == COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K || + impl->type == COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V) { + common_ngram_map_send_accepted(spec->map, n_accepted); + } + } } // self-speculative decoding @@ -433,14 +643,6 @@ llama_tokens common_speculative_gen_self_draft( common_speculative * spec, const llama_tokens & tokens, llama_token sampled) { - common_ngram_map & map = spec->map; - if (spec->self_mode != 1) { - // Use common_ngram_map_draft to generate a draft from the current context. - llama_tokens draft_tokens; - common_ngram_map_draft(map, tokens, sampled, draft_tokens); - return draft_tokens; - } - // Simple implementation of self-speculative decoding without draft model, without ngram-map. // common_speculative_self & self_state = spec->self_state; @@ -513,7 +715,16 @@ llama_tokens common_speculative_gen_self_draft( } void common_speculative_print_stats(const struct common_speculative * spec) { - if (spec->map.drafts_generated_tokens > 0) { // only print if we have some stats - common_ngram_map_print_stats(spec->map); + if (spec == nullptr) { + return; } + for (const auto & impl : spec->impls) { + LOG_INF("statistics %s: #calls = %zu, #gen drafts = %zu, #acc drafts = %zu, #gen tokens = %zu, #acc tokens = %zu\n", + common_speculative_type_to_str(impl.type).c_str(), + impl.drafts_call_count, + impl.drafts_generated_count, + impl.drafts_accepted_count, + impl.drafts_generated_tokens, + impl.drafts_accepted_tokens); + } } diff --git a/common/speculative.h b/common/speculative.h index 94d4538876..7fdfd6116a 100644 --- a/common/speculative.h +++ b/common/speculative.h @@ -12,11 +12,19 @@ struct common_speculative_params { float p_min = 0.75f; // min probability required to accept a token in the draft }; +// comma separated list of all types +std::string common_speculative_type_name_str(); + +// convert string to type +enum common_speculative_type common_speculative_type_from_name(const std::string & name); + +// convert type to string +std::string common_speculative_type_to_str(enum common_speculative_type type); + struct common_speculative * common_speculative_init( struct llama_context * ctx_tgt, struct llama_context * ctx_dft, - const uint16_t self_mode = 0, // 0: off, 1: self speculative, 2: n-grams (keys) only, 3: n-grams/m-grams (key-values) - const std::vector self_cfg = { 12, 48, 3, 1 } // ngram size, mgram size, check rate, min hits + const std::vector configs = {} // incubator config (options not yet in common_params) ); void common_speculative_free(struct common_speculative * spec); diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index ed4cd44ca6..46bb8194d1 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -260,7 +260,7 @@ struct server_slot { // Checks if a draft model is active or self-speculation using context-tokens bool can_speculate() const { - return ctx_dft || task->params.speculative.self_mode; + return task->params.speculative.configs.size() > 0; } void add_token(const completion_token_output & token) { @@ -397,8 +397,8 @@ struct server_slot { "draft acceptance rate = %0.5f (%5d accepted / %5d generated)\n", draft_ratio, n_draft_accepted, n_draft_total ); - common_speculative_print_stats(spec); } + common_speculative_print_stats(spec); } json to_json(bool only_metrics = false) const { @@ -776,8 +776,7 @@ private: } slot.spec = common_speculative_init(slot.ctx, slot.ctx_dft, - params_base.speculative.self_mode, - params_base.speculative.self_cfg); + params_base.speculative.configs); if (slot.spec == nullptr) { SRV_ERR("%s", "failed to create speculator\n"); return false; @@ -785,11 +784,10 @@ private: for (auto & pair : params_base.speculative.replacements) { common_speculative_add_replacement_tgt_dft(slot.spec, pair.first.c_str(), pair.second.c_str()); } - } else if (params_base.speculative.self_mode) { - SLT_INF(slot, "init spec for self-speculative decoding, slot %d\n", i); + } else if (params_base.speculative.configs.size() > 0) { + SLT_INF(slot, "init spec for speculative decoding without draft model, slot %d\n", i); slot.spec = common_speculative_init(nullptr, nullptr, - params_base.speculative.self_mode, - params_base.speculative.self_cfg); + params_base.speculative.configs); } SLT_INF(slot, "new slot, n_ctx = %d\n", slot.n_ctx); diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 0a721c3142..bcfb429f7a 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -5,6 +5,7 @@ #include "llama.h" #include "chat.h" #include "sampling.h" +#include "speculative.h" #include "json-schema-to-grammar.h" using json = nlohmann::ordered_json; @@ -237,8 +238,39 @@ task_params server_task::params_from_json_cmpl( params.speculative.n_min = json_value(data, "speculative.n_min", defaults.speculative.n_min); params.speculative.n_max = json_value(data, "speculative.n_max", defaults.speculative.n_max); params.speculative.p_min = json_value(data, "speculative.p_min", defaults.speculative.p_min); - params.speculative.self_mode = json_value(data, "speculative.self_mode", defaults.speculative.self_mode); - params.speculative.self_cfg = json_value(data, "speculative.self_cfg", defaults.speculative.self_cfg); + //params.speculative.self_mode = json_value(data, "speculative.self_mode", defaults.speculative.self_mode); + //params.speculative.self_cfg = json_value(data, "speculative.self_cfg", defaults.speculative.self_cfg); + // Set params.speculative.configs. Use json-array "speculative.configs" if provided in data, otherwise use {} + { + params.speculative.configs = defaults.speculative.configs; + const auto & configs = data.find("speculative.configs"); + if (configs != data.end() && configs->is_array()) { + params.speculative.configs.clear(); + for (const auto & config : *configs) { + if (config.is_object()) { + // config should have keys "type" and "config" (optional) + const auto & type = config.find("type"); + if (type != config.end() && type->is_string()) { + const auto type_name = type->get(); + const auto type_enum = common_speculative_type_from_name(type_name); + if (type_enum != COMMON_SPECULATIVE_TYPE_COUNT) { + common_speculative_config cfg(type_enum); + const auto & cfg_map = config.find("config"); + if (cfg_map != config.end() && cfg_map->is_object()) { + for (const auto & [key, value] : cfg_map->items()) { + cfg.config[key] = value.get(); + } + } + params.speculative.configs.push_back(cfg); + } else { + SRV_WRN("Unknown speculative type: %s\n", type_name.c_str()); + } + + } + } + } + } + } params.speculative.n_min = std::min(params.speculative.n_max, params.speculative.n_min); params.speculative.n_min = std::max(params.speculative.n_min, 0);