diff --git a/common/arg.cpp b/common/arg.cpp index 4f828dc258..bf8b7ab913 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -629,7 +629,8 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context { bool has_draft =!params.speculative.model.path.empty(); bool has_draft_eagle3 = false; // TODO PR-18039: if params.speculative.eagle3 - bool has_lookup_caches = !params.lookup_cache_static.empty() && !params.lookup_cache_dynamic.empty(); + bool has_lookup_caches = !params.speculative.lookup_cache_static.empty() + && !params.speculative.lookup_cache_dynamic.empty(); bool has_simple = (params.speculative.draftless_type == COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE); bool found_config_draft = false; bool found_config_eagle3 = false; @@ -1253,14 +1254,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex {"-lcs", "--lookup-cache-static"}, "FNAME", "path to static lookup cache to use for lookup decoding (not updated by generation)", [](common_params & params, const std::string & value) { - params.lookup_cache_static = value; + params.speculative.lookup_cache_static = value; } ).set_examples({LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER})); add_opt(common_arg( {"-lcd", "--lookup-cache-dynamic"}, "FNAME", "path to dynamic lookup cache to use for lookup decoding (updated by generation)", [](common_params & params, const std::string & value) { - params.lookup_cache_dynamic = value; + params.speculative.lookup_cache_dynamic = value; } ).set_examples({LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER})); add_opt(common_arg( diff --git a/common/common.h b/common/common.h index c1c4681f05..d8bde215d3 100644 --- a/common/common.h +++ b/common/common.h @@ -281,11 +281,16 @@ struct common_params_speculative { struct common_params_model model; + // draftless: + common_speculative_type draftless_type = COMMON_SPECULATIVE_TYPE_NONE; // type of speculative decoding without a draft model - uint16_t spec_ngram_size_n = 12; - uint16_t spec_ngram_size_m = 48; + uint16_t spec_ngram_size_n = 12; // ngram size for lookup + uint16_t spec_ngram_size_m = 48; // mgram size for speculative tokens std::vector configs = {}; // list of speculative configs to try + + std::string lookup_cache_static = ""; // path of static ngram cache file for lookup decoding // NOLINT + std::string lookup_cache_dynamic = ""; // path of dynamic ngram cache file for lookup decoding // NOLINT }; struct common_params_vocoder { @@ -403,8 +408,6 @@ struct common_params { std::string path_prompt_cache = ""; // path to file for saving/loading prompt eval state // NOLINT std::string input_prefix = ""; // string to prefix user inputs with // NOLINT std::string input_suffix = ""; // string to suffix user inputs with // NOLINT - std::string lookup_cache_static = ""; // path of static ngram cache file for lookup decoding // NOLINT - std::string lookup_cache_dynamic = ""; // path of dynamic ngram cache file for lookup decoding // NOLINT std::string logits_file = ""; // file for saving *all* logits // NOLINT // llama-debug specific options diff --git a/common/ngram-map.cpp b/common/ngram-map.cpp index 9c589ab6f7..bebbc72f87 100644 --- a/common/ngram-map.cpp +++ b/common/ngram-map.cpp @@ -237,7 +237,7 @@ void common_ngram_map_draft(common_ngram_map & map, map.last_draft_value_idx = slot_max; // value used for draft generation. } -void common_ngram_map_send_accepted(common_ngram_map & map, uint16_t n_accepted) { +void common_ngram_map_accept(common_ngram_map & map, uint16_t n_accepted) { if (!map.last_draft_created) { return; } diff --git a/common/ngram-map.h b/common/ngram-map.h index 6485eba186..09fa1053d2 100644 --- a/common/ngram-map.h +++ b/common/ngram-map.h @@ -62,5 +62,5 @@ void common_ngram_map_draft( const llama_tokens & inp, llama_token sampled, llama_tokens & draft); -// Update the statistics of a value after a draft was accepted. -void common_ngram_map_send_accepted(common_ngram_map & map, uint16_t n_accepted); +// Update the statistics of a value after a draft was processed. +void common_ngram_map_accept(common_ngram_map & map, uint16_t n_accepted); diff --git a/common/speculative.cpp b/common/speculative.cpp index 199965cc85..1a66576a08 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -251,12 +251,12 @@ enum common_speculative_type common_speculative_type_from_name(const std::string struct common_speculative * common_speculative_init( - struct common_params & params, + struct common_params_speculative & params, struct llama_context * ctx_tgt, struct llama_context * ctx_dft ) { std::vector> implementations = {}; - for (const common_speculative_config & config : params.speculative.configs) { + for (const common_speculative_config & config : params.configs) { LOG_INF("common_speculative_init: adding implementation %s\n", common_speculative_type_to_str(config.type).c_str()); switch (config.type) { case COMMON_SPECULATIVE_TYPE_NONE: @@ -271,7 +271,7 @@ struct common_speculative * common_speculative_init( } case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: { common_ngram_map ngram_map = get_common_ngram_map(config, - params.speculative.spec_ngram_size_n, params.speculative.spec_ngram_size_m); + params.spec_ngram_size_n, params.spec_ngram_size_m); uint16_t ngram_size_key = ngram_map.size_key; uint16_t mgram_size_value = ngram_map.size_value; uint16_t check_rate = ngram_map.check_rate; @@ -287,14 +287,14 @@ struct common_speculative * common_speculative_init( case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: { implementations.push_back(std::make_unique( (config.type), get_common_ngram_map(config, - params.speculative.spec_ngram_size_n, params.speculative.spec_ngram_size_m) + params.spec_ngram_size_n, params.spec_ngram_size_m) )); break; } case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: { implementations.push_back(std::make_unique( (config.type), get_common_ngram_map(config, - params.speculative.spec_ngram_size_n, params.speculative.spec_ngram_size_m))); + params.spec_ngram_size_n, params.spec_ngram_size_m))); break; } case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE: { @@ -746,7 +746,7 @@ llama_tokens common_speculative_use_draft_model( return result; } -void common_speculative_send_accepted(struct common_speculative * spec, const uint16_t n_accepted) { +void common_speculative_accept(struct common_speculative * spec, const uint16_t n_accepted) { common_speculative_state * impl = spec->curr_impl; if (impl != nullptr) { if (n_accepted > 0) { @@ -756,7 +756,7 @@ void common_speculative_send_accepted(struct common_speculative * spec, const ui if (impl->type == COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K || impl->type == COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V) { auto state = static_cast(impl); - common_ngram_map_send_accepted(state->map, n_accepted); + common_ngram_map_accept(state->map, n_accepted); } } } diff --git a/common/speculative.h b/common/speculative.h index 3cf3fdbf3b..6824a2652a 100644 --- a/common/speculative.h +++ b/common/speculative.h @@ -22,7 +22,7 @@ enum common_speculative_type common_speculative_type_from_name(const std::string std::string common_speculative_type_to_str(enum common_speculative_type type); struct common_speculative * common_speculative_init( - struct common_params & params, + struct common_params_speculative & params, struct llama_context * ctx_tgt, struct llama_context * ctx_dft ); @@ -45,7 +45,7 @@ llama_tokens common_speculative_gen_draft( llama_token id_last); // informs the speculative decoder that n_accepted tokens were accepted by the target model -void common_speculative_send_accepted( +void common_speculative_accept( struct common_speculative * spec, const uint16_t n_accepted); diff --git a/examples/lookup/lookup-create.cpp b/examples/lookup/lookup-create.cpp index bb94a8fe06..f7b6ea1b19 100644 --- a/examples/lookup/lookup-create.cpp +++ b/examples/lookup/lookup-create.cpp @@ -32,9 +32,9 @@ int main(int argc, char ** argv){ common_ngram_cache ngram_cache; common_ngram_cache_update(ngram_cache, LLAMA_NGRAM_STATIC, LLAMA_NGRAM_STATIC, inp, inp.size(), true); - fprintf(stderr, "%s: hashing done, writing file to %s\n", __func__, params.lookup_cache_static.c_str()); + fprintf(stderr, "%s: hashing done, writing file to %s\n", __func__, params.speculative.lookup_cache_static.c_str()); - common_ngram_cache_save(ngram_cache, params.lookup_cache_static); + common_ngram_cache_save(ngram_cache, params.speculative.lookup_cache_static); return 0; } diff --git a/examples/lookup/lookup-stats.cpp b/examples/lookup/lookup-stats.cpp index 135f6fcab9..ae28b2e6e8 100644 --- a/examples/lookup/lookup-stats.cpp +++ b/examples/lookup/lookup-stats.cpp @@ -46,18 +46,18 @@ int main(int argc, char ** argv){ { const int64_t t_start_draft_us = ggml_time_us(); - if (!params.lookup_cache_static.empty()) { + if (!params.speculative.lookup_cache_static.empty()) { try { - ngram_cache_static = common_ngram_cache_load(params.lookup_cache_static); + ngram_cache_static = common_ngram_cache_load(params.speculative.lookup_cache_static); } catch (std::ifstream::failure const &) { - LOG_ERR("failed to open static lookup cache: %s", params.lookup_cache_static.c_str()); + LOG_ERR("failed to open static lookup cache: %s", params.speculative.lookup_cache_static.c_str()); exit(1); } } - if (!params.lookup_cache_dynamic.empty()) { + if (!params.speculative.lookup_cache_dynamic.empty()) { try { - ngram_cache_dynamic = common_ngram_cache_load(params.lookup_cache_dynamic); + ngram_cache_dynamic = common_ngram_cache_load(params.speculative.lookup_cache_dynamic); } catch (std::ifstream::failure const &) {} // if the file does not exist it will simply be created at the end of the program } diff --git a/examples/lookup/lookup.cpp b/examples/lookup/lookup.cpp index 27f159940a..8e73138a5f 100644 --- a/examples/lookup/lookup.cpp +++ b/examples/lookup/lookup.cpp @@ -51,18 +51,18 @@ int main(int argc, char ** argv){ const int64_t t_start_draft_us = ggml_time_us(); common_ngram_cache_update(ngram_cache_context, LLAMA_NGRAM_MIN, LLAMA_NGRAM_MAX, inp, inp.size(), false); - if (!params.lookup_cache_static.empty()) { + if (!params.speculative.lookup_cache_static.empty()) { try { - ngram_cache_static = common_ngram_cache_load(params.lookup_cache_static); + ngram_cache_static = common_ngram_cache_load(params.speculative.lookup_cache_static); } catch (std::ifstream::failure const &) { - LOG_ERR("failed to open static lookup cache: %s", params.lookup_cache_static.c_str()); + LOG_ERR("failed to open static lookup cache: %s", params.speculative.lookup_cache_static.c_str()); exit(1); } } - if (!params.lookup_cache_dynamic.empty()) { + if (!params.speculative.lookup_cache_dynamic.empty()) { try { - ngram_cache_dynamic = common_ngram_cache_load(params.lookup_cache_dynamic); + ngram_cache_dynamic = common_ngram_cache_load(params.speculative.lookup_cache_dynamic); } catch (std::ifstream::failure const &) {} // if the file does not exist it will simply be created at the end of the program } @@ -210,7 +210,7 @@ int main(int argc, char ** argv){ // Update dynamic ngram cache with context ngram cache and save it to disk: common_ngram_cache_merge(ngram_cache_dynamic, ngram_cache_context); - common_ngram_cache_save(ngram_cache_dynamic, params.lookup_cache_dynamic); + common_ngram_cache_save(ngram_cache_dynamic, params.speculative.lookup_cache_dynamic); LOG("\n\n"); diff --git a/examples/speculative-simple/speculative-simple.cpp b/examples/speculative-simple/speculative-simple.cpp index 0439209131..a941da5c40 100644 --- a/examples/speculative-simple/speculative-simple.cpp +++ b/examples/speculative-simple/speculative-simple.cpp @@ -132,7 +132,7 @@ int main(int argc, char ** argv) { params_spec.n_reuse = llama_n_ctx(ctx_dft) - n_draft; params_spec.p_min = p_min; - struct common_speculative * spec = common_speculative_init(params, ctx_tgt, ctx_dft); + struct common_speculative * spec = common_speculative_init(params.speculative, ctx_tgt, ctx_dft); for (auto &pair : params.speculative.replacements) { common_speculative_add_replacement_tgt_dft(spec, pair.first.c_str(), pair.second.c_str()); } diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 20ec61940c..564736cb61 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -48,8 +48,6 @@ enum server_state { struct server_slot { int id; - llama_batch batch_spec = {}; - // TODO: change to unique_ptrs for consistency: llama_context * ctx = nullptr; llama_context * ctx_dft = nullptr; @@ -604,8 +602,6 @@ private: common_speculative_free(slot.spec); slot.spec = nullptr; - - llama_batch_free(slot.batch_spec); } llama_batch_free(batch); @@ -766,8 +762,6 @@ private: slot.prompt.tokens.has_mtmd = mctx != nullptr; if (model_dft) { - slot.batch_spec = llama_batch_init(params_base.speculative.n_max + 1, 0, 1); - // TODO: rework speculative decoding [TAG_SERVER_SPEC_REWORK] slot.ctx_dft = llama_init_from_model(model_dft, cparams_dft); if (slot.ctx_dft == nullptr) { @@ -775,7 +769,7 @@ private: return false; } - slot.spec = common_speculative_init(params_base, slot.ctx, slot.ctx_dft); + slot.spec = common_speculative_init(params_base.speculative, slot.ctx, slot.ctx_dft); if (slot.spec == nullptr) { SRV_ERR("%s", "failed to create speculator\n"); return false; @@ -784,7 +778,7 @@ private: common_speculative_add_replacement_tgt_dft(slot.spec, pair.first.c_str(), pair.second.c_str()); } } else if (params_base.speculative.configs.size() > 0) { - slot.spec = common_speculative_init(params_base, nullptr, nullptr); + slot.spec = common_speculative_init(params_base.speculative, nullptr, nullptr); } SLT_INF(slot, "new slot, n_ctx = %d\n", slot.n_ctx); @@ -1185,14 +1179,6 @@ private: slot.task = std::make_unique(std::move(task)); - // initialize draft batch - // TODO: rework speculative decoding [TAG_SERVER_SPEC_REWORK] - if (slot.can_speculate()) { - llama_batch_free(slot.batch_spec); - - slot.batch_spec = llama_batch_init(task.params.speculative.n_max + 1, 0, 1); - } - slot.state = slot.task->is_child() ? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt : SLOT_STATE_STARTED; @@ -2817,8 +2803,8 @@ private: // update how many tokens out of those tested were accepted slot.n_draft_accepted += ids.size() - 1; - // inform the speculative decoding about the accepted tokens - common_speculative_send_accepted(slot.spec, ids.size() - 1); + // inform the speculative decoding about the number of accepted tokens + common_speculative_accept(slot.spec, ids.size() - 1); // rollback to the state before sampling the draft tokens slot.prompt.tokens.keep_first(slot.prompt.n_tokens() - n_draft);