diff --git a/common/arg.cpp b/common/arg.cpp index dd92e4720b..403cc2b781 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -4132,6 +4132,38 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.speculative.draft.n_min = value; } ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_MIN")); + add_opt(common_arg( + {"--spec-synth-len"}, "L", + "target mean synthetic acceptance length, including the target token (benchmarking only)", + [](common_params & params, const std::string & value) { + const std::string text = string_strip(value); + size_t pos = 0; + const double length = std::stod(text, &pos); + if (pos != text.size() || length == -1.0) { + throw std::invalid_argument("invalid value"); + } + params.speculative.synth_len = length; + } + ).set_spec().set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_SYNTH_LEN")); + add_opt(common_arg( + {"--spec-synth-rates"}, "P0,P1,...", + "comma-separated unconditional per-position synthetic acceptance probabilities (benchmarking only)", + [](common_params & params, const std::string & value) { + const auto values = string_split(value, ','); + std::vector rates; + rates.reserve(values.size()); + for (const auto & raw : values) { + const std::string text = string_strip(raw); + size_t pos = 0; + const double rate = std::stod(text, &pos); + if (pos != text.size()) { + throw std::invalid_argument("invalid value"); + } + rates.push_back(rate); + } + params.speculative.synth_rates = std::move(rates); + } + ).set_spec().set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_SYNTH_RATES")); add_opt(common_arg( {"--spec-draft-p-split", "--draft-p-split"}, "P", diff --git a/common/common.h b/common/common.h index 9593e10d9a..1cfb01e1ef 100644 --- a/common/common.h +++ b/common/common.h @@ -370,6 +370,9 @@ struct common_params_speculative_ngram_cache { struct common_params_speculative { std::vector types = { COMMON_SPECULATIVE_TYPE_NONE }; + double synth_len = -1.0; + std::vector synth_rates; + // used by Simple, MTP, Eagle3, etc. - all methods that require some kind of draft model common_params_speculative_draft draft; @@ -384,6 +387,10 @@ struct common_params_speculative { return !draft.mparams.empty(); } + bool has_synth() const { + return synth_len != -1.0 || !synth_rates.empty(); + } + uint32_t need_n_rs_seq() const { bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) { return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK; diff --git a/common/speculative.cpp b/common/speculative.cpp index 4eef2212e7..393a73bc39 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -138,6 +139,7 @@ struct common_speculative_impl { const common_speculative_type type; uint32_t n_seq; + int32_t n_max; // maximum draft length after implementation-specific limits size_t n_call_begin = 0; // number of times this implementation was called for refresh. size_t n_call_draft = 0; // number of times this implementation was called for generation. @@ -157,7 +159,7 @@ struct common_speculative_impl { int64_t t_draft_us = 0; // total time spent in generating drafts in this implementation in microseconds. int64_t t_accept_us = 0; // total time spent in accumulation of this implementation in microseconds. - common_speculative_impl(common_speculative_type type, uint32_t n_seq) : type(type), n_seq(n_seq) {} + common_speculative_impl(common_speculative_type type, uint32_t n_seq, int32_t n_max) : type(type), n_seq(n_seq), n_max(n_max) {} virtual ~common_speculative_impl() = default; @@ -182,7 +184,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { std::vector smpls; common_speculative_impl_draft_simple(const common_params_speculative & params, uint32_t n_seq) - : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, n_seq) + : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, n_seq, params.draft.n_max) , params(params.draft) { auto * ctx_dft = this->params.ctx_dft; @@ -452,7 +454,7 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { std::vector g_embd_buf; common_speculative_impl_draft_eagle3(const common_params_speculative & params, uint32_t n_seq) - : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, n_seq) + : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, n_seq, params.draft.n_max) , params(params.draft) { SPC_TRC("%s", "adding speculative implementation 'draft-eagle3'\n"); @@ -937,7 +939,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq, common_speculative_type type = COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH) - : common_speculative_impl(type, n_seq) + : common_speculative_impl(type, n_seq, params.draft.n_max) , params(params.draft) , is_dspark(type == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK) { @@ -983,6 +985,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { this->params.n_max = std::min(this->params.n_max, n_draft_max); this->params.n_min = std::min(this->params.n_min, n_draft_max); } + this->n_max = this->params.n_max; batch = llama_batch_init(llama_n_batch(ctx_dft), 0, n_seq); batch_inject = llama_batch_init(llama_n_batch(ctx_dft), n_embd_dec, n_seq); @@ -1315,7 +1318,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { std::vector> chain_h; common_speculative_impl_draft_mtp(const common_params_speculative & params, uint32_t n_seq) - : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq) + : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq, params.draft.n_max) , params(params.draft) { auto * ctx_tgt = this->params.ctx_tgt; @@ -1382,6 +1385,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { c.reserve((size_t) (this->params.n_max + 1) * n_embd); } } + this->n_max = this->params.n_max; pending_h.assign(n_seq, std::vector(n_embd, 0.0f)); @@ -1726,7 +1730,7 @@ struct common_speculative_impl_ngram_simple : public common_speculative_impl { common_speculative_impl_ngram_simple( const common_params_speculative & params, uint32_t n_seq, common_ngram_simple_config config) - : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, n_seq) + : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, n_seq, params.ngram_simple.size_m) , params(params.ngram_simple) , config(config) { @@ -1770,7 +1774,7 @@ struct common_speculative_impl_ngram_map_k : public common_speculative_impl { const common_ngram_map & config, uint32_t n_seq) : common_speculative_impl(config.key_only ? COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K - : COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, n_seq) + : COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, n_seq, config.size_value) { for (uint32_t i = 0; i < n_seq; i++) { this->config.push_back(config); @@ -1841,7 +1845,7 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { common_speculative_impl_ngram_mod( const common_params_speculative & params, uint32_t n_seq) - : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_MOD, n_seq) + : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_MOD, n_seq, params.ngram_mod.n_max) , params(params.ngram_mod) , mod(params.ngram_mod.n_match, 4*1024*1024) , verbose(std::getenv("LLAMA_TRACE") != nullptr) { @@ -2017,7 +2021,7 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl { const std::string & path_dynamic, bool save_dynamic, bool save_static) - : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, n_seq) + : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, n_seq, n_draft) , params(params.ngram_cache) , n_draft(n_draft) , save_dynamic(save_dynamic) @@ -2138,6 +2142,8 @@ struct common_speculative { // which implementaion was used for a given seq_id std::vector impl_last; + + std::vector synth_probs; }; static common_ngram_map get_common_ngram_map( @@ -2316,6 +2322,101 @@ int32_t common_speculative_n_max(const common_params_speculative * spec) { return n_max; } +int32_t common_speculative_n_max(const common_speculative * spec) { + int32_t n_max = 0; + + if (spec == nullptr) { + return n_max; + } + + for (const auto & impl : spec->impls) { + n_max = std::max(n_max, std::max(0, impl->n_max)); + } + + return n_max; +} + +std::vector common_speculative_synth_rates_resolve(const common_params_speculative * spec, int32_t n_max) { + const bool has_length = spec->synth_len != -1.0; + const bool has_rates = !spec->synth_rates.empty(); + + if (!has_length && !has_rates) { + return {}; + } + if (has_length && has_rates) { + throw std::invalid_argument("synthetic acceptance length and rates are mutually exclusive"); + } + + if (n_max <= 0) { + throw std::invalid_argument("synthetic acceptance requires at least one speculative token"); + } + + if (has_rates) { + const auto & rates = spec->synth_rates; + if (rates.size() != (size_t) n_max) { + throw std::invalid_argument(string_format( + "synthetic acceptance rates must contain %d values, got %zu", n_max, rates.size())); + } + + for (size_t i = 0; i < rates.size(); ++i) { + if (!std::isfinite(rates[i]) || rates[i] < 0.0 || rates[i] > 1.0) { + throw std::invalid_argument("synthetic acceptance rates must be finite and within [0, 1]"); + } + if (i > 0 && rates[i] > rates[i - 1]) { + throw std::invalid_argument("synthetic acceptance rates must be monotonically non-increasing"); + } + } + + return rates; + } + + const double length = spec->synth_len; + const double length_max = (double) n_max + 1.0; + if (!std::isfinite(length) || length < 1.0 || length > length_max) { + throw std::invalid_argument(string_format( + "synthetic acceptance length must be finite and within [1, %.0f]", length_max)); + } + + double p = 0.0; + if (length == length_max) { + p = 1.0; + } else if (length > 1.0) { + double p_min = 0.0; + double p_max = 1.0; + for (int i = 0; i < 32; ++i) { + const double p_mid = 0.5 * (p_min + p_max); + double sum = 0.0; + double term = p_mid; + for (int32_t j = 0; j < n_max; ++j) { + sum += term; + term *= p_mid; + } + + if (sum < length - 1.0) { + p_min = p_mid; + } else { + p_max = p_mid; + } + } + p = 0.5 * (p_min + p_max); + } + + std::vector rates; + rates.reserve(n_max); + double rate = p; + for (int32_t i = 0; i < n_max; ++i) { + rates.push_back(rate); + rate *= p; + } + + return rates; +} + +const std::vector & common_speculative_get_synth_probs(const common_speculative * spec) { + GGML_ASSERT(spec); + return spec->synth_probs; +} + common_params common_base_params_to_speculative(const common_params & params) { const bool has_draft = params.speculative.has_dft(); @@ -2568,13 +2669,39 @@ common_speculative * common_speculative_init(common_params_speculative & params, return nullptr; } - auto * result = new common_speculative { - /* .dparams = */ common_speculative_draft_params_vec(n_seq), - /* .impls = */ std::move(impls), - /* .impl_last = */ std::vector(n_seq, nullptr) - }; + common_speculative_ptr result(new common_speculative { + /* .dparams = */ common_speculative_draft_params_vec(n_seq), + /* .impls = */ std::move(impls), + /* .impl_last = */ std::vector(n_seq, nullptr), + /* .synth_probs = */ {}, + }); - return result; + const int32_t n_max_configured = common_speculative_n_max(¶ms); + const int32_t n_max_effective = common_speculative_n_max(result.get()); + const auto rates = common_speculative_synth_rates_resolve(¶ms, n_max_effective); + + std::vector rates_str; + rates_str.reserve(rates.size()); + result->synth_probs.reserve(rates.size()); + double rate_prev = 1.0; + double acceptance_length = 1.0; + for (const double rate : rates) { + result->synth_probs.push_back(rate_prev > 0.0 ? rate / rate_prev : 0.0); + rates_str.push_back(string_format("%.6g", rate)); + rate_prev = rate; + acceptance_length += rate; + } + if (!result->synth_probs.empty()) { + SPC_WRN("%s", "synthetic speculative acceptance is enabled for benchmarking; generated output is not valid\n"); + if (n_max_effective != n_max_configured) { + SPC_WRN("synthetic acceptance draft limit was reduced from %d to %d by the initialized speculative implementations\n", + n_max_configured, n_max_effective); + } + SPC_INF("synthetic acceptance: n_max = %zu, mean length = %.6f, rates = [%s]\n", + rates.size(), acceptance_length, string_join(rates_str, ", ").c_str()); + } + + return result.release(); } void common_speculative_free(common_speculative * spec) { diff --git a/common/speculative.h b/common/speculative.h index 12ae31b7de..22505891f7 100644 --- a/common/speculative.h +++ b/common/speculative.h @@ -26,6 +26,15 @@ std::string common_speculative_type_to_str(enum common_speculative_type type); // return the max number of draft tokens based on the speculative parameters int32_t common_speculative_n_max(const common_params_speculative * spec); +// return the max number of draft tokens from the initialized implementations +int32_t common_speculative_n_max(const common_speculative * spec); + +// validate and resolve the unconditional synthetic acceptance rates +std::vector common_speculative_synth_rates_resolve(const common_params_speculative * spec, int32_t n_max); + +// return the conditional synthetic acceptance probabilities +const std::vector & common_speculative_get_synth_probs(const common_speculative * spec); + common_params common_base_params_to_speculative(const common_params & params); struct common_speculative_output_limits { diff --git a/docs/speculative.md b/docs/speculative.md index 0f9f8a3d97..ffb1e34c7f 100644 --- a/docs/speculative.md +++ b/docs/speculative.md @@ -212,6 +212,15 @@ Use `--backend-sampling` to run supported target-model samplers on the model bac Unsupported samplers and device layouts fall back to CPU sampling. Tensor split mode does not support backend sampling. A fixed seed produces repeatable random draws, but stochastic CPU and backend sampling can still select different tokens because floating-point operations can differ between implementations and devices. Use greedy sampling when exact output matching is required. +### Synthetic Acceptance + +`llama-server` and `llama-cli` can replace normal speculative verification with synthetic decisions for benchmarking. The generated output is not valid model output because accepted draft tokens do not have to match the target model. + +Use exactly one of these options: + +- `--spec-synth-rates P0,P1,...` sets unconditional per-position acceptance probabilities. Entry `i` is the probability that the first `i+1` draft tokens are all accepted. The number of entries must match the effective maximum draft length. Values must be finite, within `[0, 1]`, and monotonically non-increasing. +- `--spec-synth-len L` sets the target mean acceptance length, including the target token. For `K` maximum draft tokens, `L` must be within `[1, K+1]`. The server finds a constant conditional probability `p` such that `p + p^2 + ... + p^K = L - 1`, then uses unconditional rates `[p, p^2, ..., p^K]`. + ### General Speculative Parameters ``` diff --git a/tests/test-arg-parser.cpp b/tests/test-arg-parser.cpp index ba58f852eb..e0907631ab 100644 --- a/tests/test-arg-parser.cpp +++ b/tests/test-arg-parser.cpp @@ -4,6 +4,7 @@ #include "llama.h" #include "speculative.h" +#include #include #include #include @@ -34,6 +35,62 @@ static void test(void) { std::numeric_limits::max(), std::numeric_limits::max()); + { + common_params_speculative spec; + spec.synth_len = 3.4; + + auto assert_invalid = [](const common_params_speculative & value, int32_t n_max) { + try { + common_speculative_synth_rates_resolve(&value, n_max); + assert(false); + } catch (const std::invalid_argument &) { + } + }; + + const auto rates = common_speculative_synth_rates_resolve(&spec, 4); + assert(rates.size() == 4); + assert(std::abs(rates[0] - 0.80581) < 1e-5); + assert(std::abs(rates[1] - 0.64933) < 1e-5); + assert(std::abs(rates[2] - 0.52323) < 1e-5); + assert(std::abs(rates[3] - 0.42163) < 1e-5); + assert(std::abs(1.0 + rates[0] + rates[1] + rates[2] + rates[3] - 3.4) < 1e-8); + + spec.synth_len = 1.0; + assert(common_speculative_synth_rates_resolve(&spec, 4) == std::vector({0.0, 0.0, 0.0, 0.0})); + + spec.synth_len = 5.0; + assert(common_speculative_synth_rates_resolve(&spec, 4) == std::vector({1.0, 1.0, 1.0, 1.0})); + + spec.synth_len = 5.1; + assert_invalid(spec, 4); + + spec.synth_len = std::numeric_limits::quiet_NaN(); + assert_invalid(spec, 4); + + spec.synth_len = 0.0; + assert_invalid(spec, 4); + + spec.synth_len = -1.0; + spec.synth_rates = {0.8, 0.6, 0.4}; + assert_invalid(spec, 4); + + spec.synth_rates = {0.8, 0.6, 0.4, 0.2}; + assert(common_speculative_synth_rates_resolve(&spec, 4) == spec.synth_rates); + + spec.synth_rates = {0.8, 0.9, 0.4, 0.2}; + assert_invalid(spec, 4); + + spec.synth_rates = {0.8, std::numeric_limits::quiet_NaN(), 0.4, 0.2}; + assert_invalid(spec, 4); + + spec.synth_rates = {0.8, 0.6, 0.4, -0.2}; + assert_invalid(spec, 4); + + spec.synth_rates = {0.8, 0.6, 0.4, 0.2}; + spec.synth_len = 3.0; + assert_invalid(spec, 4); + } + { common_params base; base.n_parallel = 4; @@ -197,6 +254,26 @@ static void test(void) { assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_SPECULATIVE)); assert(params.speculative.draft.n_max == 123); + { + common_params synth_params; + argv = {"binary_name", "--spec-synth-len", "3.4"}; + assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER)); + assert(synth_params.speculative.synth_len == 3.4); + } + + { + common_params synth_params; + argv = {"binary_name", "--spec-synth-rates", "0.8,0.6,0.2"}; + assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER)); + assert(synth_params.speculative.synth_rates == std::vector({0.8, 0.6, 0.2})); + } + + { + common_params synth_params; + argv = {"binary_name", "--spec-synth-len", "3.4x"}; + assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER)); + } + argv = {"binary_name", "-lm", "none"}; assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON)); assert(params.load_mode == LLAMA_LOAD_MODE_NONE); diff --git a/tools/cli/README.md b/tools/cli/README.md index 0fba70a90e..e663cfa3b2 100644 --- a/tools/cli/README.md +++ b/tools/cli/README.md @@ -200,6 +200,8 @@ | `--spec-draft-n-cpu-moe, --spec-draft-ncmoe, -ncmoed, --n-cpu-moe-draft N` | keep the Mixture of Experts (MoE) weights of the first N layers in the CPU for the draft model
(env: LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE) | | `--spec-draft-n-max N` | number of tokens to draft for speculative decoding (default: 3)
(env: LLAMA_ARG_SPEC_DRAFT_N_MAX) | | `--spec-draft-n-min N` | minimum number of draft tokens to use for speculative decoding (default: 0)
(env: LLAMA_ARG_SPEC_DRAFT_N_MIN) | +| `--spec-synth-len L` | target mean synthetic acceptance length, including the target token (benchmarking only)
(env: LLAMA_ARG_SPEC_SYNTH_LEN) | +| `--spec-synth-rates P0,P1,...` | comma-separated unconditional per-position synthetic acceptance probabilities (benchmarking only)
(env: LLAMA_ARG_SPEC_SYNTH_RATES) | | `--spec-draft-p-split, --draft-p-split P` | speculative decoding split probability (default: 0.10)
(env: LLAMA_ARG_SPEC_DRAFT_P_SPLIT) | | `--spec-draft-p-min, --draft-p-min P` | minimum speculative decoding probability (greedy) (default: 0.00)
(env: LLAMA_ARG_SPEC_DRAFT_P_MIN) | | `--spec-draft-backend-sampling, --no-spec-draft-backend-sampling` | offload draft sampling to the backend (default: enabled)
(env: LLAMA_ARG_SPEC_DRAFT_BACKEND_SAMPLING) | diff --git a/tools/server/README.md b/tools/server/README.md index f49cdb2729..6fd27f1389 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -259,6 +259,8 @@ For the full list of features, please refer to [server's changelog](https://gith | `--spec-draft-n-cpu-moe, --spec-draft-ncmoe, -ncmoed, --n-cpu-moe-draft N` | keep the Mixture of Experts (MoE) weights of the first N layers in the CPU for the draft model
(env: LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE) | | `--spec-draft-n-max N` | number of tokens to draft for speculative decoding (default: 3)
(env: LLAMA_ARG_SPEC_DRAFT_N_MAX) | | `--spec-draft-n-min N` | minimum number of draft tokens to use for speculative decoding (default: 0)
(env: LLAMA_ARG_SPEC_DRAFT_N_MIN) | +| `--spec-synth-len L` | target mean synthetic acceptance length, including the target token (benchmarking only)
(env: LLAMA_ARG_SPEC_SYNTH_LEN) | +| `--spec-synth-rates P0,P1,...` | comma-separated unconditional per-position synthetic acceptance probabilities (benchmarking only)
(env: LLAMA_ARG_SPEC_SYNTH_RATES) | | `--spec-draft-p-split, --draft-p-split P` | speculative decoding split probability (default: 0.10)
(env: LLAMA_ARG_SPEC_DRAFT_P_SPLIT) | | `--spec-draft-p-min, --draft-p-min P` | minimum speculative decoding probability (greedy) (default: 0.00)
(env: LLAMA_ARG_SPEC_DRAFT_P_MIN) | | `--spec-draft-backend-sampling, --no-spec-draft-backend-sampling` | offload draft sampling to the backend (default: enabled)
(env: LLAMA_ARG_SPEC_DRAFT_BACKEND_SAMPLING) | diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 9fdfcae563..e6c991f7cf 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -51,6 +52,50 @@ static common_speculative_output_limits server_output_limits(const common_params return result; } +// synthetic draft verification for benchmarking - accept draft tokens at random instead of by match with the target +// on replay the draft was already accepted before a context checkpoint restore, so repeat the same decisions +static std::vector server_sample_and_accept_synth( + common_sampler * smpl, + llama_context * ctx, + const std::vector & idxs, + const llama_tokens & draft, + const std::vector & synth_probs, + std::mt19937 & rng, + bool is_replay) { + GGML_ASSERT(idxs.size() == draft.size() + 1); + GGML_ASSERT(synth_probs.size() >= draft.size()); + + std::vector result; + result.reserve(idxs.size()); + + const llama_vocab * vocab = llama_model_get_vocab(llama_get_model(ctx)); + std::uniform_real_distribution dist(0.0, 1.0); + for (size_t i = 0; i < draft.size(); ++i) { + const llama_token id = common_sampler_sample(smpl, ctx, idxs[i]); + const bool accept = is_replay || dist(rng) < synth_probs[i]; + // do not accept a drafted EOG token - it would end the generation early + // on replay the last token is from the target and can be EOG, so skip this check + if (accept && (is_replay || !llama_vocab_is_eog(vocab, draft[i]))) { + // synthetic draft tokens do not advance grammar or reasoning state + // the last replay token is from the target and must advance both + const bool is_replay_target = is_replay && i + 1 == draft.size(); + common_sampler_accept(smpl, draft[i], is_replay_target); + result.push_back(draft[i]); + continue; + } + + common_sampler_accept(smpl, id, true); + result.push_back(id); + return result; + } + + const llama_token id = common_sampler_sample(smpl, ctx, idxs[draft.size()]); + common_sampler_accept(smpl, id, true); + result.push_back(id); + + return result; +} + // state diagram: https://github.com/ggml-org/llama.cpp/pull/9283 enum slot_state { SLOT_STATE_IDLE, @@ -211,6 +256,7 @@ struct server_slot { std::vector spec_i_batch; common_prompt_checkpoint spec_ckpt; bool spec_is_replay = false; + std::mt19937 spec_synth_rng; // TODO: move members that belong to the task (such as `generated_text`, `has_new_line`) to task_results_state // see https://github.com/ggml-org/llama.cpp/pull/18283#issuecomment-3710175837 @@ -1194,6 +1240,9 @@ private: spec.reset(common_speculative_init(params_base.speculative, params_base.n_parallel)); } catch (const std::exception & e) { SRV_ERR("failed to initialize speculative decoding context: %s\n", e.what()); + if (params_base.speculative.has_synth()) { + return false; + } } } @@ -1209,6 +1258,11 @@ private: model_dft = nullptr; } + if (!spec && params_base.speculative.has_synth()) { + SRV_ERR("%s", "synthetic acceptance requires an initialized speculative decoding context\n"); + return false; + } + for (int i = 0; i < params_base.n_parallel; i++) { server_slot & slot = slots[i]; @@ -1717,6 +1771,13 @@ private: SLT_TRC(slot, "sampler chain: %s\n", common_sampler_print(slot.smpl.get()).c_str()); SLT_TRC(slot, "sampler params: \n%s\n", task.params.sampling.print().c_str()); + + if (spec && !common_speculative_get_synth_probs(spec.get()).empty()) { + const uint32_t seed = task.params.sampling.seed == LLAMA_DEFAULT_SEED + ? std::random_device{}() + : task.params.sampling.seed; + slot.spec_synth_rng.seed(seed); + } } else { slot.smpl.reset(); } @@ -3802,7 +3863,12 @@ private: common_sampler_ptr smpl_save(common_sampler_clone(slot.smpl.get())); GGML_ASSERT(slot.spec_i_batch.size() == n_draft + 1); - auto accepted = common_sampler_sample_and_accept_n(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft); + const auto & synth_probs = common_speculative_get_synth_probs(spec.get()); + auto accepted = synth_probs.empty() + ? common_sampler_sample_and_accept_n(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft) + : server_sample_and_accept_synth( + slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft, + synth_probs, slot.spec_synth_rng, slot.spec_is_replay); slot.spec_i_batch.clear(); GGML_ASSERT(accepted.size() >= 1); @@ -3868,7 +3934,7 @@ private: auto & n_accepted_per_pos = slot.n_accepted_per_pos; if (n_accepted_per_pos.empty()) { - n_accepted_per_pos.resize(common_speculative_n_max(¶ms_base.speculative), 0); + n_accepted_per_pos.resize(common_speculative_n_max(spec.get()), 0); } for (size_t i = 0; i < n_accepted && i < n_accepted_per_pos.size(); ++i) { n_accepted_per_pos[i]++; diff --git a/tools/server/tests/unit/test_speculative.py b/tools/server/tests/unit/test_speculative.py index 5837195006..22b523954e 100644 --- a/tools/server/tests/unit/test_speculative.py +++ b/tools/server/tests/unit/test_speculative.py @@ -52,6 +52,18 @@ def test_with_and_without_draft(): assert tokens_no_draft == tokens_draft + server.stop() + create_server() + assert server.spec_draft_n_max is not None + server.spec_synth_rates = [0.0] * server.spec_draft_n_max + server.start() + res = server.make_request("POST", "/completion", data=request) + + assert res.status_code == 200 + assert res.body["timings"]["draft_n"] > 0 + assert res.body["timings"]["draft_n_accepted"] == 0 + assert res.body["tokens"] == tokens_no_draft + def test_different_draft_min_draft_max(): global server @@ -80,6 +92,66 @@ def test_different_draft_min_draft_max(): last_content = res.body["content"] +def test_synth_is_deterministic(): + global server + assert server.spec_draft_n_max is not None + server.spec_synth_rates = [0.75 ** (i + 1) for i in range(server.spec_draft_n_max)] + server.start() + + request = { + "prompt": "I believe the meaning of life is", + "temperature": 0.2, + "top_k": 5, + "seed": 4242, + "n_predict": 32, + } + responses = [server.make_request("POST", "/completion", data=request) for _ in range(2)] + + for res in responses: + assert res.status_code == 200 + assert res.body["timings"]["draft_n"] > 0 + assert responses[0].body["timings"]["draft_n"] == responses[1].body["timings"]["draft_n"] + assert responses[0].body["timings"]["draft_n_accepted"] == responses[1].body["timings"]["draft_n_accepted"] + + +def test_synth_ignores_target_tokens(): + global server + assert server.spec_draft_n_max is not None + server.spec_synth_rates = [1.0] * server.spec_draft_n_max + server.start() + + res = server.make_request("POST", "/completion", data={ + "prompt": "I believe the meaning of life is", + "temperature": 0.0, + "seed": 4242, + "n_predict": 32, + }) + + assert res.status_code == 200 + assert res.body["timings"]["draft_n"] > 0 + assert res.body["timings"]["draft_n_accepted"] == res.body["timings"]["draft_n"] + + res = server.make_request("POST", "/completion", data={ + "prompt": "I believe the meaning of life is", + "temperature": 0.0, + "seed": 4242, + "n_predict": 6, + "grammar": 'root ::= "a"{5,5}', + }) + assert res.status_code == 200, res.body + + res = server.make_request("POST", "/completion", data={ + "prompt": "Respond with only: OK", + "temperature": 0.0, + "seed": 4242, + "n_predict": 64, + "ignore_eos": True, + }) + assert res.status_code == 200, res.body + assert res.body["tokens_predicted"] == 64 + assert res.body["stop_type"] == "limit" + + def test_slot_ctx_not_exceeded(): global server server.n_ctx = 256 diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.py index a0d2dfa3c5..5a4f31a53e 100644 --- a/tools/server/tests/utils.py +++ b/tools/server/tests/utils.py @@ -99,6 +99,8 @@ class ServerProcess: spec_type: str | None = None spec_draft_n_min: int | None = None spec_draft_n_max: int | None = None + spec_synth_len: float | None = None + spec_synth_rates: List[float] | None = None no_ui: bool | None = None jinja: bool | None = None reasoning_format: Literal['deepseek', 'none', 'nothink'] | None = None @@ -245,6 +247,11 @@ class ServerProcess: server_args.extend(["--spec-draft-n-max", self.spec_draft_n_max]) if self.spec_draft_n_min: server_args.extend(["--spec-draft-n-min", self.spec_draft_n_min]) + if self.spec_synth_len is not None: + server_args.extend(["--spec-synth-len", self.spec_synth_len]) + if self.spec_synth_rates is not None: + rates = ",".join(str(rate) for rate in self.spec_synth_rates) + server_args.extend(["--spec-synth-rates", rates]) if self.no_ui: server_args.append("--no-ui") if self.no_models_autoload: