Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-04 03:51:17 +02:00
Code Issues Packages Projects Releases Wiki Activity

spec: Add benchmark-only synthetic speculative acceptance options (#27711)

Browse Source 
* Add benchmark-only synthetic speculative acceptance to llama-server and llama-cli

* Address review comments

* Address review comments

* Add some comments in the code
This commit is contained in:
Gaurav Garg
2026-08-27 16:23:42 +05:30
committed by GitHub
parent deae5ee133
commit 2bb9bddafa
11 changed files with 427 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+32
View File
@@ -4132,6 +4132,38 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.speculative.draft.n_min = value;
}
).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_MIN"));
add_opt(common_arg(
{"--spec-synth-len"}, "L",
"target mean synthetic acceptance length, including the target token (benchmarking only)",
[](common_params & params, const std::string & value) {
const std::string text = string_strip(value);
size_t pos = 0;
const double length = std::stod(text, &pos);
if (pos != text.size() || length == -1.0) {
throw std::invalid_argument("invalid value");
}
params.speculative.synth_len = length;
}
).set_spec().set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_SYNTH_LEN"));
add_opt(common_arg(
{"--spec-synth-rates"}, "P0,P1,...",
"comma-separated unconditional per-position synthetic acceptance probabilities (benchmarking only)",
[](common_params & params, const std::string & value) {
const auto values = string_split<std::string>(value, ',');
std::vector<double> rates;
rates.reserve(values.size());
for (const auto & raw : values) {
const std::string text = string_strip(raw);
size_t pos = 0;
const double rate = std::stod(text, &pos);
if (pos != text.size()) {
throw std::invalid_argument("invalid value");
}
rates.push_back(rate);
}
params.speculative.synth_rates = std::move(rates);
}
).set_spec().set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_SYNTH_RATES"));
add_opt(common_arg(
{"--spec-draft-p-split", "--draft-p-split"}, "P",