Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-08 22:09:02 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add a numeric context parameter declaring the maximum outputs one sequence

Browse Source
This commit is contained in:
Gaurav Garg
2026-07-05 02:48:13 +05:30
parent 0eddfa0b4c
commit eab4e8444f
14 changed files with 120 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-context.cpp
+15
View File
@@ -51,6 +51,18 @@ static uint32_t server_n_outputs_max(const common_params & params) {
return std::max<int32_t>(1, n_outputs);
}
static uint32_t server_n_sampling_outputs_per_seq_max(const common_params & params) {
if (params.embedding ||
(params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED && params.pooling_type != LLAMA_POOLING_TYPE_NONE)) {
return 1;
}
const int32_t n_outputs = common_speculative_n_outputs_per_seq_max(
params.n_batch, common_speculative_n_max(&params.speculative));
return std::max<int32_t>(1, n_outputs);
}
// state diagram: https://github.com/ggml-org/llama.cpp/pull/9283
enum slot_state {
SLOT_STATE_IDLE,
@@ -1063,6 +1075,7 @@ private:
params_base = params;
params_base.n_outputs_max = server_n_outputs_max(params_base);
params_base.n_sampling_outputs_per_seq_max = server_n_sampling_outputs_per_seq_max(params_base);
const bool has_mmproj = !params.mmproj.path.empty();
const bool has_draft = params.speculative.has_dft();
@@ -1143,6 +1156,7 @@ private:
bool measure_model_bytes = has_draft;
common_params params_dft = common_base_params_to_speculative(params_base);
params_dft.n_sampling_outputs_per_seq_max = 1;
auto mparams_dft = common_model_params_to_llama(params_dft);
auto cparams_dft = common_context_params_to_llama(params_dft);
@@ -1231,6 +1245,7 @@ private:
// progress callback
params_dft.load_progress_callback = load_progress_callback;
params_dft.load_progress_callback_user_data = &load_progress_spec;
params_dft.n_sampling_outputs_per_seq_max = 1;
spec_init = common_speculative_init_from_params(params_dft, model_tgt, ctx_tgt);
model_dft = spec_init->model();