Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-02 02:51:08 +02:00
Code Issues Packages Projects Releases Wiki Activity

llama : support multi-output backend sampling (#25532)

Browse Source 
* Enable backend sampling with token speculation

* Clamp the mask sum before converting it into the sampled index

* Add a numeric context parameter declaring the maximum outputs one sequence

* More fixes

* Don't reuse memory for output views.

* Match dist between CPU and GPU

* Fix CPU and backend sampling mismatches

* Simpify some of the changes

* Fix tests on Vulkan

* More test fixes

* Rebase changes

* Rebase and address review comments

* Address review comments

* Address review comments

* Update src/llama-sampler.cpp

Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>

---------

Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
This commit is contained in:
Gaurav Garg
2026-08-10 19:28:56 +05:30
committed by GitHub
parent d2f83055d6
commit dd1ea52433
24 changed files with 1305 additions and 391 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/sampling.cpp
+20
View File
@@ -518,6 +518,26 @@ struct common_sampler * common_sampler_clone(common_sampler * gsmpl) {
};
}
void common_sampler_copy(const common_sampler * src, common_sampler * dst) {
if (!src || !dst || src == dst) {
return;
}
GGML_ASSERT((src->grmr == nullptr) == (dst->grmr == nullptr));
GGML_ASSERT((src->rbudget == nullptr) == (dst->rbudget == nullptr));
llama_sampler_copy(src->grmr, dst->grmr);
llama_sampler_copy(src->rbudget, dst->rbudget);
llama_sampler_copy(src->chain, dst->chain);
dst->params = src->params;
dst->prev = src->prev;
dst->cur = src->cur;
dst->cur_p = src->cur_p;
dst->cur_p.data = src->cur_p.data ? dst->cur.data() : nullptr; // re-point to dst's buffer
dst->t_total_us = src->t_total_us;
}
void common_perf_print(const struct llama_context * ctx, const struct common_sampler * gsmpl) {
// TODO: measure grammar performance