Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 01:04:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

examples : update batched to use backend sampling

Browse Source 
This commit updates the batched example to demonstrate how to use
backend samplers.
This commit is contained in:
Daniel Bevenius
2025-11-24 16:37:22 +01:00
parent 25f33806d3
commit d0bea21a3c
2 changed files with 25 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/batched/batched.cpp
+13
View File
@@ -2,6 +2,7 @@
#include "common.h"
#include "log.h"
#include "llama.h"
#include "sampling.h"
#include <algorithm>
#include <cstdio>
@@ -64,6 +65,18 @@ int main(int argc, char ** argv) {
ctx_params.n_ctx = n_kv_req;
ctx_params.n_batch = std::max(n_predict, n_parallel);
std::vector<llama_sampler_seq_config> sampler_configs(n_parallel);
if (params.sampling.backend_sampling) {
for (int32_t i = 0; i < n_parallel; ++i) {
llama_sampler * backend_sampler = common_sampler_backend_init(model, params.sampling);
if (backend_sampler) {
sampler_configs[i] = { i, backend_sampler };
}
}
ctx_params.samplers = sampler_configs.data();
ctx_params.n_samplers = n_parallel;
}
llama_context * ctx = llama_init_from_model(model, ctx_params);
auto sparams = llama_sampler_chain_default_params();