Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 09:15:04 +02:00
Code Issues Packages Projects Releases Wiki Activity

sampling : use pinned memory for backend sampling buffers

Browse Source
This commit is contained in:
Daniel Bevenius
2025-11-21 14:02:16 +01:00
parent c1625620f6
commit 61ffe41dc1
7 changed files with 358 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/llama.h
+10 -7
View File
@@ -974,20 +974,23 @@ extern "C" {
// The index matches llama_get_backend_sampled_token_ith().
// Returns NULL if no probabilites were generated.
LLAMA_API float * llama_get_backend_sampled_probs_ith(struct llama_context * ctx, int32_t i);
//
// Get the number of backend sampled probabilites for the ith token.
LLAMA_API uint32_t llama_get_backend_sampled_probs_count_ith(struct llama_context * ctx, int32_t i);
// Get the backend sampled logits for the ith token
// Returns NULL if no logits were sampled.
LLAMA_API float * llama_get_backend_sampled_logits_ith(struct llama_context * ctx, int32_t i);
// Get the backend sampled token ids associated with the sampled logits for the ith token
// Returns NULL if no logits were sampled.
LLAMA_API llama_token * llama_get_backend_sampled_token_ids_ith(struct llama_context * ctx, int32_t i);
//
// Get the number of backend sampled logits for the ith token.
LLAMA_API uint32_t llama_get_backend_sampled_logits_count_ith(struct llama_context * ctx, int32_t i);
// Get the number of backend sampled probabilites for the ith token.
LLAMA_API uint32_t llama_get_backend_sampled_probs_count_ith(struct llama_context * ctx, int32_t i);
// Get the backend sampled candidates (token ids) for the ith token
// Returns NULL if no candidates were sampled.
LLAMA_API llama_token * llama_get_backend_sampled_candidates_ith(struct llama_context * ctx, int32_t i);
//
// Get the number of backend sampled candidates for the ith token.
LLAMA_API uint32_t llama_get_backend_sampled_candidates_count_ith(struct llama_context * ctx, int32_t i);
//
// Vocab