Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-08 22:09:02 +02:00
Code Issues Packages Projects Releases Wiki Activity

Match dist between CPU and GPU

Browse Source
This commit is contained in:
Gaurav Garg
2026-07-07 20:54:05 +05:30
parent 813c39b225
commit a24097dc57
7 changed files with 170 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-context.cpp
+4 -16
View File
@@ -222,7 +222,6 @@ struct server_slot {
std::vector<int32_t> spec_i_batch;
common_prompt_checkpoint spec_ckpt;
bool spec_is_replay = false;
common_sampler_ptr spec_smpl_save;
// TODO: move members that belong to the task (such as `generated_text`, `has_new_line`) to task_results_state
// see https://github.com/ggml-org/llama.cpp/pull/18283#issuecomment-3710175837
@@ -360,7 +359,6 @@ struct server_slot {
spec_draft.clear();
spec_i_batch.clear();
spec_ckpt.clear();
spec_smpl_save.reset();
}
generated_tokens.clear();
generated_token_probs.clear();
@@ -3110,11 +3108,6 @@ private:
// update the batch with the sampled/drafted tokens
iterate(generating, [&](server_slot & slot) {
GGML_ASSERT(!slot.spec_smpl_save);
if (!slot.spec_draft.empty()) {
// backend sampling advances the sampler during llama_decode()
slot.spec_smpl_save.reset(common_sampler_clone(slot.smpl.get()));
}
slot.handle_last_sampled_token(batch);
});
@@ -3894,8 +3887,7 @@ private:
// verify and try to accept the draft
{
GGML_ASSERT(slot.spec_smpl_save);
common_sampler_ptr smpl_save = std::move(slot.spec_smpl_save);
common_sampler_ptr smpl_save(common_sampler_clone(slot.smpl.get()));
GGML_ASSERT(slot.spec_i_batch.size() == n_draft + 1);
auto accepted = common_sampler_sample_and_accept_n(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft);
@@ -3933,16 +3925,12 @@ private:
slot.mem.seq_rm(slot.id, ckpt.pos_max + 1, -1);
slot.prompt.tokens.keep_first(ckpt.n_tokens);
const bool restore_backend_sampler = slot.backend_sampling;
if (restore_backend_sampler) {
llama_set_sampler(slot.ctx_tgt, slot.id, nullptr);
if (slot.backend_sampling) {
slot.backend_sampling = llama_set_sampler(
slot.ctx_tgt, slot.id, common_sampler_get(smpl_save.get()));
}
slot.smpl = std::move(smpl_save);
if (restore_backend_sampler) {
slot.backend_sampling = llama_set_sampler(
slot.ctx_tgt, slot.id, common_sampler_get(slot.smpl.get()));
}
return;
}