Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 01:05:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge commit '11b068d06605288ce7917534b46d52b47823dc13' into concedo_experimental

Browse Source 
# Conflicts:
#	CONTRIBUTING.md
#	docs/backend/SYCL.md
#	docs/install.md
#	docs/speculative.md
#	ggml/src/ggml-hip/CMakeLists.txt
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	ggml/src/ggml-sycl/common.hpp
#	ggml/src/ggml-sycl/element_wise.cpp
#	ggml/src/ggml-sycl/fattn-onednn.cpp
#	ggml/src/ggml-sycl/ggml-sycl.cpp
#	ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp
#	ggml/src/ggml-webgpu/ggml-webgpu.cpp
#	ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl
#	ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl
#	tests/test-backend-ops.cpp
#	tests/test-chat.cpp
#	tests/test-llama-archs.cpp
#	tools/cli/README.md
#	tools/llama-bench/llama-bench.cpp
#	tools/mtmd/CMakeLists.txt
#	tools/server/README.md
This commit is contained in:
Concedo
2026-07-31 17:06:19 +08:00
parent 49ff71991e 11b068d066
commit 98926f27c1
81 changed files with 5314 additions and 518 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-context.cpp
+30 -48
View File
@@ -164,6 +164,8 @@ struct server_slot {
llama_context * ctx_tgt = nullptr;
llama_context * ctx_dft = nullptr;
common_memory mem;
// multimodal
mtmd_context * mctx = nullptr;
mtmd::batch_ptr mbatch = nullptr;
@@ -253,10 +255,7 @@ struct server_slot {
void prompt_clear() {
SLT_TRC(*this, "clearing prompt with %zu tokens\n", prompt.tokens.size());
common_context_seq_rm(ctx_tgt, id, -1, -1);
if (ctx_dft) {
common_context_seq_rm(ctx_dft, id, -1, -1);
}
mem.seq_rm(id, -1, -1);
prompt.clear();
}
@@ -668,13 +667,8 @@ struct server_slot {
void copy_state_to(server_slot & other) const {
GGML_ASSERT(state == SLOT_STATE_DONE_PROMPT);
common_context_seq_rm(ctx_tgt, other.id, -1, -1);
common_context_seq_cp(ctx_tgt, id, other.id, -1, -1);
if (ctx_dft) {
common_context_seq_rm(ctx_dft, other.id, -1, -1);
common_context_seq_cp(ctx_dft, id, other.id, -1, -1);
}
mem.seq_rm(other.id, -1, -1);
mem.seq_cp(id, other.id, -1, -1);
other.n_decoded = n_decoded;
other.n_remaining = n_remaining;
@@ -1302,6 +1296,7 @@ private:
slot.id = i;
slot.ctx_tgt = ctx_tgt;
slot.ctx_dft = ctx_dft;
slot.mem.init(ctx_tgt, ctx_dft);
slot.spec = spec.get();
slot.n_ctx = n_ctx_slot;
@@ -1542,7 +1537,7 @@ private:
// find the slot that has at least n% prompt similarity
if (slot_prompt_similarity != 0.0f) {
float sim_best = 0;
float f_sim_best = 0;
for (server_slot & slot : slots) {
if (task.id_slot != -1 && slot.id != task.id_slot) {
@@ -1551,6 +1546,7 @@ private:
// skip the slot if it is not available
if (slot.is_processing()) {
SLT_TRC(slot, " - skipping, is_processing = %d\n", slot.is_processing());
continue;
}
@@ -1558,26 +1554,30 @@ private:
// skip the slot if it does not contains cached tokens
if (tokens.empty()) {
SLT_TRC(slot, "%s", " - skipping, slot is empty\n");
continue;
}
// fraction of the Longest Common Prefix length with respect to the input prompt length
const float sim_cur = float(tokens.get_common_prefix(task.tokens)) / task.tokens.size();
const size_t lcp_len = tokens.get_common_prefix(task.tokens);
const float f_sim_cur = float(lcp_len) / task.tokens.size();
SLT_TRC(slot, " - checking sim = %.3f (%zu/%zu) > %.3f\n", f_sim_cur, lcp_len, task.tokens.size(), slot_prompt_similarity);
// select the current slot if the criteria match
if (sim_cur > sim_best && sim_cur > slot_prompt_similarity) {
sim_best = sim_cur;
if (f_sim_cur > f_sim_best && f_sim_cur > slot_prompt_similarity) {
f_sim_best = f_sim_cur;
ret = &slot;
}
}
if (ret != nullptr) {
const float f_keep = (sim_best*task.tokens.size()) / ret->prompt.tokens.size();
const float f_keep = (f_sim_best*task.tokens.size()) / ret->prompt.tokens.size();
if (task.id_slot == -1) {
SLT_INF(*ret, "selected slot by LCP similarity, sim_best = %.3f (> %.3f thold), f_keep = %.3f\n",
sim_best, slot_prompt_similarity, f_keep);
SLT_INF(*ret, "selected slot by LCP similarity, f_sim_best = %.3f (> %.3f thold), f_keep = %.3f\n",
f_sim_best, slot_prompt_similarity, f_keep);
}
// if we are about to lose a large portion of the existing context - save it in the prompt cache
@@ -2881,13 +2881,8 @@ private:
SLT_WRN(slot, "slot context shift, n_keep = %d, n_left = %d, n_discard = %d\n", n_keep, n_left, n_discard);
common_context_seq_rm (ctx_tgt, slot.id, n_keep , n_keep + n_discard);
common_context_seq_add(ctx_tgt, slot.id, n_keep + n_discard, slot.prompt.n_tokens(), -n_discard);
if (ctx_dft) {
common_context_seq_rm (ctx_dft, slot.id, n_keep , n_keep + n_discard);
common_context_seq_add(ctx_dft, slot.id, n_keep + n_discard, slot.prompt.tokens.pos_next(), -n_discard);
}
slot.mem.seq_rm (slot.id, n_keep , n_keep + n_discard);
slot.mem.seq_add(slot.id, n_keep + n_discard, slot.prompt.tokens.pos_next(), -n_discard);
// add generated tokens to cache
// ref: https://github.com/ggml-org/llama.cpp/pull/16818#discussion_r2473269481
@@ -2998,7 +2993,9 @@ private:
ckpt.load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
}
common_context_seq_rm(ctx_dft, slot.id, ckpt.pos_max + 1, -1);
if (!llama_memory_seq_rm(llama_get_memory(ctx_dft), slot.id, ckpt.pos_max + 1, -1)) {
GGML_ABORT("failed to remove sequence %d\n", slot.id);
}
}
if (!draft.empty()) {
@@ -3201,13 +3198,8 @@ private:
const int64_t kv_shift = (int64_t) head_p - (int64_t) head_c;
common_context_seq_rm (ctx_tgt, slot.id, head_p, head_c);
common_context_seq_add(ctx_tgt, slot.id, head_c, head_c + n_match, kv_shift);
if (ctx_dft) {
common_context_seq_rm (ctx_dft, slot.id, head_p, head_c);
common_context_seq_add(ctx_dft, slot.id, head_c, head_c + n_match, kv_shift);
}
slot.mem.seq_rm (slot.id, head_p, head_c);
slot.mem.seq_add(slot.id, head_c, head_c + n_match, kv_shift);
for (size_t i = 0; i < n_match; i++) {
slot.prompt.tokens.set_token(head_p + i, slot.prompt.tokens[head_c + i]);
@@ -3379,10 +3371,7 @@ private:
SLT_TRC(slot, "cached n_tokens = %d, memory_seq_rm [%d, end)\n", slot.prompt.n_tokens(), p0);
common_context_seq_rm(ctx_tgt, slot.id, p0, -1);
if (ctx_dft) {
common_context_seq_rm(ctx_dft, slot.id, p0, -1);
}
slot.mem.seq_rm(slot.id, p0, -1);
// If using an alora, there may be uncached tokens that come
// before the invocation sequence. When this happens, the
@@ -3837,18 +3826,14 @@ private:
SLT_DBG(slot, "restoring speculative checkpoint (pos_min = %d, pos_max = %d, size = %zu)\n", ckpt.pos_min, ckpt.pos_max, ckpt.size());
{
ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
common_context_seq_rm(slot.ctx_tgt, slot.id, ckpt.pos_max + 1, -1);
}
ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
if (slot.ctx_dft) {
ckpt.load_dft(slot.ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
common_context_seq_rm(slot.ctx_dft, slot.id, ckpt.pos_max + 1, -1);
}
slot.mem.seq_rm(slot.id, ckpt.pos_max + 1, -1);
slot.prompt.tokens.keep_first(ckpt.n_tokens);
slot.smpl = std::move(smpl_save);
@@ -3889,10 +3874,7 @@ private:
slot.sampled = ids.back(); // last accepted token
SLT_DBG(slot, "add accepted tokens: sampled=%d, ids.size=%zu, n_draft=%zu\n", slot.sampled, ids.size(), n_draft);
common_context_seq_rm(slot.ctx_tgt, slot.id, slot.prompt.tokens.pos_next(), -1);
if (slot.ctx_dft) {
common_context_seq_rm(slot.ctx_dft, slot.id, slot.prompt.tokens.pos_next(), -1);
}
slot.mem.seq_rm(slot.id, slot.prompt.tokens.pos_next(), -1);
for (size_t i = 0; i < ids.size(); ++i) {
completion_token_output result;
tools/server/server-schema.cpp
+1
View File
@@ -209,6 +209,7 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
->set_hard_limits(0.0f, 1.0f)
->set_desc("Minimum speculative decoding probability for draft tokens (0 = greedy)"));
add((new field_str("speculative.type"))
->set_desc("Speculative decoding method (for debugging and research purposes)")
->set_handler([&](field_eval_context & ctx, const json & data) {
tools/server/server-task.cpp
+8 -6
View File
@@ -1742,9 +1742,9 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
const int lcp_best = prompt.tokens.get_common_prefix(tokens_new);
float f_keep_best = prompt.tokens.size() > 0 ? float(lcp_best) / prompt.tokens.size() : -1.0f; // empty slot: any cache entry wins
float sim_best = float(lcp_best) / tokens_new.size();
float f_sim_best = float(lcp_best) / tokens_new.size();
SRV_TRC(" - looking for better prompt, base f_keep = %.3f, sim = %.3f\n", f_keep_best, sim_best);
SRV_TRC(" - looking for better prompt, base f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best);
auto it_best = states.end();
@@ -1753,23 +1753,25 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
const int lcp_cur = it->prompt.tokens.get_common_prefix(tokens_new);
const float f_keep_cur = float(lcp_cur) / it->prompt.tokens.size();
const float sim_cur = float(lcp_cur) / tokens_new.size();
const float f_sim_cur = float(lcp_cur) / tokens_new.size();
SRV_TRC(" - prompt with length %7zu, lcp = %7d, f_keep = %.3f, f_sim = %.3f\n", it->prompt.tokens.size(), lcp_cur, f_keep_cur, f_sim_cur);
// don't trash large prompts
if (f_keep_cur < 0.25f) {
continue;
}
if (f_keep_best < f_keep_cur && sim_best < sim_cur) {
if (f_keep_best < f_keep_cur && f_sim_best < f_sim_cur) {
f_keep_best = f_keep_cur;
sim_best = sim_cur;
f_sim_best = f_sim_cur;
it_best = it;
}
}
if (it_best != states.end()) {
SRV_TRC(" - found better prompt with f_keep = %.3f, sim = %.3f\n", f_keep_best, sim_best);
SRV_TRC(" - found better prompt with f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best);
{
auto & data = it_best->data.main;
tools/server/server-task.h
+1 -1
View File
@@ -650,7 +650,7 @@ struct server_prompt_cache {
server_prompt_cache_state * alloc(const server_prompt & prompt, size_t state_size_main, size_t state_size_drft);
bool load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_main, llama_context * ctx_drft, int32_t id_slot);
bool load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot);
void update();
};