Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-17 08:19:49 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge commit '34af94cd9ab277632e27caeec2d41de2fd091b31' into concedo_experimental

Browse Source 
# Conflicts:
#	.github/workflows/docker.yml
#	.github/workflows/make-release.yml
#	.github/workflows/release.yml
#	.pi/gg/SYSTEM.md
#	CMakeLists.txt
#	build-xcframework.sh
#	docs/development/HOWTO-add-model.md
#	docs/ops.md
#	docs/ops/SYCL.csv
#	docs/speculative.md
#	examples/sycl/update-ops-doc.sh
#	ggml/CMakeLists.txt
#	ggml/src/ggml-sycl/cpy.cpp
#	ggml/src/ggml-sycl/ggml-sycl.cpp
#	scripts/make-release-checks.sh
#	scripts/sync-ggml.last
#	tests/test-chat-auto-parser.cpp
#	tests/test-chat.cpp
#	tests/test-jinja.cpp
#	tests/test-llama-archs.cpp
#	tests/testing.h
#	tools/llama-bench/llama-bench.cpp
#	tools/server/README.md
This commit is contained in:
Concedo
2026-08-20 17:08:46 +08:00
parent 1fc6e11ab8 34af94cd9a
commit f0cd0225aa
135 changed files with 3116 additions and 312 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/speculative.cpp
+15 -8
View File
@@ -926,6 +926,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
// draft-dspark: the draft carries a Markov head and uses an anchor-first block layout
const bool is_dspark;
// dspark speculators
bool sample_from_anchor = true;
const int32_t * target_layer_ids = nullptr; // model_dft's extract layer indices
uint32_t target_layer_ids_n = 0;
@@ -960,16 +963,20 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
if (llama_model_meta_val_str(model_dft, "dflash.block_size", buf, sizeof(buf)) >= 0) {
block_size = std::atoi(buf);
}
if (llama_model_meta_val_str(model_dft, "dflash.sample_from_anchor", buf, sizeof(buf)) >= 0) {
sample_from_anchor = std::strcmp(buf, "true") == 0;
}
}
mask_token_id = llama_vocab_mask(llama_model_get_vocab(model_dft));
LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(type).c_str());
LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min);
LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u\n", __func__, block_size, mask_token_id, target_layer_ids_n);
LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u, sample_from_anchor=%s\n", __func__,
block_size, mask_token_id, target_layer_ids_n, sample_from_anchor ? "true" : "false");
// DFlash input is [id_last, <mask> * (block_size-1)]: in-place denoising yields at most
// block_size-1 draft tokens, DSpark yield a full block_size draft tokens
const int32_t n_draft_max = is_dspark ? block_size : block_size - 1;
// block_size-1 draft tokens, anchor-first DSpark yields a full block_size draft tokens
const int32_t n_draft_max = is_dspark && sample_from_anchor ? block_size : block_size - 1;
if (this->params.n_max > n_draft_max || this->params.n_min > n_draft_max) {
LOG_WRN("%s: requested draft size (n_max=%d, n_min=%d) exceeds the trained block size %d -- clamping to %d\n",
__func__, this->params.n_max, this->params.n_min, block_size, n_draft_max);
@@ -1175,7 +1182,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
const int32_t n_draft = params.n_max;
const int32_t n_block_tokens = n_draft + (is_dspark ? 0 : 1);
const int32_t n_block_tokens = n_draft + (is_dspark && sample_from_anchor ? 0 : 1);
i_block_beg[seq_id] = batch.n_tokens;
n_block [seq_id] = n_block_tokens;
for (int32_t i = 0; i < n_block_tokens; ++i) {
@@ -1208,11 +1215,11 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
auto & result = *dp.result;
if (is_dspark) {
// DSpark predicts the next token from position 0 and optionally truncates
// at the first position below the confidence threshold.
// DSpark: read from the first draft slot, truncate below the confidence threshold
const float * conf = params.p_min > 0.0f ? llama_get_embeddings_nextn(ctx_dft) : nullptr;
for (int32_t i = 0; i < n_block_tokens; ++i) {
// bonus-anchor drafts read the mask positions only, like DFlash
const int32_t i_draft_beg = sample_from_anchor ? 0 : 1;
for (int32_t i = i_draft_beg; i < n_block_tokens; ++i) {
const int32_t idx = beg + i;
if (conf && conf[(size_t) idx * n_embd_dec] < params.p_min) {