Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-03 11:35:48 +02:00
Code Issues Packages Projects Releases Wiki Activity

model: add DSpark support for Nemotron3.5 (#27804)

Browse Source 
* model: add DSpark support for Nemotron3.5

* Update src/models/dflash.cpp

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>

---------

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
This commit is contained in:
Ruixiang Wang
2026-08-28 01:49:27 +02:00
committed by GitHub
parent e70802a01f
commit ca3d5a3e10
6 changed files with 71 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/speculative.cpp
+17 -1
View File
@@ -935,6 +935,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
// dspark speculators
bool sample_from_anchor = true;
// block-internal attention
bool causal_attn = false;
const int32_t * target_layer_ids = nullptr; // model_dft's extract layer indices
uint32_t target_layer_ids_n = 0;
@@ -972,12 +975,25 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
if (llama_model_meta_val_str(model_dft, "dflash.sample_from_anchor", buf, sizeof(buf)) >= 0) {
sample_from_anchor = std::strcmp(buf, "true") == 0;
}
if (llama_model_meta_val_str(model_dft, "dflash.attention.causal", buf, sizeof(buf)) >= 0) {
causal_attn = std::strcmp(buf, "true") == 0;
}
}
selector_top_k = llama_model_dflash_selector_top_k(model_dft);
is_dflash2 = selector_top_k > 0;
mask_token_id = llama_vocab_mask(llama_model_get_vocab(model_dft));
if (is_dspark && this->params.p_min > 0.0f) {
char buf[16] = {};
const bool has_conf =
llama_model_meta_val_str(model_dft, "dflash.has_confidence_head", buf, sizeof(buf)) < 0 ||
std::strcmp(buf, "true") == 0;
if (!has_conf) {
throw std::runtime_error("DSpark draft has no confidence head: please set --spec-draft-p-min 0");
}
}
LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(type).c_str());
LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min);
LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u, sample_from_anchor=%s\n", __func__,
@@ -1036,7 +1052,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
// DFlash2 reads its selector lattice from h_nextn and never consumes raw logits.
llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ !is_dflash2);
llama_set_causal_attn(ctx_dft, false); // DFlash needs non-causal attention
llama_set_causal_attn(ctx_dft, causal_attn); // DFlash needs non-causal attention unless the model says otherwise
}
~common_speculative_impl_draft_dflash() override {