Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 09:15:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source 
# Conflicts:
#	.github/workflows/build-cache.yml
#	.github/workflows/build-cmake-pkg.yml
#	.github/workflows/build-cpu.yml
#	.github/workflows/build-cuda-windows.yml
#	.github/workflows/build-sanitize.yml
#	.github/workflows/release.yml
#	.github/workflows/winget.yml
#	CMakeLists.txt
#	README.md
#	app/llama.cpp
#	cmake/llama-config.cmake.in
#	cmake/llama.pc.in
#	common/CMakeLists.txt
#	common/build-info.h
#	docs/backend/OPENVINO.md
#	docs/backend/SYCL.md
#	docs/preset.md
#	ggml/CMakeLists.txt
#	ggml/cmake/ggml-config.cmake.in
#	ggml/src/ggml-cpu/kleidiai/kleidiai.cpp
#	ggml/src/ggml-hip/CMakeLists.txt
#	ggml/src/ggml-openvino/ggml-decoder.cpp
#	ggml/src/ggml-openvino/ggml-decoder.h
#	ggml/src/ggml-openvino/ggml-openvino-extra.cpp
#	ggml/src/ggml-openvino/ggml-openvino-extra.h
#	ggml/src/ggml-openvino/ggml-openvino.cpp
#	ggml/src/ggml-openvino/ggml-quants.cpp
#	ggml/src/ggml-openvino/ggml-quants.h
#	ggml/src/ggml-openvino/openvino/decoder.h
#	ggml/src/ggml-openvino/openvino/node_context.h
#	ggml/src/ggml-openvino/openvino/op/cpy.cpp
#	ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp
#	ggml/src/ggml-openvino/openvino/op/get_rows.cpp
#	ggml/src/ggml-openvino/openvino/op/l2_norm.cpp
#	ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp
#	ggml/src/ggml-openvino/openvino/op/repeat.cpp
#	ggml/src/ggml-openvino/openvino/op/reshape.cpp
#	ggml/src/ggml-openvino/openvino/op/rms_norm.cpp
#	ggml/src/ggml-openvino/openvino/op/rope.cpp
#	ggml/src/ggml-openvino/openvino/op/scale.cpp
#	ggml/src/ggml-openvino/openvino/op/set_rows.cpp
#	ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp
#	ggml/src/ggml-openvino/openvino/op/view.cpp
#	ggml/src/ggml-openvino/openvino/op_table.cpp
#	ggml/src/ggml-openvino/openvino/op_table.h
#	ggml/src/ggml-openvino/openvino/translate_session.cpp
#	ggml/src/ggml-openvino/openvino/utils.cpp
#	ggml/src/ggml-openvino/openvino/utils.h
#	ggml/src/ggml-openvino/utils.cpp
#	ggml/src/ggml-openvino/utils.h
#	ggml/src/ggml-sycl/common.hpp
#	ggml/src/ggml-sycl/concat.cpp
#	ggml/src/ggml-sycl/dmmv.cpp
#	ggml/src/ggml-sycl/element_wise.cpp
#	ggml/src/ggml-sycl/element_wise.hpp
#	ggml/src/ggml-sycl/fusion.cpp
#	ggml/src/ggml-sycl/fusion.hpp
#	ggml/src/ggml-sycl/gated_delta_net.cpp
#	ggml/src/ggml-sycl/gated_delta_net.hpp
#	ggml/src/ggml-sycl/ggml-sycl.cpp
#	ggml/src/ggml-sycl/mmvq.cpp
#	ggml/src/ggml-sycl/mmvq.hpp
#	scripts/sync-ggml.last
#	src/CMakeLists.txt
#	tests/test-backend-ops.cpp
#	tests/test-chat.cpp
#	tests/test-gguf.cpp
#	tests/test-quantize-stats.cpp
#	tools/cvector-generator/cvector-generator.cpp
#	tools/mtmd/CMakeLists.txt
#	tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte
This commit is contained in:
Concedo
2026-08-14 18:10:18 +08:00
parent da91226fbc 885c5bbe8e
commit 886a42d446
380 changed files with 5639 additions and 4009 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/speculative.cpp
+86
View File
@@ -2,6 +2,7 @@
#include "common.h"
#include "ggml.h"
#include "ggml-cpp.h"
#include "llama.h"
#include "log.h"
#include "ngram-cache.cpp"
@@ -912,6 +913,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
std::vector<common_sampler_ptr> smpls;
// backend sampler chain per seq, attached to ctx_dft
std::vector<llama_sampler *> backend_chains;
int32_t n_embd_dec = 0; // draft hidden size
int32_t n_embd_enc = 0; // target_layer_ids_n * target_hidden_size
int32_t n_embd_tgt = 0; // target model hidden size
@@ -985,6 +989,22 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
s.reset(common_sampler_init(model_dft, sparams));
}
// offload draft sampling to the backend
backend_chains.assign(n_seq, nullptr);
if (this->params.backend_sampling) {
for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) {
llama_sampler * chain = llama_sampler_chain_init(llama_sampler_chain_default_params());
llama_sampler_chain_add(chain, llama_sampler_init_top_k(10));
if (!llama_set_sampler(ctx_dft, seq_id, chain)) {
SPC_WRN("backend offload failed for seq_id=%d; using CPU sampler\n", (int) seq_id);
llama_sampler_free(chain);
chain = nullptr;
}
backend_chains[seq_id] = chain;
}
}
// turn on extraction of the target layers' input embeddings
for (uint32_t k = 0; k < target_layer_ids_n; ++k) {
llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true);
@@ -995,6 +1015,18 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
}
~common_speculative_impl_draft_dflash() override {
auto * ctx_dft = this->params.ctx_dft;
for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) backend_chains.size(); ++seq_id) {
if (backend_chains[seq_id] == nullptr) {
continue;
}
if (ctx_dft) {
llama_set_sampler(ctx_dft, seq_id, nullptr);
}
llama_sampler_free(backend_chains[seq_id]);
}
backend_chains.clear();
llama_batch_free(batch);
llama_batch_free(batch_inject);
}
@@ -2196,6 +2228,43 @@ common_speculative_type common_speculative_type_from_name(const std::string & na
return it->second;
}
std::vector<common_speculative_type> common_speculative_types_from_gguf(const std::string & path) {
struct gguf_init_params gguf_params = {
/* .no_alloc = */ true,
/* .ctx = */ nullptr,
};
gguf_context_ptr gguf_ctx(gguf_init_from_file(path.c_str(), gguf_params));
if (!gguf_ctx) {
return {};
}
const int64_t arch_id = gguf_find_key(gguf_ctx.get(), "general.architecture");
if (arch_id < 0 || gguf_get_kv_type(gguf_ctx.get(), arch_id) != GGUF_TYPE_STRING) {
return {};
}
const std::string arch = gguf_get_val_str(gguf_ctx.get(), arch_id);
if (arch != "dflash") {
const uint32_t block_count = gguf_get_val_u32(gguf_ctx.get(), gguf_find_key(gguf_ctx.get(), (arch + ".block_count").c_str()));
if (gguf_find_tensor(gguf_ctx.get(), ("blk." + std::to_string(block_count - 1) + ".nextn.eh_proj.weight").c_str()) >= 0) {
return { COMMON_SPECULATIVE_TYPE_DRAFT_MTP };
}
return {};
}
// the Markov head distinguishes draft-dspark from draft-dflash
const auto type = gguf_find_tensor(gguf_ctx.get(), "markov_w1.weight") >= 0
? COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK
: COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH;
SPC_INF("auto-detected speculative type '%s' from the draft model metadata\n", common_speculative_type_to_str(type).c_str());
return { type };
}
static uint32_t common_get_enabled_speculative_configs(const std::vector<common_speculative_type> & configs) {
uint32_t result = 0;
for (size_t i = 0; i < configs.size(); i++) {
@@ -2263,6 +2332,23 @@ common_params common_base_params_to_speculative(const common_params & params) {
result.n_outputs_max = params.n_parallel;
result.n_outputs_max_per_seq = 1;
// dflash/dspark decode the whole noise block in a single pass and sample every block position on the backend
// TODO: refactor such properties to be announced by the speculative types
// something like `struct common_speculative_type_props common_speculative_type_get_props(...);`
const bool has_block_draft = std::any_of(
params.speculative.types.begin(), params.speculative.types.end(),
[](common_speculative_type t) {
return t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK;
});
if (has_block_draft) {
// per-seq output positions: DFlash decodes anchor + n_max masks (n_max + 1); DSpark n_max -> +1 covers both
const int32_t per_seq = std::max(1, params_spec.n_max + 1);
result.n_outputs_max = params.n_parallel * per_seq;
if (params_spec.backend_sampling) {
result.n_outputs_max_per_seq = per_seq;
}
}
return result;
}