mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-09-01 10:00:51 +02:00
Merge branch 'upstream' into concedo_experimental
# Conflicts: # docs/backend/SYCL.md # ggml/src/ggml-cpu/ggml-cpu.c # ggml/src/ggml-opencl/CMakeLists.txt # ggml/src/ggml-opencl/ggml-opencl.cpp # ggml/src/ggml-opencl/kernels/cvt.cl # ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl # ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl # ggml/src/ggml-sycl/backend.hpp # ggml/src/ggml-sycl/common.hpp # ggml/src/ggml-sycl/dmmv.cpp # ggml/src/ggml-sycl/ggml-sycl.cpp # scripts/sync_vendor.py # tests/CMakeLists.txt # tests/test-alloc.cpp # tests/test-backend-ops.cpp # tests/test-chat.cpp # tests/test-gguf.cpp # tests/test-save-load-state.cpp
This commit is contained in:
@@ -58,6 +58,12 @@ static const llm_fused_op_probe llm_fused_op_gdn_ch_probe = {
|
||||
/*.n_tokens_per_seq =*/ 16,
|
||||
};
|
||||
|
||||
static const llm_fused_op_probe llm_fused_op_lid_probe = {
|
||||
/*.op =*/ LLM_FUSED_OP_LIGHTNING_INDEXER,
|
||||
/*.name =*/ "Lightning Indexer",
|
||||
/*.n_tokens_per_seq =*/ 1,
|
||||
};
|
||||
|
||||
llama_context::llama_context(
|
||||
const llama_model & model,
|
||||
llama_context_params params) :
|
||||
@@ -229,6 +235,9 @@ llama_context::llama_context(
|
||||
cparams.fused_gdn_ch = true;
|
||||
cparams.auto_fgdn = true;
|
||||
|
||||
cparams.fused_lid = true;
|
||||
cparams.auto_flid = true;
|
||||
|
||||
// with causal attention, the batch size is limited by the context size
|
||||
cparams.n_batch = cparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch;
|
||||
|
||||
@@ -530,6 +539,12 @@ void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint3
|
||||
resolve(llm_fused_op_gdn_ch_probe, cparams.fused_gdn_ch);
|
||||
cparams.auto_fgdn = false;
|
||||
}
|
||||
|
||||
if (cparams.auto_flid) {
|
||||
LLAMA_LOG_INFO("%s: resolving fused Lightning Indexer support:\n", func);
|
||||
resolve(llm_fused_op_lid_probe, cparams.fused_lid);
|
||||
cparams.auto_flid = false;
|
||||
}
|
||||
}
|
||||
|
||||
void llama_context::sched_reserve() {
|
||||
|
||||
Reference in New Issue
Block a user