mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-09-04 03:51:17 +02:00
Merge branch 'upstream' into concedo_experimental
# Conflicts: # README.md # docs/backend/SYCL.md # ggml/CMakeLists.txt # ggml/src/ggml-cpu/repack.cpp # ggml/src/ggml-sycl/backend.hpp # ggml/src/ggml-sycl/common.hpp # ggml/src/ggml-sycl/convert.cpp # ggml/src/ggml-sycl/cpy.cpp # ggml/src/ggml-sycl/cpy.hpp # ggml/src/ggml-sycl/dequantize.hpp # ggml/src/ggml-sycl/fattn.cpp # ggml/src/ggml-sycl/fattn.hpp # ggml/src/ggml-sycl/ggml-sycl.cpp # ggml/src/ggml-sycl/mmvq.cpp # ggml/src/ggml-sycl/norm.cpp # ggml/src/ggml-sycl/norm.hpp # ggml/src/ggml-sycl/vecdotq.hpp # ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp # ggml/src/ggml-webgpu/ggml-webgpu.cpp # ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl # ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl # ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_quant_staging.tmpl # ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl # ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl # ggml/src/ggml-zendnn/CMakeLists.txt # ggml/src/ggml-zendnn/ggml-zendnn.cpp # scripts/sync-ggml.last # tests/CMakeLists.txt # tests/snapshots/qwen3.6-27b.schema # tests/test-autorelease.cpp # tests/test-backend-ops.cpp # tests/test-backend-sampler.cpp # tests/test-model-load-cancel.cpp # tests/test-quant-type-selection.cpp
This commit is contained in:
+33
-10
@@ -482,6 +482,9 @@ llama_context::llama_context(
|
||||
}
|
||||
|
||||
llama_context::~llama_context() {
|
||||
// wait for any pending asynchronous copies into the output buffers before they are freed
|
||||
synchronize();
|
||||
|
||||
if (!model.hparams.no_alloc) {
|
||||
for (size_t i = 0; i < backend_ptrs.size(); ++i) {
|
||||
ggml_backend_t backend = backend_ptrs[i];
|
||||
@@ -1427,13 +1430,17 @@ int llama_context::encode(const llama_batch & batch_inp) {
|
||||
// micro-batching is not possible for non-causal encoding, so we process the batch in a single shot
|
||||
GGML_ASSERT(cparams.n_ubatch >= n_tokens && "encoder requires n_ubatch >= n_tokens");
|
||||
|
||||
// TODO: this clear of the buffer can easily be forgotten - need something better
|
||||
// sync first so any in-flight async copies into embd_seq complete before it is freed
|
||||
if (!embd_seq.empty()) {
|
||||
synchronize();
|
||||
}
|
||||
embd_seq.clear();
|
||||
|
||||
if (t_compute_start_us == 0) {
|
||||
t_compute_start_us = ggml_time_us();
|
||||
}
|
||||
|
||||
// TODO: this clear of the buffer can easily be forgotten - need something better
|
||||
embd_seq.clear();
|
||||
|
||||
sched_reserve();
|
||||
|
||||
n_queued_tokens += n_tokens;
|
||||
@@ -1772,13 +1779,18 @@ int llama_context::decode(const llama_batch & batch_inp) {
|
||||
|
||||
// GGML_ASSERT((cparams.causal_attn || cparams.n_ubatch >= n_tokens_all) && "non-causal attention requires n_ubatch >= n_tokens");
|
||||
|
||||
// TODO: this clear of the buffer can easily be forgotten - need something better
|
||||
// sync first so any in-flight async copies into embd_seq complete before it is freed
|
||||
if (!embd_seq.empty()) {
|
||||
synchronize();
|
||||
}
|
||||
embd_seq.clear();
|
||||
|
||||
if (t_compute_start_us == 0) {
|
||||
t_compute_start_us = ggml_time_us();
|
||||
}
|
||||
n_queued_tokens += n_tokens_all;
|
||||
|
||||
// TODO: this clear of the buffer can easily be forgotten - need something better
|
||||
embd_seq.clear();
|
||||
output_swaps.clear();
|
||||
|
||||
sched_reserve();
|
||||
@@ -3551,6 +3563,22 @@ llama_context * llama_init_from_model(
|
||||
}
|
||||
}
|
||||
|
||||
if ((model->hparams.is_mla() || model->arch == LLM_ARCH_DEEPSEEK4) && params.type_k != params.type_v) {
|
||||
LLAMA_LOG_ERROR("%s: model does not support different K (%s) and V (%s) cache types\n", __func__, ggml_type_name(params.type_k), ggml_type_name(params.type_v));
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
if (ggml_is_quantized(params.type_v) && params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_ENABLED) {
|
||||
if (params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO) {
|
||||
LLAMA_LOG_INFO("%s: enabling flash_attn since it is required for quantized V cache\n", __func__);
|
||||
params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED;
|
||||
}
|
||||
if (params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_DISABLED) {
|
||||
LLAMA_LOG_ERROR("%s: quantized V cache requires flash_attn to be enabled\n", __func__);
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
if (params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED && ggml_is_quantized(params.type_k)) {
|
||||
const uint32_t blck_size = ggml_blck_size(params.type_k);
|
||||
for (uint32_t il = 0; il < model->hparams.n_layer(); ++il) {
|
||||
@@ -3573,11 +3601,6 @@ llama_context * llama_init_from_model(
|
||||
}
|
||||
}
|
||||
|
||||
if (ggml_is_quantized(params.type_v) && params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_DISABLED) {
|
||||
LLAMA_LOG_ERROR("%s: V cache quantization requires flash_attn\n", __func__);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
if (params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED &&
|
||||
params.pooling_type != model->hparams.pooling_type) {
|
||||
//user-specified pooling-type is different from the model default
|
||||
|
||||
Reference in New Issue
Block a user