Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-08-30 09:00:49 +02:00
Code Issues Packages Projects Releases Wiki Activity

vulkan : dequant q8_0 KV once in coopmat1 (#25494)

Browse Source 
* vulkan : dequant q8_0 KV once in coopmat1

Assisted-by: Claude (Opus 4.8)

* vulkan : fall back instead of aborting when FA scratch exceeds maxStorageBufferRange

* vulkan : require KV-cache layout in FA dequant path

Assisted-by: Claude (Opus 4.8)

* vulkan : skip FA dequant path on coopmat2

Assisted-by: Claude (Opus 4.8)

* tests : add contiguously-allocated quant K/V FA tests

Assisted-by: Claude (Opus 4.8)

* vulkan : trim comments

* vulkan : tighten permutation checks for FA path

* vulkan : set prealloc_x_need_sync after the FA dispatch

* vulkan : exclude Intel Xe1 from FA dequant path
This commit is contained in:
Nathanw1014
2026-08-20 01:14:15 +09:30
committed by GitHub
parent b95502ba9a
commit dc72703fc6
4 changed files with 101 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-backend-ops.cpp
+13 -5
View File
@@ -7084,9 +7084,10 @@ struct test_flash_attn_ext : public test_case {
const ggml_type type_K;
const ggml_type type_V;
std::array<int32_t, 4> permute;
const bool kv_view; // create K/V as views of a larger buffer (like a KV cache)
std::string vars() override {
return VARS_TO_STR14(hsk, hsv, nh, nr23, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_K, type_V, permute);
return VARS_TO_STR15(hsk, hsv, nh, nr23, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_K, type_V, permute, kv_view);
}
double max_nmse_err() override {
@@ -7102,9 +7103,10 @@ struct test_flash_attn_ext : public test_case {
test_flash_attn_ext(int64_t hsk = 128, int64_t hsv = 128, int64_t nh = 32, std::array<int64_t, 2> nr23 = {1, 1}, int64_t kv = 96, int64_t nb = 8,
bool mask = true, bool sinks = false, float max_bias = 0.0f, float logit_softcap = 0.0f, ggml_prec prec = GGML_PREC_F32,
ggml_type type_K = GGML_TYPE_F16, ggml_type type_V = GGML_TYPE_F16, std::array<int32_t, 4> permute = {0, 1, 2, 3})
ggml_type type_K = GGML_TYPE_F16, ggml_type type_V = GGML_TYPE_F16, std::array<int32_t, 4> permute = {0, 1, 2, 3},
bool kv_view = true)
: hsk(hsk), hsv(hsv), nh(nh), nr23(nr23), kv(kv), nb(nb), mask(mask), sinks(sinks), max_bias(max_bias), logit_softcap(logit_softcap), prec(prec),
type_K(type_K), type_V(type_V), permute(permute) {}
type_K(type_K), type_V(type_V), permute(permute), kv_view(kv_view) {}
ggml_tensor * build_graph(ggml_context * ctx) override {
const int64_t hsk_padded = GGML_PAD(hsk, ggml_blck_size(type_K));
@@ -7132,7 +7134,7 @@ struct test_flash_attn_ext : public test_case {
ggml_tensor * q = create_permuted(GGML_TYPE_F32, hsk_padded, nb, nh*nr23[0], nr23[1], false);
ggml_set_name(q, "q");
ggml_tensor * k = create_permuted(type_K, hsk_padded, kv, nh, nr23[1], true); // the K tensor is usually a view of the K cache
ggml_tensor * k = create_permuted(type_K, hsk_padded, kv, nh, nr23[1], kv_view); // the K tensor is usually a view of the K cache
ggml_set_name(k, "k");
ggml_tensor * v = nullptr;
@@ -7146,7 +7148,7 @@ struct test_flash_attn_ext : public test_case {
// - https://github.com/ggml-org/llama.cpp/pull/18986
v = ggml_view_4d(ctx, k, hsv_padded, kv, nh, nr23[1], k->nb[1], k->nb[2], k->nb[3], 0);
} else {
v = create_permuted(type_V, hsv_padded, kv, nh, nr23[1], true); // the V tensor is usually a view of the V cache
v = create_permuted(type_V, hsv_padded, kv, nh, nr23[1], kv_view); // the V tensor is usually a view of the V cache
}
ggml_set_name(v, "v");
@@ -9941,6 +9943,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
}
// dense-allocated (non-view) quant K/V at batch >= 64, in cache and native layouts
test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false));
test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {4, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false));
test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 1024, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false));
test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, false));
test_cases.emplace_back(new test_cross_entropy_loss (GGML_TYPE_F32, { 10, 5, 4, 3}));
test_cases.emplace_back(new test_cross_entropy_loss (GGML_TYPE_F32, {30000, 1, 1, 1}));
test_cases.emplace_back(new test_cross_entropy_loss_back(GGML_TYPE_F32, { 10, 5, 4, 3}));