From b28ad7ecca781fc74586b5b0f49e72abce389e43 Mon Sep 17 00:00:00 2001 From: HimariO Date: Mon, 7 Apr 2025 22:07:56 +0800 Subject: [PATCH] fix attn weight scaling after rebase --- examples/llava/clip.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp index a7fe830e0..b6edfd086 100644 --- a/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp @@ -778,7 +778,7 @@ static ggml_cgraph * clip_image_build_graph_legacy(clip_ctx * ctx, const clip_im if (full_attn) { KQ = ggml_soft_max_ext(ctx0, KQ, nullptr, 1.0f / sqrtf((float)d_head), 0.0f); } else { - KQ = ggml_soft_max_ext(ctx0, KQ, window_mask, 1.0f, 0.0f); + KQ = ggml_soft_max_ext(ctx0, KQ, window_mask, 1.0f / sqrtf((float)d_head), 0.0f); } struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ);