mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-09-09 22:39:13 +02:00
Merge branch 'upstream' into concedo_experimental
# Conflicts: # .github/workflows/build-webgpu.yml # CMakeLists.txt # common/CMakeLists.txt # docs/development/HOWTO-add-model.md # ggml/src/ggml-opencl/ggml-opencl.cpp # ggml/src/ggml-sycl/CMakeLists.txt # tests/test-arg-parser.cpp # tests/test-jinja.cpp # tests/test-llama-archs.cpp # tests/test-save-load-state.cpp # tools/cli/README.md # tools/completion/README.md # tools/llama-bench/llama-bench.cpp # tools/mtmd/CMakeLists.txt # tools/server/README.md
This commit is contained in:
+12
-1
@@ -1710,6 +1710,17 @@ ggml_tensor * llm_graph_context::build_ffn(
|
||||
cur = ggml_swiglu(ctx0, cur);
|
||||
cb(cur, "ffn_swiglu", il);
|
||||
} break;
|
||||
case LLM_FFN_SWIGLU_OAI_MOE:
|
||||
if (gate && type_gate == LLM_FFN_PAR) {
|
||||
// same alpha/limit constants as gpt-oss
|
||||
const float alpha = 1.702f;
|
||||
const float limit = 7.0f;
|
||||
cur = ggml_swiglu_oai(ctx0, cur, tmp, alpha, limit);
|
||||
cb(cur, "ffn_swiglu_oai", il);
|
||||
type_gate = LLM_FFN_SEQ;
|
||||
} else {
|
||||
GGML_ABORT("LLM_FFN_SWIGLU_OAI_MOE requires a parallel gate");
|
||||
} break;
|
||||
case LLM_FFN_GEGLU:
|
||||
{
|
||||
cur = ggml_geglu(ctx0, cur);
|
||||
@@ -2669,7 +2680,7 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il));
|
||||
}
|
||||
|
||||
const auto & kq_mask = inp->get_kq_mask();
|
||||
ggml_tensor * kq_mask = inp->get_kq_mask();
|
||||
|
||||
ggml_tensor * q = q_cur;
|
||||
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
|
||||
|
||||
Reference in New Issue
Block a user