From 9a4843cf2f1a3fc8e39f8148e92ee6bfe18e2db6 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Fri, 4 Sep 2026 06:36:51 +0200 Subject: [PATCH] src : add n_expert_used_max function (#28323) * src : add n_expert_used_max function With Commit c61b98b875eaa5e654a3f5c73b34c310d2c6ab4c ("model: add NVIDIA Nemotron-3-Puzzle-75B-A9B (NemotronHPuzzle) support (#25444)") it is now possible for each layer to have a specific number of experts but there are a few checks that need to be updated to handle this upon model loading. For example: ```console llama_model_load: error loading model: model has expert layers but no expert layers are used ``` And later: ```console /llama.cpp/src/llama-model-loader.cpp:955: GGML_ASSERT(n_ids_used > 0) failed ``` This commit adds the n_expert_used_max function so that these checks can use it. Refs: https://github.com/ggml-org/llama.cpp/pull/25444#issuecomment-5524976031 * src : use hparams.n_expert_used_max in llama_model_base::load_hparams * src : use 0 as initial value for n_expert_used_max --- src/llama-hparams.cpp | 9 +++++++++ src/llama-hparams.h | 3 +++ src/llama-model-loader.cpp | 4 ++-- src/llama-model.cpp | 10 +++------- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp index 7df82ffe2..34b3c6880 100644 --- a/src/llama-hparams.cpp +++ b/src/llama-hparams.cpp @@ -87,6 +87,15 @@ uint32_t llama_hparams::n_expert_used(uint32_t il) const { GGML_ABORT("fatal error"); } +uint32_t llama_hparams::n_expert_used_max() const { + uint32_t val = 0; + for (uint32_t il = 0; il < n_layer_all; ++il) { + val = std::max(val, n_expert_used(il)); + } + + return val; +} + uint32_t llama_hparams::n_gqa(uint32_t il) const { const uint32_t n_head = this->n_head(il); const uint32_t n_head_kv = this->n_head_kv(il); diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 2f238a174..e9029ff34 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -392,6 +392,9 @@ struct llama_hparams { uint32_t n_expert_used(uint32_t il = 0) const; + // return the maximum n_expert_used across all layers + uint32_t n_expert_used_max() const; + uint32_t n_gqa(uint32_t il = 0) const; uint32_t n_rot(uint32_t il = 0) const; diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 49f3c4f8e..91bb5e7cc 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -951,7 +951,7 @@ static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * w case GGML_OP_MUL_MAT_ID: { // Used for either MoE expert routing or embedded adapter routing - const int n_ids_used = hparams.router_layer >= 0 ? 1 : hparams.n_expert_used(); + const int n_ids_used = hparams.router_layer >= 0 ? 1 : hparams.n_expert_used_max(); GGML_ASSERT(n_ids_used > 0); ggml_tensor * b = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_ids_used, 512); ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_ids_used, 512); @@ -964,7 +964,7 @@ static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * w } break; case GGML_OP_ADD_ID: { - const int n_expert_used = hparams.n_expert_used(); + const int n_expert_used = hparams.n_expert_used_max(); GGML_ASSERT(n_expert_used > 0); ggml_tensor * a = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_expert_used, 512); ggml_tensor * c = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_expert_used, 512); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 6344f2d8a..f22e35ed9 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1254,10 +1254,7 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { } // models may route a different number of experts per layer, so validate the maximum - uint32_t n_expert_used_max = 0; - for (uint32_t il = 0; il < hparams.n_layer_all; ++il) { - n_expert_used_max = std::max(n_expert_used_max, hparams.n_expert_used(il)); - } + uint32_t n_expert_used_max = hparams.n_expert_used_max(); GGML_ASSERT(hparams.n_expert <= LLAMA_MAX_EXPERTS); GGML_ASSERT(n_expert_used_max <= hparams.n_expert); @@ -1509,10 +1506,9 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { // TODO: move to a separate function const auto tn = LLM_TN(arch); - const int64_t n_expert = hparams.n_expert; - const int64_t n_expert_used = hparams.n_expert_used(); + const int64_t n_expert = hparams.n_expert; - if (n_expert > 0 && n_expert_used == 0) { + if (n_expert > 0 && hparams.n_expert_used_max() == 0) { throw std::runtime_error("model has expert layers but no expert layers are used"); }