diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp
index 7df82ffe2..34b3c6880 100644
--- a/src/llama-hparams.cpp
+++ b/src/llama-hparams.cpp
@@ -87,6 +87,15 @@ uint32_t llama_hparams::n_expert_used(uint32_t il) const {
     GGML_ABORT("fatal error");
 }
 
+uint32_t llama_hparams::n_expert_used_max() const {
+    uint32_t val = 0;
+    for (uint32_t il = 0; il < n_layer_all; ++il) {
+        val = std::max(val, n_expert_used(il));
+    }
+
+    return val;
+}
+
 uint32_t llama_hparams::n_gqa(uint32_t il) const {
     const uint32_t n_head    = this->n_head(il);
     const uint32_t n_head_kv = this->n_head_kv(il);
diff --git a/src/llama-hparams.h b/src/llama-hparams.h
index 2f238a174..e9029ff34 100644
--- a/src/llama-hparams.h
+++ b/src/llama-hparams.h
@@ -392,6 +392,9 @@ struct llama_hparams {
 
     uint32_t n_expert_used(uint32_t il = 0) const;
 
+    // return the maximum n_expert_used across all layers
+    uint32_t n_expert_used_max() const;
+
     uint32_t n_gqa(uint32_t il = 0) const;
 
     uint32_t n_rot(uint32_t il = 0) const;
diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp
index 49f3c4f8e..91bb5e7cc 100644
--- a/src/llama-model-loader.cpp
+++ b/src/llama-model-loader.cpp
@@ -951,7 +951,7 @@ static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * w
         case GGML_OP_MUL_MAT_ID:
             {
                 // Used for either MoE expert routing or embedded adapter routing
-                const int n_ids_used = hparams.router_layer >= 0 ? 1 : hparams.n_expert_used();
+                const int n_ids_used = hparams.router_layer >= 0 ? 1 : hparams.n_expert_used_max();
                 GGML_ASSERT(n_ids_used > 0);
                 ggml_tensor * b = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_ids_used, 512);
                 ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_ids_used, 512);
@@ -964,7 +964,7 @@ static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * w
             } break;
         case GGML_OP_ADD_ID:
             {
-                const int n_expert_used = hparams.n_expert_used();
+                const int n_expert_used = hparams.n_expert_used_max();
                 GGML_ASSERT(n_expert_used > 0);
                 ggml_tensor * a = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_expert_used, 512);
                 ggml_tensor * c = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_expert_used, 512);
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 6344f2d8a..f22e35ed9 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -1254,10 +1254,7 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
     }
 
     // models may route a different number of experts per layer, so validate the maximum
-    uint32_t n_expert_used_max = 0;
-    for (uint32_t il = 0; il < hparams.n_layer_all; ++il) {
-        n_expert_used_max = std::max(n_expert_used_max, hparams.n_expert_used(il));
-    }
+    uint32_t n_expert_used_max = hparams.n_expert_used_max();
 
     GGML_ASSERT(hparams.n_expert <= LLAMA_MAX_EXPERTS);
     GGML_ASSERT(n_expert_used_max <= hparams.n_expert);
@@ -1509,10 +1506,9 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
         // TODO: move to a separate function
         const auto tn = LLM_TN(arch);
 
-        const int64_t n_expert      = hparams.n_expert;
-        const int64_t n_expert_used = hparams.n_expert_used();
+        const int64_t n_expert = hparams.n_expert;
 
-        if (n_expert > 0 && n_expert_used == 0) {
+        if (n_expert > 0 && hparams.n_expert_used_max() == 0) {
             throw std::runtime_error("model has expert layers but no expert layers are used");
         }