From e750b887a82719c27200b71545f63ed78ec24719 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 2 Sep 2026 19:19:54 +0300 Subject: [PATCH 01/17] common, server : enable preserve_reasoning kwarg by default, log its effective state (#28174) * common, server : enable preserve_reasoning kwarg by default, log its effective state If the preserve_reasoning chat template kwarg is not specified explicitly via --reasoning-preserve / --no-reasoning-preserve, it is enabled by default after argument processing. The server logs the effective state of the kwarg, warns that it is enabled by default when the template supports it, and only warns "has no effect" when it was enabled explicitly on a template that does not support it. Setting the kwarg via --chat-template-kwargs is deprecated. Assisted-by: pi:llama.cpp/Qwen3.8-27B * cont : update comment Co-authored-by: Xuan-Son Nguyen --------- Co-authored-by: Xuan-Son Nguyen --- common/arg.cpp | 12 +++++++++++- common/common.h | 3 ++- tools/server/server-context.cpp | 15 +++++++++++++-- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 79405b59e..2669cacd6 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -960,6 +960,11 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context )); } + // if the preserve_reasoning kwarg was not specified explicitly, enable it by default + if (!params.default_template_kwargs.count("preserve_reasoning")) { + params.default_template_kwargs["preserve_reasoning"] = "true"; + } + return true; } @@ -3553,6 +3558,10 @@ common_params_context common_params_parser_init(common_params & params, llama_ex LOG_WRN("Setting 'enable_thinking' via --chat-template-kwargs is deprecated. " "Use --reasoning on / --reasoning off instead.\n"); } + if (item.key() == "preserve_reasoning") { + LOG_WRN("Setting 'preserve_reasoning' via --chat-template-kwargs is deprecated. " + "Use --reasoning-preserve / --no-reasoning-preserve instead.\n"); + } params.default_template_kwargs[item.key()] = item.value().dump(); } } @@ -3743,7 +3752,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex add_opt(common_arg( {"--reasoning-preserve"}, {"--no-reasoning-preserve"}, - "preserve reasoning trace in the full history, not just the last assistant message (default: template default)\n" + "preserve reasoning trace in the full history, not just the last assistant message (default: enabled)\n" "compatible with certain templates having 'supports_preserve_reasoning' capability\n" "example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking", [](common_params & params, bool value) { @@ -3752,6 +3761,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } else { params.default_template_kwargs["preserve_reasoning"] = "false"; } + params.preserve_reasoning_specified = true; } ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_REASONING_PRESERVE")); add_opt(common_arg( diff --git a/common/common.h b/common/common.h index 4e9448bb1..63d0badd0 100644 --- a/common/common.h +++ b/common/common.h @@ -270,7 +270,7 @@ struct common_params_sampling { COMMON_SAMPLER_TYPE_TEMPERATURE, }; - common_grammar grammar; // optional grammar constraint (user / output-format / tool-calls) + common_grammar grammar; // optional grammar constraint (user / output-format / tool-calls) bool grammar_lazy = false; std::vector grammar_triggers; // optional triggers (for lazy grammars) std::set preserved_tokens; @@ -657,6 +657,7 @@ struct common_params { std::string ssl_file_cert = ""; // NOLINT std::map default_template_kwargs; + bool preserve_reasoning_specified = false; // CLI params std::string server_base; // if set, connect to this server instead of starting a new one diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index f5477356d..f78cfb36d 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1493,11 +1493,22 @@ private: auto caps = common_chat_templates_get_caps(chat_params.tmpls.get()); auto it = params_base.default_template_kwargs.find("preserve_reasoning"); bool supported = caps.at("supports_preserve_reasoning"); - bool enabled = it != params_base.default_template_kwargs.end(); + bool specified = params_base.preserve_reasoning_specified; + // note: the kwarg is enabled by default if not specified explicitly, so check the value + bool enabled = it != params_base.default_template_kwargs.end() && it->second == "true"; + if (supported) { + SRV_TRC("preserve_reasoning kwarg: %s\n", + it == params_base.default_template_kwargs.end() ? "unset (template default)" : it->second.c_str()); + } else { + SRV_TRC("%s", "preserve_reasoning kwarg: not supported by template\n"); + } + if (supported && !specified) { + SRV_WRN("%s", "chat template supports preserving reasoning, it is enabled by default (may use more tokens, disable via --no-reasoning-preserve)\n"); + } if (supported && !enabled) { SRV_INF("%s", "chat template supports preserving reasoning, consider enabling it via --reasoning-preserve\n"); } - if (!supported && enabled) { + if (!supported && specified && enabled) { SRV_WRN("%s", "chat template does NOT support preserving reasoning, --reasoning-preserve has no effect\n"); } } From 3d3d7c81813067fc8c185da017e0af03b4269b1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Gallou=C3=ABt?= Date: Wed, 2 Sep 2026 18:54:11 +0200 Subject: [PATCH 02/17] ggml-cuda : remove unused vars (#28235) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Adrien Gallouët --- ggml/src/ggml-cuda/mmq-vec-dot.cuh | 11 ----------- ggml/src/ggml-cuda/mmq.cuh | 5 ----- 2 files changed, 16 deletions(-) diff --git a/ggml/src/ggml-cuda/mmq-vec-dot.cuh b/ggml/src/ggml-cuda/mmq-vec-dot.cuh index d57343386..4d1c398fc 100644 --- a/ggml/src/ggml-cuda/mmq-vec-dot.cuh +++ b/ggml/src/ggml-cuda/mmq-vec-dot.cuh @@ -148,7 +148,6 @@ static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma( typedef tile<16, 8, int, input_layout> tile_B; typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. @@ -204,7 +203,6 @@ static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma( typedef tile< 8, 8, int> tile_B; typedef tile<16, 8, int> tile_C; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. @@ -320,7 +318,6 @@ template static __device__ __forceinline_ typedef tile<16, 8, int, input_layout> tile_B; typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. @@ -371,7 +368,6 @@ template static __device__ __forceinline_ typedef tile< 8, 8, int> tile_B; typedef tile<16, 8, int> tile_C; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. @@ -486,7 +482,6 @@ template static __device__ __forceinline_ typedef tile<16, 4, int, input_layout> tile_B; typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. @@ -537,7 +532,6 @@ template static __device__ __forceinline_ typedef tile< 8, 4, int> tile_B; typedef tile<16, 8, int> tile_C; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. @@ -686,7 +680,6 @@ template static __device__ __forceinline_ typedef tile<16, 4, int, input_layout> tile_B; typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. @@ -756,7 +749,6 @@ template static __device__ __forceinline_ typedef tile< 8, 4, int> tile_B; typedef tile<16, 8, int> tile_C; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. @@ -1023,7 +1015,6 @@ template static __device__ __forceinline_ typedef tile<16, 4, int, input_layout> tile_B; typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. @@ -1075,7 +1066,6 @@ template static __device__ __forceinline_ typedef tile< 8, 4, int> tile_B; typedef tile<16, 8, int> tile_C; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. @@ -1190,7 +1180,6 @@ template static __device__ __forceinline_ typedef tile<8, 8, int> tile_B; typedef tile<16, 8, float> tile_C; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); constexpr int ntx = rows_per_warp / tile_C::I; diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index c978b4421..b4a747720 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -481,9 +481,6 @@ static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma( typedef tile<16, 8, int> tile_C; #endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. @@ -540,8 +537,6 @@ struct ggml_cuda_mmq_util_funcs { template static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_funcs() { - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - if (!ggml_cuda_mmq_get_config(type, J, fallback).use_mma_data_layout()) { switch (type) { case GGML_TYPE_Q1_0: From d5fec32a87e8ea35d00fba50813f82a37f13b399 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Wed, 2 Sep 2026 19:13:20 +0200 Subject: [PATCH 03/17] ci : enable hf-jobs on server-cuda (#28258) --- .github/workflows/server-self-hosted.yml | 34 ++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/.github/workflows/server-self-hosted.yml b/.github/workflows/server-self-hosted.yml index 675ddbaaa..d9ad2fcd0 100644 --- a/.github/workflows/server-self-hosted.yml +++ b/.github/workflows/server-self-hosted.yml @@ -102,7 +102,7 @@ jobs: ./tests.sh server-cuda: - runs-on: [self-hosted, llama-server, Linux, NVIDIA] + runs-on: "hf-jobs-t4-small:cuda13" steps: - name: Clone @@ -112,12 +112,42 @@ jobs: fetch-depth: 0 ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }} + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y cmake libssl-dev python3 python3-venv python3-pip + + - name: ccache + uses: ggml-org/ccache-action@v1.2.24 + with: + restore: false + save: false + + - name: ccache-buckets-restore + uses: ./.github/actions/ccache-buckets + with: + key: self-hosted-server-cuda + folder: llama.cpp + hf_bucket: ggml-org/cache + - name: Build id: cmake_build run: | - cmake -B build -DGGML_CUDA=ON -DGGML_SCHED_NO_REALLOC=ON + cmake -B build -DGGML_CUDA=ON -DGGML_SCHED_NO_REALLOC=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc cmake --build build --config Release -j $(nproc) --target llama-server + - name: ccache-buckets-save + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + uses: ./.github/actions/ccache-buckets + env: + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} + with: + key: self-hosted-server-cuda + folder: llama.cpp + evict-old-files: 1d + hf_bucket: ggml-org/cache + save: true + - name: Python setup id: setup_python run: | From 9400c8946e4da5e7694f2c26d6d4e50e14b690fa Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Wed, 2 Sep 2026 19:14:46 +0200 Subject: [PATCH 04/17] model: correctly support input vision for deepseek4 (#28154) * model: correctly support input vision for deepseek4 * nits --- conversion/deepseek.py | 8 ++++++-- gguf-py/gguf/constants.py | 3 +++ src/llama-arch.cpp | 2 ++ src/llama-arch.h | 1 + src/llama-hparams.h | 4 ++++ src/llama-kv-cache.cpp | 4 +++- src/llama-model.h | 1 + src/models/deepseek4.cpp | 14 +++++++++++++- tools/mtmd/mtmd.cpp | 1 + 9 files changed, 34 insertions(+), 4 deletions(-) diff --git a/conversion/deepseek.py b/conversion/deepseek.py index c244e94ec..abaed7880 100644 --- a/conversion/deepseek.py +++ b/conversion/deepseek.py @@ -578,8 +578,7 @@ class DeepseekV4Model(TextModel): @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, gen = item - if (name.startswith(("aligner.", "image_")) - or name.endswith(".ffn.gate.bias_vl")): + if name.startswith(("aligner.", "image_")): return None if name.startswith("mtp."): if not cls.mtp_only: @@ -856,6 +855,7 @@ class DeepseekV4Model(TextModel): "ffn_norm.weight": (gguf.MODEL_TENSOR.FFN_NORM, ".weight"), "ffn.gate.weight": (gguf.MODEL_TENSOR.FFN_GATE_INP, ".weight"), "ffn.gate.bias": (gguf.MODEL_TENSOR.FFN_EXP_PROBS_B, ".bias"), + "ffn.gate.bias_vl": (gguf.MODEL_TENSOR.FFN_EXP_PROBS_B_VL, ".bias"), "ffn.gate.tid2eid": (gguf.MODEL_TENSOR.FFN_GATE_TID2EID, ".weight"), "ffn.shared_experts.w1.weight": (gguf.MODEL_TENSOR.FFN_GATE_SHEXP, ".weight"), "ffn.shared_experts.w2.weight": (gguf.MODEL_TENSOR.FFN_DOWN_SHEXP, ".weight"), @@ -881,6 +881,10 @@ class DeepseekV4Model(TextModel): if re.match(r"layers\.\d+\.ffn\.experts\.\d+\.w[123]\.(weight|scale)$", name): return [] + # hash layers route text tokens via tid2eid and image tokens via bias_vl; gate.bias is unused + if name.endswith(".ffn.gate.bias") and bid is not None and bid < self.hparams["num_hash_layers"]: + return [] + tensor_key, suffix = self._map_dsv4_tensor_name(name, bid) if tensor_key == gguf.MODEL_TENSOR.FFN_GATE_TID2EID: return [] diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 56477c198..b85f62a31 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -697,6 +697,7 @@ class MODEL_TENSOR(IntEnum): FFN_DOWN_CHEXP = auto() FFN_UP_CHEXP = auto() FFN_EXP_PROBS_B = auto() + FFN_EXP_PROBS_B_VL = auto() # deepseek4 vision (bias for image tokens) FFN_GATE_TID2EID = auto() MOE_LATENT_DOWN = auto() # nemotron 3 super MOE_LATENT_UP = auto() # nemotron 3 super @@ -1449,6 +1450,7 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.FFN_UP_EXP: "blk.{bid}.ffn_up_exps", MODEL_TENSOR.FFN_GATE_UP_EXP: "blk.{bid}.ffn_gate_up_exps", MODEL_TENSOR.FFN_EXP_PROBS_B: "blk.{bid}.exp_probs_b", + MODEL_TENSOR.FFN_EXP_PROBS_B_VL: "blk.{bid}.exp_probs_b_vl", MODEL_TENSOR.FFN_GATE_TID2EID: "blk.{bid}.ffn_gate_tid2eid", MODEL_TENSOR.MOE_LATENT_DOWN: "blk.{bid}.ffn_latent_down", # nemotron 3 super MODEL_TENSOR.MOE_LATENT_UP: "blk.{bid}.ffn_latent_up", # nemotron 3 super @@ -3839,6 +3841,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_GATE_INP, MODEL_TENSOR.FFN_GATE_TID2EID, MODEL_TENSOR.FFN_EXP_PROBS_B, + MODEL_TENSOR.FFN_EXP_PROBS_B_VL, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE_EXP, MODEL_TENSOR.FFN_DOWN_EXP, diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 7adb87411..446de4ae2 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -457,6 +457,7 @@ static const std::map LLM_TENSOR_NAMES = { { LLM_TENSOR_FFN_UP_SHEXP, "blk.%d.ffn_up_shexp" }, { LLM_TENSOR_FFN_DOWN_SHEXP, "blk.%d.ffn_down_shexp" }, { LLM_TENSOR_FFN_EXP_PROBS_B, "blk.%d.exp_probs_b" }, + { LLM_TENSOR_FFN_EXP_PROBS_B_VL, "blk.%d.exp_probs_b_vl" }, { LLM_TENSOR_FFN_LATENT_DOWN, "blk.%d.ffn_latent_down" }, { LLM_TENSOR_FFN_LATENT_UP, "blk.%d.ffn_latent_up" }, { LLM_TENSOR_ATTN_NORM_2, "blk.%d.attn_norm_2" }, @@ -896,6 +897,7 @@ static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_FFN_GATE_CHEXPS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}}, {LLM_TENSOR_FFN_UP_CHEXPS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}}, {LLM_TENSOR_FFN_EXP_PROBS_B, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, + {LLM_TENSOR_FFN_EXP_PROBS_B_VL, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, // altup / laurel (gemma 3n) {LLM_TENSOR_PER_LAYER_TOKEN_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}}, {LLM_TENSOR_PER_LAYER_MODEL_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, diff --git a/src/llama-arch.h b/src/llama-arch.h index ca7d55a5f..0c0b99483 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -477,6 +477,7 @@ enum llm_tensor { LLM_TENSOR_FFN_GATE_CHEXPS, LLM_TENSOR_FFN_UP_CHEXPS, LLM_TENSOR_FFN_EXP_PROBS_B, + LLM_TENSOR_FFN_EXP_PROBS_B_VL, LLM_TENSOR_FFN_LATENT_DOWN, LLM_TENSOR_FFN_LATENT_UP, LLM_TENSOR_ATTN_Q_NORM, diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 1411692a8..390d3543d 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -161,6 +161,10 @@ struct llama_hparams { // the size of the sliding window (0 - no SWA) uint32_t n_swa = 0; + // deepseek4 vision: when decoding non-causally (multimodal input), SWA is not applied between tokens of the current ubatch (the image span); older tokens are still window-clipped + // for other models (like gemma 3, gemma 4): SWA is always applied to match transformers implementation + bool swa_full_non_causal = false; + // if is_swa_impl[il] == 1, then layer il is SWA // if is_swa_impl[il] == 0, then layer il is dense (i.e. non-SWA) // by default, all layers are dense diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index 3e4a4d56f..f22054c3d 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -1681,7 +1681,9 @@ static void set_input_kq_mask_impl(const args_set_input_kq_mask & args, T * data // apply SWA if any if (swa) { - if (llama_hparams::is_masked_swa(n_swa, swa_type, p0, p1)) { + // see llama_hparams::swa_full_non_causal + const bool in_span = !causal && args.hparams.swa_full_non_causal && p0 >= seq_pos_min[seq_id]; + if (!in_span && llama_hparams::is_masked_swa(n_swa, swa_type, p0, p1)) { goto skip; } } diff --git a/src/llama-model.h b/src/llama-model.h index 38066538e..ee6bb5ac3 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -362,6 +362,7 @@ struct llama_layer { struct ggml_tensor * ffn_up_b = nullptr; // b3 struct ggml_tensor * ffn_act = nullptr; struct ggml_tensor * ffn_exp_probs_b = nullptr; + struct ggml_tensor * ffn_exp_probs_b_vl = nullptr; // deepseek4 vision (bias for image tokens) struct ggml_tensor * ffn_gate_tid2eid = nullptr; struct ggml_tensor * dflash_attn_conv_base = nullptr; diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp index 222f22249..680516b23 100644 --- a/src/models/deepseek4.cpp +++ b/src/models/deepseek4.cpp @@ -66,6 +66,9 @@ void llama_model_deepseek4::load_arch_hparams(llama_model_loader & ml) { } hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.set_swa_pattern(0); + // tokens of an image span attend bidirectionally to the whole span, the window only applies to older tokens + // ref: get_window_topk_idxs_visible in the reference impl + hparams.swa_full_non_causal = true; for (uint32_t il = hparams.n_layer(); il < hparams.n_layer_all; ++il) { hparams.is_swa_impl[il] = true; } @@ -156,6 +159,8 @@ void llama_model_deepseek4::load_arch_tensors(llama_model_loader & ml) { } else { layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, flags); } + // vision variant only: routing bias for image tokens + layer.ffn_exp_probs_b_vl = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B_VL, "bias", i), {n_expert}, flags | TENSOR_NOT_REQUIRED); layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, flags); layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, flags); @@ -1275,7 +1280,14 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p const auto & layer = model.layers[il]; ggml_tensor * selected_experts = nullptr; ggml_tensor * exp_probs_b = layer.ffn_exp_probs_b; - if ((uint32_t) il < hparams.dsv4_hash_layer_count) { + + // may apply exp_probs_b_vl is input is from mtmd + const bool is_media = ubatch.embd != nullptr; + if (is_media) { + if (layer.ffn_exp_probs_b_vl) { + exp_probs_b = layer.ffn_exp_probs_b_vl; + } + } else if ((uint32_t) il < hparams.dsv4_hash_layer_count) { selected_experts = ggml_get_rows(ctx0, layer.ffn_gate_tid2eid, res->t_inp_tokens); exp_probs_b = nullptr; } diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index d2b88b1e4..f39f0974e 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -2132,6 +2132,7 @@ bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk case PROJECTOR_TYPE_GEMMA3: case PROJECTOR_TYPE_GEMMA4V: case PROJECTOR_TYPE_GEMMA4UV: + case PROJECTOR_TYPE_DEEPSEEK4V: return true; default: return false; From cff184438e0683c31d0566b96e969cdcd313779a Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Wed, 2 Sep 2026 12:49:11 -0500 Subject: [PATCH 05/17] Update ROCm to 10.0.0 release (#27803) --- .github/actions/windows-setup-rocm/action.yml | 2 +- .github/workflows/release.yml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/windows-setup-rocm/action.yml b/.github/actions/windows-setup-rocm/action.yml index aecbcf14f..f8f55af11 100644 --- a/.github/actions/windows-setup-rocm/action.yml +++ b/.github/actions/windows-setup-rocm/action.yml @@ -24,7 +24,7 @@ runs: write-host "Installing ROCm wheels for multi-arch support" # Install ROCm wheels for multi-arch support (this may take several minutes) - python -m pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ "rocm[libraries,devel]==${{ inputs.version }}" + python -m pip install --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries,devel]==${{ inputs.version }}" # Pre-expand the devel tree so it is included in the cache write-host "Initializing ROCm devel tree" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0d3145004..1101fef15 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -725,7 +725,7 @@ jobs: strategy: matrix: include: - - ROCM_VERSION: "7.14.0" + - ROCM_VERSION: "10.0.0" gpu_targets: "gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1152;gfx1153;gfx1200;gfx1201" build: x64 @@ -1279,7 +1279,7 @@ jobs: strategy: matrix: include: - - ROCM_VERSION: "7.14.0" + - ROCM_VERSION: "10.0.0" gpu_targets: "gfx908;gfx90a;gfx942;gfx950;gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1152;gfx1200;gfx1201" build: 'x64' @@ -1333,7 +1333,7 @@ jobs: # libraries = HIP runtime and CMake configs needed for linking # devel = compilers, headers, static libs python -m pip install --upgrade pip - python -m pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ "rocm[libraries,devel]==${{ matrix.ROCM_VERSION }}" + python -m pip install --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries,devel]==${{ matrix.ROCM_VERSION }}" # Get ROCm installation paths using the rocm-sdk CLI tool ROCM_PATH=$(rocm-sdk path --root) @@ -1703,7 +1703,7 @@ jobs: - [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz) - [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz) - [Ubuntu arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz) - - [Ubuntu x64 (ROCm 7.14)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-7.14-x64.tar.gz) + - [Ubuntu x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-10.0-x64.tar.gz) - [Ubuntu x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ needs.ubuntu-24-openvino.outputs.openvino_version }}-x64.tar.gz) - [Ubuntu x64 (SYCL FP32)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp32-x64.tar.gz) - [Ubuntu x64 (SYCL FP16)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp16-x64.tar.gz) @@ -1721,7 +1721,7 @@ jobs: - [Windows x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-x64.zip) - [Windows x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ needs.windows-openvino.outputs.openvino_version }}-x64.zip) - [Windows x64 (SYCL)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-sycl-x64.zip) - - [Windows x64 (ROCm 7.14)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-rocm-7.14-x64.zip) + - [Windows x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-rocm-10.0-x64.zip) **openEuler:** - [DISABLED](https://github.com/ggml-org/llama.cpp/pull/23705) From 03dbcc53e168a9b1aea3fccaf24b555e3a42d7db Mon Sep 17 00:00:00 2001 From: Niklas Wenzel Date: Wed, 2 Sep 2026 19:54:48 +0200 Subject: [PATCH 06/17] ci : check for missing autoreleasepools (#27884) * ci: check for missing autoreleasepools * ci : generalize graphics device name pattern --- .github/workflows/build-apple.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-apple.yml b/.github/workflows/build-apple.yml index c4d26f2eb..6b9a79115 100644 --- a/.github/workflows/build-apple.yml +++ b/.github/workflows/build-apple.yml @@ -66,7 +66,13 @@ jobs: -DGGML_RPC=ON \ -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3 time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) - leaks -atExit -- ./build/bin/test-thread-safety -hf ggml-org/gemma-3-270m-qat-GGUF -ngl 99 -p "$(printf 'hello %.0s' {1..128})" -n 16 -c 512 -ub 32 -np 2 -t 2 -lv 1 + + - name: Check for leaks + run: | + cmd=(./build/bin/test-thread-safety -hf ggml-org/gemma-3-270m-qat-GGUF -ngl 99 -p "$(printf 'hello %.0s' {1..128})" -n 16 -c 512 -ub 32 -np 2 -t 2 -lv 1) + leaks -atExit -- "${cmd[@]}" + # Graphics devices are leaked by Metal in Apple code sometimes, so we ignore those leaks + OBJC_DEBUG_MISSING_POOLS=YES "${cmd[@]}" 2>&1 | awk '{ print } index($0, "autoreleased with no pool in place") && !/class [a-zA-Z0-9]+Device autoreleased/ { found = 1 } END { exit found }' - name: Test id: cmake_test From 8c0b9cd04ac76d75bcf7b3da398fb00de03489ba Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Wed, 2 Sep 2026 20:09:46 +0200 Subject: [PATCH 07/17] metal : fix memory query under low-memory conditions (#27701) * metal: Fix memory query under low-memory conditions * Simply variable name Co-authored-by: Georgi Gerganov * Write it even shorter Co-authored-by: Niklas Wenzel --------- Co-authored-by: Georgi Gerganov Co-authored-by: Niklas Wenzel --- ggml/src/ggml-metal/ggml-metal-device.m | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index ef8108424..e20a4e891 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1471,8 +1471,10 @@ void ggml_metal_device_event_synchronize(ggml_metal_device_t dev, ggml_metal_eve void ggml_metal_device_get_memory(ggml_metal_device_t dev, size_t * free, size_t * total) { if (@available(macOS 10.12, iOS 16.0, *)) { - *total = dev->mtl_device.recommendedMaxWorkingSetSize; - *free = *total - dev->mtl_device.currentAllocatedSize; + *total = dev->mtl_device.recommendedMaxWorkingSetSize; + size_t cur = dev->mtl_device.currentAllocatedSize; + // it's possible to allocate more than `recommendedMaxWorkingSetSize` + *free = *total > cur ? *total - cur : 0; } else { *free = 0; *total = 0; From 9cc33944f9b7a44243618d5522adae357d7fdc90 Mon Sep 17 00:00:00 2001 From: Isaac <34376531+init-22@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:43:12 +0530 Subject: [PATCH 08/17] metal : add fa-vec tunings for M3 (#28236) --- ggml/src/ggml-metal/ggml-metal-tuning.cpp | 101 ++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/ggml/src/ggml-metal/ggml-metal-tuning.cpp b/ggml/src/ggml-metal/ggml-metal-tuning.cpp index c89a905df..b66fe6524 100644 --- a/ggml/src/ggml-metal/ggml-metal-tuning.cpp +++ b/ggml/src/ggml-metal/ggml-metal-tuning.cpp @@ -1468,6 +1468,107 @@ constexpr fa_vec_entry_t fa_vec_tuned_table[] = { { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, { { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 32, 32, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 32, 32, 1, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 32, 32, 2, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 32, 32, 3, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 64, 64, 3, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 64, 64, 1, 4 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 64, 64, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 64, 64, 3, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 96, 96, 1, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 96, 96, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 96, 96, 2, 4 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 96, 96, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 128, 128, 1, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 128, 128, 2, 3 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 128, 128, 3, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 192, 3, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 192, 1, 4 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 192, 2, 4 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 128, 2, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 128, 3, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 128, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 128, 2, 4 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 128, 3, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 192, 128, 3, 2 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 256, 256, 1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 256, 256, 3, 0 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 256, 256, 1, 1 }, { 1, 1 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 256, 256, 1, 2 }, { 1, 1 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 256, 256, 1, 4 }, { 1, 1 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 256, 256, 2, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 320, 256, 2, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 320, 256, 3, 0 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 320, 256, 1, 2 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 320, 256, 1, 4 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 512, 512, 2, 0 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 512, 512, 3, 0 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 512, 512, 3, 1 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 512, 512, 3, 3 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 576, 512, 2, 0 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 576, 512, 2, 1 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 576, 512, 2, 2 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_F16, 576, 512, 3, 1 }, { 4, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 32, 32, 2, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 64, 64, 2, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 96, 96, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 96, 96, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 96, 96, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 128, 128, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 128, 128, 3, 3 }, { 4, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 128, 128, 3, 4 }, { 1, 1 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 192, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 192, 2, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 192, 2, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 192, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 128, 1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 128, 2, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 128, 1, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 192, 128, 3, 2 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 256, 256, 3, 0 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 320, 256, 1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 320, 256, 3, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 320, 256, 1, 1 }, { 1, 2 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 32, 32, 1, 1 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 32, 32, 1, 3 }, { 2, 4 } }, { { GGML_METAL_DEVICE_M3_PRO, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } }, From 7339054744f109c4cd89b75689dbb8a2c154d60e Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Wed, 2 Sep 2026 21:20:10 +0200 Subject: [PATCH 09/17] mtmd: add mtmd_tokenize_from_parts() (#28250) * add mtmd_tokenize_from_parts * use it in mtmd-cli * move add_special to call level --- tests/test-mtmd-c-api.c | 33 ++++++++++++++++++++++ tests/test-mtmd-impl.cpp | 2 +- tools/mtmd/README-dev.md | 1 + tools/mtmd/mtmd-cli.cpp | 58 ++++++++++++++++++++++++++++---------- tools/mtmd/mtmd-internal.h | 6 ++-- tools/mtmd/mtmd.cpp | 54 +++++++++++++++++++++++++++++++---- tools/mtmd/mtmd.h | 27 +++++++++++++++--- 7 files changed, 154 insertions(+), 27 deletions(-) diff --git a/tests/test-mtmd-c-api.c b/tests/test-mtmd-c-api.c index 970d8a600..664c56cf8 100644 --- a/tests/test-mtmd-c-api.c +++ b/tests/test-mtmd-c-api.c @@ -130,6 +130,39 @@ int main(void) { } printf("Chunk save/load round-trip OK\n"); + // test input validation of mtmd_tokenize_from_parts() + // invalid parts are rejected before the ctx is used, so NULL ctx is OK here + { + mtmd_input_chunks * out = mtmd_input_chunks_init(); + mtmd_bitmap * bmp = mtmd_bitmap_init(4, 4, NULL); // placeholder bitmap + struct mtmd_input_text txt = { "hello", 5, false, false }; + struct mtmd_input_text txt_null = { NULL, 0, false, false }; + + struct mtmd_input_part part_both = { &txt, bmp }; + struct mtmd_input_part part_neither = { NULL, NULL }; + struct mtmd_input_part part_null_text = { &txt_null, NULL }; + const mtmd_input_part * parts[1]; + int32_t rc; + + parts[0] = &part_both; + rc = mtmd_tokenize_from_parts(NULL, out, parts, 1, false); + printf("tokenize part with both text and bitmap rc = %d (expect 1)\n", rc); + assert(rc == 1); + + parts[0] = &part_neither; + rc = mtmd_tokenize_from_parts(NULL, out, parts, 1, false); + printf("tokenize part with neither text nor bitmap rc = %d (expect 1)\n", rc); + assert(rc == 1); + + parts[0] = &part_null_text; + rc = mtmd_tokenize_from_parts(NULL, out, parts, 1, false); + printf("tokenize part with null text pointer rc = %d (expect 1)\n", rc); + assert(rc == 1); + + mtmd_bitmap_free(bmp); + mtmd_input_chunks_free(out); + } + // Free the chunks mtmd_input_chunks_free(chunks); diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp index df18b0a42..2ec6b2391 100644 --- a/tests/test-mtmd-impl.cpp +++ b/tests/test-mtmd-impl.cpp @@ -80,7 +80,7 @@ MAKE_TEST(test_temporal_merge_grouping) { // spec chars: // v = video frame, w = video frame of another size, a = audio, i = plain image, t = text auto make_parts = [&pool](const std::string & spec) { - std::vector parts; + std::vector parts; for (char c : spec) { if (c == 't') { parts.push_back({ "hello", nullptr }); diff --git a/tools/mtmd/README-dev.md b/tools/mtmd/README-dev.md index e14906823..b85627d2a 100644 --- a/tools/mtmd/README-dev.md +++ b/tools/mtmd/README-dev.md @@ -20,6 +20,7 @@ In short: A typical pipeline of the core libmtmd is as follows: - A bitmap (RGB image or PCM audio) is created - Bitmap and the text prompt is provided to `mtmd_tokenize()` that breaks the input into chunks + - Alternatively, `mtmd_tokenize_from_parts()` takes a list of pre-split text/media parts instead of a marker-based prompt - The tokenizer function first expands a "lazy" bitmap if it finds one. Typically, this is used by video, so that one media token corresponds to one input bitmap - For models that support "fused" temporal frames like Qwen-VL, the tokenizer tries to merge pair of consecutive frames into one batch. Only bitmaps marked by `mtmd_bitmap_set_mergeable()` are merged - The preprocessor will then be called, which produces a list of chunks diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cpp index 97678c6b2..ba18b3e32 100644 --- a/tools/mtmd/mtmd-cli.cpp +++ b/tools/mtmd/mtmd-cli.cpp @@ -109,16 +109,15 @@ struct mtmd_cli_context { mtmd_cli_context(common_params & params) : llama_init(common_init_from_params(params)) { model = llama_init->model(); lctx = llama_init->context(); + if (!model || !lctx) { + exit(1); + } vocab = llama_model_get_vocab(model); smpl = common_sampler_init(model, params.sampling); n_threads = params.cpuparams.n_threads; batch = llama_batch_init(1, 0, 1); // batch for next token generation n_batch = params.n_batch; - if (!model || !lctx) { - exit(1); - } - init_vision_context(params); if (!mtmd_helper_model_can_chat(lctx, ctx_vision.get())) { @@ -265,21 +264,50 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) { auto formatted_chat = chat_add_and_format(ctx, msg); LOG_DBG("formatted_chat.prompt: %s\n", formatted_chat.c_str()); - mtmd_input_text text; - text.text = formatted_chat.data(); - text.text_len = formatted_chat.size(); - text.add_special = add_bos; - text.parse_special = true; - if (g_is_interrupted) return 0; - mtmd::input_chunks chunks(mtmd_input_chunks_init()); + // note: we replace the marker here instead of letting mtmd_tokenize() to do that + // because we want to demonstrate how to use mtmd_tokenize_from_parts() + + // split the formatted chat on the media marker to get text segments + const std::string marker = mtmd_default_marker(); + std::vector segments; + size_t start = 0; + size_t pos; + while ((pos = formatted_chat.find(marker, start)) != std::string::npos) { + segments.push_back(formatted_chat.substr(start, pos - start)); + start = pos + marker.size(); + } + segments.push_back(formatted_chat.substr(start)); + auto bitmaps_c_ptr = ctx.bitmaps.c_ptr(); - int32_t res = mtmd_tokenize(ctx.ctx_vision.get(), + if (segments.size() - 1 != bitmaps_c_ptr.size()) { + LOG_ERR("Number of media markers (%zu) does not match number of loaded media (%zu)\n", + segments.size() - 1, bitmaps_c_ptr.size()); + return 1; + } + + // interleave text and media parts + std::vector texts(segments.size()); + std::vector parts; + for (size_t i = 0; i < segments.size(); i++) { + texts[i] = {segments[i].data(), segments[i].size(), /* add_special */ false, /* parse_special */ true}; + parts.push_back({&texts[i], nullptr}); + if (i < bitmaps_c_ptr.size()) { + parts.push_back({nullptr, bitmaps_c_ptr[i]}); + } + } + std::vector parts_ptr; + for (const auto & p : parts) { + parts_ptr.push_back(&p); + } + + mtmd::input_chunks chunks(mtmd_input_chunks_init()); + int32_t res = mtmd_tokenize_from_parts(ctx.ctx_vision.get(), chunks.ptr.get(), // output - &text, // text - bitmaps_c_ptr.data(), - bitmaps_c_ptr.size()); + parts_ptr.data(), + parts_ptr.size(), + add_bos); if (res != 0) { LOG_ERR("Unable to tokenize prompt, res = %d\n", res); return 1; diff --git a/tools/mtmd/mtmd-internal.h b/tools/mtmd/mtmd-internal.h index 067fa88b9..e7c62773e 100644 --- a/tools/mtmd/mtmd-internal.h +++ b/tools/mtmd/mtmd-internal.h @@ -10,10 +10,12 @@ #define MTMD_INTERNAL_HEADER // bitmap is null for text parts -struct mtmd_input_part { +struct mtmd_internal_part { std::string text; const mtmd_bitmap * bitmap; + // only used for text parts + bool parse_special = false; }; // [QWEN_VIDEO] merged parts are erased from `parts`, so one group always maps to one part -std::vector> mtmd_group_mergeable_bitmaps(std::vector & parts, int n_merge); +std::vector> mtmd_group_mergeable_bitmaps(std::vector & parts, int n_merge); diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index f39f0974e..a13b0556b 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -1097,7 +1097,7 @@ void mtmd_free(mtmd_context * ctx) { delete ctx; } -std::vector> mtmd_group_mergeable_bitmaps(std::vector & parts, int n_merge) { +std::vector> mtmd_group_mergeable_bitmaps(std::vector & parts, int n_merge) { std::vector> output; for (size_t i = 0; i < parts.size(); i++) { if (parts[i].bitmap == nullptr) { @@ -1124,7 +1124,7 @@ struct mtmd_tokenizer { bool parse_special; const llama_vocab * vocab; - using part = mtmd_input_part; + using part = mtmd_internal_part; std::vector parts; // these will be freed when mtmd_tokenizer finishes std::vector bm_from_lazy; // TODO @ngxson : refactor, free bm_from_lazy progressively @@ -1160,7 +1160,7 @@ struct mtmd_tokenizer { } parts.push_back({"", bitmaps[i_bm++]}); } else { - parts.push_back({std::move(part), nullptr}); + parts.push_back({std::move(part), nullptr, parse_special}); } } @@ -1177,6 +1177,26 @@ struct mtmd_tokenizer { expand_lazy_bitmaps(); } + mtmd_tokenizer(mtmd_context * ctx, + const mtmd_input_part ** input_parts, + size_t n_parts, + bool add_special) : ctx(ctx) { + this->add_special = add_special; + parse_special = true; // only used for text returned by lazy bitmaps + vocab = ctx->vocab; + + for (size_t i = 0; i < n_parts; i++) { + const mtmd_input_part * p = input_parts[i]; + if (p->text != nullptr) { + parts.push_back({std::string(p->text->text, p->text->text_len), nullptr, p->text->parse_special}); + } else { + parts.push_back({"", p->bitmap}); + } + } + + expand_lazy_bitmaps(); + } + void expand_lazy_bitmaps() { std::vector expanded; expanded.reserve(parts.size()); @@ -1201,7 +1221,7 @@ struct mtmd_tokenizer { LOG_DBG("%s: lazy callback returned bitmap with dimensions %d x %d\n", __func__, out_bm->nx, out_bm->ny); } else if (out_str) { auto & ptr = text_from_lazy.emplace_back(out_str); // remember to free it later - expanded.push_back({ptr, nullptr}); + expanded.push_back({ptr, nullptr, parse_special}); LOG_DBG("%s: lazy callback returned text: %s\n", __func__, out_str); } } else if (res == -1) { @@ -1245,7 +1265,7 @@ struct mtmd_tokenizer { return res; } } else { - add_text(p.text, parse_special); + add_text(p.text, p.parse_special); } } @@ -1727,6 +1747,30 @@ int32_t mtmd_tokenize(mtmd_context * ctx, } } +int32_t mtmd_tokenize_from_parts(mtmd_context * ctx, + mtmd_input_chunks * output, + const mtmd_input_part ** parts, + size_t n_parts, + bool add_special) { + for (size_t i = 0; i < n_parts; i++) { + if ((parts[i]->text == nullptr) == (parts[i]->bitmap == nullptr)) { + LOG_ERR("%s: part %zu must have either text or bitmap set, not both\n", __func__, i); + return 1; + } + if (parts[i]->text != nullptr && parts[i]->text->text == nullptr) { + LOG_ERR("%s: part %zu has null text pointer\n", __func__, i); + return 1; + } + } + try { + mtmd_tokenizer tokenizer(ctx, parts, n_parts, add_special); + return tokenizer.tokenize(output); + } catch (const std::exception & e) { + LOG_ERR("%s: error: %s\n", __func__, e.what()); + return 2; + } +} + static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * image_tokens, std::vector & out_embd) { clip_ctx * ctx_clip = ctx->ctx_v; if (!ctx_clip) { diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index ef88efd31..0c2f9886e 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -73,6 +73,12 @@ struct mtmd_input_text { bool parse_special; }; +struct mtmd_input_part { + // only text or bitmap can be set, not both + const struct mtmd_input_text * text; + const struct mtmd_bitmap * bitmap; +}; + // // C API // @@ -83,6 +89,7 @@ typedef struct mtmd_image_tokens mtmd_image_tokens; typedef struct mtmd_input_chunk mtmd_input_chunk; typedef struct mtmd_input_chunks mtmd_input_chunks; typedef struct mtmd_input_text mtmd_input_text; +typedef struct mtmd_input_part mtmd_input_part; typedef struct mtmd_batch mtmd_batch; typedef bool (*mtmd_progress_callback)(float progress, void * user_data); @@ -276,10 +283,10 @@ struct mtmd_decoder_pos { // return relative position (for example, embedding 0 will have position (0, 0, 0); remember to adjust it to the current absolute position) MTMD_API struct mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos(const mtmd_image_tokens * image_tokens, llama_pos pos_0, size_t i); -// tokenize an input text prompt and a list of bitmaps (images/audio) -// the prompt must have the input image marker (default: "<__media__>") in it +// tokenize an input text prompt and a list of bitmaps (image/audio) +// the prompt must have the input media marker (default: "<__media__>") in it // the default marker is defined by mtmd_default_marker() -// the marker will be replaced with the image/audio chunk +// the marker will be replaced with the media chunk // for example: // "here is an image: <__media__>\ndescribe it in detail." // this will gives 3 chunks: @@ -291,13 +298,25 @@ MTMD_API struct mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos(const mtmd_im // return values: // 0 on success // 1 on number of bitmaps not matching the number of markers -// 2 on image preprocessing error +// 2 on media preprocessing error MTMD_API int32_t mtmd_tokenize(mtmd_context * ctx, mtmd_input_chunks * output, const mtmd_input_text * text, const mtmd_bitmap ** bitmaps, size_t n_bitmaps); +// same as mtmd_tokenize(), but takes an array of mtmd_input_part +// use cases: +// - when you don't want to use media markers (they will be tokenized as normal text) +// - when you want to control parse_special for each text part +// note: per-part add_special will be ignored +// return 1 if a part has both text and bitmap set (or neither) +MTMD_API int32_t mtmd_tokenize_from_parts(mtmd_context * ctx, + mtmd_input_chunks * output, + const mtmd_input_part ** parts, + size_t n_parts, + bool add_special); + DEPRECATED(MTMD_API int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens), "use mtmd_encode_chunk() instead"); From f027c4f1b025e05d6a2fc3b741047bda07b85ef7 Mon Sep 17 00:00:00 2001 From: cqderek Date: Thu, 3 Sep 2026 03:59:36 +0800 Subject: [PATCH 10/17] ggml-hexagon: add F16 support for unary ops (#28228) Extend the HTP backend's F16 unary op coverage to include ABS on top of the existing NORM/RMS_NORM/L2_NORM/SCALE/CLAMP/SQR/SQRT set. - Add hvx_abs_f16_{aa,au,ua,uu} + dispatcher in hvx-arith.h, mirroring the sqr_f16 kernel structure and using the existing hvx_vec_abs_f16() sign-bit-clear helper - Add abs_f16() row-wise dispatch and DEFINE_UNARY_TASK_F16(unary_abs, ...) in unary-ops.c, wired into execute_op_unary()'s op_type/task_func switches - Register HTP_OP_UNARY_ABS in htp_op_is_unary() (unary-ops.h) so that ggml_hexagon_precompute_unary_params() fills kernel_params (n_threads, VTCM layout) for ABS nodes -- required for the F16 path to function - Narrow the F16 GGML_OP_UNARY gate in ggml_hexagon_supported_unary() (ggml-hexagon.cpp) to allow GGML_UNARY_OP_ABS specifically, instead of rejecting all GGML_OP_UNARY ops for F16 - Merge the separate execute_op_unary_f32()/execute_op_unary_f16() functions into a single execute_op_unary(), branching on an is_f16 flag for the parts that actually differ by type (elem_size, the early F16 op-support check, and which task_func table to use) while keeping the F32-only tiled/RMS_NORM_MUL paths intact -- per review feedback to avoid duplicating the shared VTCM/DMA plumbing Verified on-device (QRD8850, Hexagon v81) via test-backend-ops -o ABS: 8/8 passing (F16 + F32, HTP0, no CPU fallback). Regression-checked SQR/CLAMP/SQRT (F16+F32) and NORM/RMS_NORM/L2_NORM/SCALE (F32; their F16 paths have no CPU reference kernel in test-backend-ops and cannot be correctness-tested there independent of this change). --- ggml/src/ggml-hexagon/ggml-hexagon.cpp | 38 +++- ggml/src/ggml-hexagon/htp/hvx-arith.h | 171 +++++++++++++++- ggml/src/ggml-hexagon/htp/hvx-log.h | 29 +++ ggml/src/ggml-hexagon/htp/hvx-norm.h | 197 +++++++++++++++++++ ggml/src/ggml-hexagon/htp/hvx-scale.h | 66 +++++++ ggml/src/ggml-hexagon/htp/hvx-sqrt.h | 63 ++++++ ggml/src/ggml-hexagon/htp/unary-ops.c | 260 ++++++++++++++++++++++--- ggml/src/ggml-hexagon/htp/unary-ops.h | 17 +- 8 files changed, 797 insertions(+), 44 deletions(-) diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 04fb9a223..104201daf 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -4005,8 +4005,10 @@ static void ggml_hexagon_precompute_unary_params( kparams->n_threads = n_threads; - const size_t src0_data_row_size = src0->ne[0] * sizeof(float); - const size_t dst_data_row_size = dst->ne[0] * sizeof(float); + const size_t elem_size = ggml_type_size(src0->type); + + const size_t src0_data_row_size = src0->ne[0] * elem_size; + const size_t dst_data_row_size = dst->ne[0] * ggml_type_size(dst->type); const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128); const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, 128); @@ -4020,7 +4022,7 @@ static void ggml_hexagon_precompute_unary_params( if (op == HTP_OP_RMS_NORM_MUL) { GGML_ASSERT(src1 != nullptr); - src1_data_row_size = src1->ne[0] * sizeof(float); + src1_data_row_size = src1->ne[0] * ggml_type_size(src1->type); src1_row_size_aligned = hex_round_up(src1_data_row_size, 128); broadcast_weight = (src1->ne[1] * src1->ne[2] * src1->ne[3] == 1); } @@ -4034,7 +4036,7 @@ static void ggml_hexagon_precompute_unary_params( htp_unary_vtcm_layout_build(&L, op, src0->ne[0], dst->ne[0], op == HTP_OP_RMS_NORM_MUL ? src1->ne[0] : 0, - broadcast_weight, n_threads, sess->vtcm_size, + broadcast_weight, n_threads, sess->vtcm_size, elem_size, &col_tile, &vtcm_row_per_thread); kparams->col_tile = col_tile; @@ -4451,15 +4453,39 @@ static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * ses const struct ggml_tensor * src0 = op->src[0]; const struct ggml_tensor * dst = op; - if (src0->type != GGML_TYPE_F32) { + if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) { return false; } - if (dst->type != GGML_TYPE_F32) { + if (dst->type != src0->type) { return false; } if (!ggml_is_contiguous_rows(src0)) { return false; } + + // F16 device kernels only cover this explicit whitelist (must stay in sync with + // the is_f16 whitelist in execute_op_unary(), unary-ops.c). + if (src0->type == GGML_TYPE_F16) { + switch (op->op) { + case GGML_OP_NORM: + case GGML_OP_RMS_NORM: + case GGML_OP_L2_NORM: + case GGML_OP_SCALE: + case GGML_OP_CLAMP: + case GGML_OP_SQR: + case GGML_OP_SQRT: + case GGML_OP_LOG: + break; + case GGML_OP_UNARY: + if (ggml_get_unary_op(op) != GGML_UNARY_OP_ABS) { + return false; + } + break; + default: + return false; + } + } + if (!ggml_are_same_shape(src0, dst)) { return false; } diff --git a/ggml/src/ggml-hexagon/htp/hvx-arith.h b/ggml/src/ggml-hexagon/htp/hvx-arith.h index 765c35776..5ef746342 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-arith.h +++ b/ggml/src/ggml-hexagon/htp/hvx-arith.h @@ -358,6 +358,54 @@ static inline void hvx_clamp_scalar_f32(uint8_t * restrict dst, const uint8_t * } } +#define HVX_OP_CLAMP_SCALAR_F16(v) \ + ({ \ + HVX_VectorPred pred_cap_right = Q6_Q_vcmp_gt_VhfVhf(v, max_vec); \ + HVX_VectorPred pred_cap_left = Q6_Q_vcmp_gt_VhfVhf(min_vec, v); \ + HVX_Vector tmp = Q6_V_vmux_QVV(pred_cap_right, max_vec, v); \ + Q6_V_vmux_QVV(pred_cap_left, min_vec, tmp); \ + }) + +static inline void hvx_clamp_scalar_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) { + const HVX_Vector min_vec = hvx_vec_splat_f16(min); + const HVX_Vector max_vec = hvx_vec_splat_f16(max); + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_CLAMP_SCALAR_F16); +} + +static inline void hvx_clamp_scalar_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) { + const HVX_Vector min_vec = hvx_vec_splat_f16(min); + const HVX_Vector max_vec = hvx_vec_splat_f16(max); + assert((unsigned long) dst % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_CLAMP_SCALAR_F16); +} + +static inline void hvx_clamp_scalar_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) { + const HVX_Vector min_vec = hvx_vec_splat_f16(min); + const HVX_Vector max_vec = hvx_vec_splat_f16(max); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_CLAMP_SCALAR_F16); +} + +static inline void hvx_clamp_scalar_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) { + const HVX_Vector min_vec = hvx_vec_splat_f16(min); + const HVX_Vector max_vec = hvx_vec_splat_f16(max); + hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_CLAMP_SCALAR_F16); +} + +static inline void hvx_clamp_scalar_f16(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, const int num_elems) { + if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { + hvx_clamp_scalar_f16_aa(dst, src, min, max, num_elems); + } else if (hex_is_aligned((void *) dst, 128)) { + hvx_clamp_scalar_f16_au(dst, src, min, max, num_elems); + } else if (hex_is_aligned((void *) src, 128)) { + hvx_clamp_scalar_f16_ua(dst, src, min, max, num_elems); + } else { + hvx_clamp_scalar_f16_uu(dst, src, min, max, num_elems); + } +} + // // Abs // @@ -386,11 +434,69 @@ static inline void hvx_abs_f32_aa(uint8_t * restrict dst, const uint8_t * restri } } +#define hvx_abs_f16_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t elem_size = sizeof(_Float16); \ + const uint32_t epv = 128 / elem_size; \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = hvx_vec_abs_f16(vsrc[i]); \ + } \ + if (nloe) { \ + HVX_Vector v = hvx_vec_abs_f16(vsrc[i]); \ + vec_store((void *) &vdst[i], nloe * elem_size, v); \ + } \ + } while(0) + +static inline void hvx_abs_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_abs_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); +} + +static inline void hvx_abs_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + hvx_abs_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a); +} + +static inline void hvx_abs_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) src % 128 == 0); + hvx_abs_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u); +} + +static inline void hvx_abs_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + hvx_abs_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); +} + +static inline void hvx_abs_f16(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t num_elems) { + if (hex_is_aligned((void *) dst, 128)) { + if (hex_is_aligned((void *) src, 128)) { + hvx_abs_f16_aa(dst, src, num_elems); + } else { + hvx_abs_f16_au(dst, src, num_elems); + } + } else { + if (hex_is_aligned((void *) src, 128)) { + hvx_abs_f16_ua(dst, src, num_elems); + } else { + hvx_abs_f16_uu(dst, src, num_elems); + } + } +} + // // Square // -#define hvx_sqr_f32_loop_body(dst_type, src_type, vec_store) \ +#define hvx_sqr_f32_loop_body(dst_type, src_type, vec_store) \ do { \ dst_type * restrict vdst = (dst_type *) dst; \ src_type * restrict vsrc = (src_type *) src; \ @@ -404,10 +510,10 @@ static inline void hvx_abs_f32_aa(uint8_t * restrict dst, const uint8_t * restri \ _Pragma("unroll(4)") \ for (; i < nvec; i++) { \ - vdst[i] = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \ + vdst[i] = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \ } \ if (nloe) { \ - HVX_Vector v = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \ + HVX_Vector v = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \ vec_store((void *) &vdst[i], nloe * elem_size, v); \ } \ } while(0) @@ -448,6 +554,64 @@ static inline void hvx_sqr_f32(uint8_t * restrict dst, const uint8_t * restrict } } +#define hvx_sqr_f16_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t elem_size = sizeof(_Float16); \ + const uint32_t epv = 128 / elem_size; \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = HVX_OP_MUL_F16(vsrc[i], vsrc[i]); \ + } \ + if (nloe) { \ + HVX_Vector v = HVX_OP_MUL_F16(vsrc[i], vsrc[i]); \ + vec_store((void *) &vdst[i], nloe * elem_size, v); \ + } \ + } while(0) + +static inline void hvx_sqr_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_sqr_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); +} + +static inline void hvx_sqr_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + hvx_sqr_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a); +} + +static inline void hvx_sqr_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) src % 128 == 0); + hvx_sqr_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u); +} + +static inline void hvx_sqr_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + hvx_sqr_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); +} + +static inline void hvx_sqr_f16(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t num_elems) { + if (hex_is_aligned((void *) dst, 128)) { + if (hex_is_aligned((void *) src, 128)) { + hvx_sqr_f16_aa(dst, src, num_elems); + } else { + hvx_sqr_f16_au(dst, src, num_elems); + } + } else { + if (hex_is_aligned((void *) src, 128)) { + hvx_sqr_f16_ua(dst, src, num_elems); + } else { + hvx_sqr_f16_uu(dst, src, num_elems); + } + } +} + #undef HVX_OP_ADD_F32 #undef HVX_OP_SUB_F32 #undef HVX_OP_MUL_F32 @@ -464,6 +628,7 @@ static inline void hvx_sqr_f32(uint8_t * restrict dst, const uint8_t * restrict #undef hvx_scalar_loop_body #undef HVX_OP_MIN_SCALAR #undef HVX_OP_CLAMP_SCALAR +#undef HVX_OP_CLAMP_SCALAR_F16 #undef DEFINE_HVX_BINARY_OP_VARIANTS #undef HVX_BINARY_DISPATCHER #undef UNUSED diff --git a/ggml/src/ggml-hexagon/htp/hvx-log.h b/ggml/src/ggml-hexagon/htp/hvx-log.h index a209f88d5..491041d5a 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-log.h +++ b/ggml/src/ggml-hexagon/htp/hvx-log.h @@ -86,4 +86,33 @@ static inline void hvx_log_f32_aa(uint8_t * restrict dst, const uint8_t * restri } } +// Compute log(x) for f16 by promoting to f32, applying hvx_vec_log_f32, and narrowing back. +static inline void hvx_log_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; + + const uint32_t nvec = n / VLEN_FP16; + const uint32_t nloe = n % VLEN_FP16; + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); + HVX_Vector r0 = hvx_vec_log_f32(Q6_V_lo_W(p)); + HVX_Vector r1 = hvx_vec_log_f32(Q6_V_hi_W(p)); + vdst[i] = hvx_vec_f32_to_f16(r0, r1); + } + if (nloe) { + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); + HVX_Vector r0 = hvx_vec_log_f32(Q6_V_lo_W(p)); + HVX_Vector r1 = hvx_vec_log_f32(Q6_V_hi_W(p)); + HVX_Vector v = hvx_vec_f32_to_f16(r0, r1); + hvx_vec_store_a((void *) &vdst[i], nloe * SIZEOF_FP16, v); + } +} + #endif /* HVX_LOG_H */ diff --git a/ggml/src/ggml-hexagon/htp/hvx-norm.h b/ggml/src/ggml-hexagon/htp/hvx-norm.h index a8645e412..7ea945a33 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-norm.h +++ b/ggml/src/ggml-hexagon/htp/hvx-norm.h @@ -254,4 +254,201 @@ static inline void hvx_fast_l2_norm_f32(const uint8_t * restrict src, } } +// F16 norm kernels: reduce and scale in f32 (via promote/narrow), matching the +// precision-preserving pattern used by the flash-attn f16 kernels. + +static inline void hvx_fast_rms_norm_f16(const uint8_t * restrict src, + uint8_t * restrict dst, + const int num_elems, + float epsilon) { + + const HVX_Vector * restrict v_src = (HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + + const int nvec = num_elems / VLEN_FP16; // number of full f16 vectors + const int nloe = num_elems % VLEN_FP16; // leftover elements + + HVX_Vector sum_v = Q6_V_vsplat_R(0x00000000); + HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]); + HVX_Vector p0 = Q6_V_lo_W(p); + HVX_Vector p1 = Q6_V_hi_W(p); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0)); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1)); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_VectorPair p = hvx_vec_f16_to_f32(v1); + HVX_Vector p0 = Q6_V_lo_W(p); + HVX_Vector p1 = Q6_V_hi_W(p); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0)); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1)); + } + + sum_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v)); + + HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems); + HVX_Vector denom_v = hvx_vec_inverse_f32(t_v); + HVX_Vector mean_v = Q6_Vqf32_vmpy_VsfVsf(sum_v, denom_v); + HVX_Vector mean_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(mean_v, epsilon_v); + + HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v)); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]); + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v)); + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v)); + v_dst[i] = hvx_vec_f32_to_f16(r0, r1); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_VectorPair p = hvx_vec_f16_to_f32(v1); + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v)); + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v)); + HVX_Vector result = hvx_vec_f32_to_f16(r0, r1); + hvx_vec_store_a(&v_dst[nvec], nloe * SIZEOF_FP16, result); + } +} + +static inline void hvx_fast_norm_f16(const uint8_t * restrict src, + uint8_t * restrict dst, + const int num_elems, + float epsilon) { + + const HVX_Vector * restrict v_src = (HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + + const int nvec = num_elems / VLEN_FP16; + const int nloe = num_elems % VLEN_FP16; + + HVX_Vector sum_sq_v = Q6_V_vsplat_R(0x00000000); + HVX_Vector sum_x_v = Q6_V_vsplat_R(0x00000000); + HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]); + HVX_Vector p0 = Q6_V_lo_W(p); + HVX_Vector p1 = Q6_V_hi_W(p); + sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0)); + sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1)); + sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p0, Q6_V_vzero())); + sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p1, Q6_V_vzero())); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_VectorPair p = hvx_vec_f16_to_f32(v1); + HVX_Vector p0 = Q6_V_lo_W(p); + HVX_Vector p1 = Q6_V_hi_W(p); + sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0)); + sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1)); + sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p0, Q6_V_vzero())); + sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p1, Q6_V_vzero())); + } + + sum_sq_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_sq_v)); + sum_x_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_x_v)); + + HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems); + HVX_Vector denom_v = hvx_vec_inverse_f32(t_v); + HVX_Vector mean_sq_v = Q6_Vqf32_vmpy_VsfVsf(sum_sq_v, denom_v); + HVX_Vector mean_x_v = Q6_Vqf32_vmpy_VsfVsf(sum_x_v, denom_v); + HVX_Vector mean_x_sq_v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(mean_x_v), Q6_Vsf_equals_Vqf32(mean_x_v)); + HVX_Vector var_v = Q6_Vqf32_vsub_Vqf32Vqf32(mean_sq_v, mean_x_sq_v); + HVX_Vector var_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(var_v, epsilon_v); + + HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(var_epsilon_v)); + HVX_Vector mean_x_b = hvx_vec_repl_f32(Q6_Vsf_equals_Vqf32(mean_x_v)); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]); + HVX_Vector d0 = Q6_Vqf32_vsub_VsfVsf(Q6_V_lo_W(p), mean_x_b); + HVX_Vector d1 = Q6_Vqf32_vsub_VsfVsf(Q6_V_hi_W(p), mean_x_b); + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d0), scale_v)); + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d1), scale_v)); + v_dst[i] = hvx_vec_f32_to_f16(r0, r1); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_VectorPair p = hvx_vec_f16_to_f32(v1); + HVX_Vector d0 = Q6_Vqf32_vsub_VsfVsf(Q6_V_lo_W(p), mean_x_b); + HVX_Vector d1 = Q6_Vqf32_vsub_VsfVsf(Q6_V_hi_W(p), mean_x_b); + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d0), scale_v)); + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d1), scale_v)); + HVX_Vector result = hvx_vec_f32_to_f16(r0, r1); + hvx_vec_store_a(&v_dst[nvec], nloe * SIZEOF_FP16, result); + } +} + +static inline void hvx_fast_l2_norm_f16(const uint8_t * restrict src, + uint8_t * restrict dst, + const int num_elems, + float epsilon) { + + const HVX_Vector * restrict v_src = (HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + + const int nvec = num_elems / VLEN_FP16; + const int nloe = num_elems % VLEN_FP16; + + HVX_Vector sum_v = hvx_vec_splat_f32(0.0f); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]); + HVX_Vector p0 = Q6_V_lo_W(p); + HVX_Vector p1 = Q6_V_hi_W(p); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0)); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1)); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_VectorPair p = hvx_vec_f16_to_f32(v1); + HVX_Vector p0 = Q6_V_lo_W(p); + HVX_Vector p1 = Q6_V_hi_W(p); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0)); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1)); + } + + HVX_Vector sum_sf = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v)); + HVX_Vector rsqrt_v = hvx_vec_rsqrt_f32(sum_sf); + HVX_Vector sqrt_v = hvx_vec_inverse_f32(rsqrt_v); + HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon); + HVX_Vector denom_v = Q6_Vsf_vmax_VsfVsf(sqrt_v, epsilon_v); + HVX_Vector scale_v = hvx_vec_inverse_f32(denom_v); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]); + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v)); + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v)); + v_dst[i] = hvx_vec_f32_to_f16(r0, r1); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_VectorPair p = hvx_vec_f16_to_f32(v1); + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v)); + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v)); + HVX_Vector result = hvx_vec_f32_to_f16(r0, r1); + hvx_vec_store_a(&v_dst[nvec], nloe * SIZEOF_FP16, result); + } +} + #endif // HVX_NORM_H diff --git a/ggml/src/ggml-hexagon/htp/hvx-scale.h b/ggml/src/ggml-hexagon/htp/hvx-scale.h index c65c98639..9b1a28f52 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-scale.h +++ b/ggml/src/ggml-hexagon/htp/hvx-scale.h @@ -130,4 +130,70 @@ static inline void hvx_scale_offset_f32(uint8_t * restrict dst, const uint8_t * } } +// Scale+offset computed by promoting f16 -> f32, then narrowing the result back to f16. +#define hvx_scale_offset_f16_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + HVX_Vector vs = hvx_vec_splat_f32(scale); \ + HVX_Vector vo = hvx_vec_splat_f32(offset); \ + \ + const uint32_t nvec = n / VLEN_FP16; \ + const uint32_t nloe = n % VLEN_FP16; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; ++i) { \ + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \ + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), vs), vo)); \ + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), vs), vo)); \ + vdst[i] = hvx_vec_f32_to_f16(r0, r1); \ + } \ + if (nloe) { \ + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \ + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), vs), vo)); \ + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), vs), vo)); \ + HVX_Vector v = hvx_vec_f32_to_f16(r0, r1); \ + vec_store((void *) &vdst[i], nloe * SIZEOF_FP16, v); \ + } \ + } while(0) + +static inline void hvx_scale_offset_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) { + assert((size_t) dst % 128 == 0); + assert((size_t) src % 128 == 0); + hvx_scale_offset_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); +} + +static inline void hvx_scale_offset_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) { + assert((size_t) dst % 128 == 0); + hvx_scale_offset_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a); +} + +static inline void hvx_scale_offset_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) { + assert((size_t) src % 128 == 0); + hvx_scale_offset_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u); +} + +static inline void hvx_scale_offset_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) { + hvx_scale_offset_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); +} + +static inline void hvx_scale_offset_f16(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) { + if (((size_t) dst & 127) == 0) { + if (((size_t) src & 127) == 0) { + hvx_scale_offset_f16_aa(dst, src, n, scale, offset); + } else { + hvx_scale_offset_f16_au(dst, src, n, scale, offset); + } + } else { + if (((size_t) src & 127) == 0) { + hvx_scale_offset_f16_ua(dst, src, n, scale, offset); + } else { + hvx_scale_offset_f16_uu(dst, src, n, scale, offset); + } + } +} + #endif // HVX_SCALE_H diff --git a/ggml/src/ggml-hexagon/htp/hvx-sqrt.h b/ggml/src/ggml-hexagon/htp/hvx-sqrt.h index e31a1006d..abdded5ce 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +++ b/ggml/src/ggml-hexagon/htp/hvx-sqrt.h @@ -123,4 +123,67 @@ static inline void hvx_sqrt_f32(uint8_t * restrict dst, const uint8_t * restrict } } +// Compute sqrt(x) for f16 by promoting to f32, applying hvx_vec_rsqrt_f32, and narrowing back. +#define hvx_sqrt_f16_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t nvec = n / VLEN_FP16; \ + const uint32_t nloe = n % VLEN_FP16; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \ + HVX_Vector r0 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_lo_W(p)), Q6_V_lo_W(p)); \ + HVX_Vector r1 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_hi_W(p)), Q6_V_hi_W(p)); \ + vdst[i] = hvx_vec_f32_to_f16(r0, r1); \ + } \ + if (nloe) { \ + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \ + HVX_Vector r0 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_lo_W(p)), Q6_V_lo_W(p)); \ + HVX_Vector r1 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_hi_W(p)), Q6_V_hi_W(p)); \ + HVX_Vector v = hvx_vec_f32_to_f16(r0, r1); \ + vec_store((void *) &vdst[i], nloe * SIZEOF_FP16, v); \ + } \ + } while(0) + +static inline void hvx_sqrt_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_sqrt_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); +} + +static inline void hvx_sqrt_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + hvx_sqrt_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a); +} + +static inline void hvx_sqrt_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) src % 128 == 0); + hvx_sqrt_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u); +} + +static inline void hvx_sqrt_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + hvx_sqrt_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); +} + +static inline void hvx_sqrt_f16(uint8_t * restrict dst, const uint8_t * restrict src, const int num_elems) { + if ((unsigned long) dst % 128 == 0) { + if ((unsigned long) src % 128 == 0) { + hvx_sqrt_f16_aa(dst, src, num_elems); + } else { + hvx_sqrt_f16_au(dst, src, num_elems); + } + } else { + if ((unsigned long) src % 128 == 0) { + hvx_sqrt_f16_ua(dst, src, num_elems); + } else { + hvx_sqrt_f16_uu(dst, src, num_elems); + } + } +} + #endif /* HVX_SQRT_H */ diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c index 1a632bf56..5e62b4a9b 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.c +++ b/ggml/src/ggml-hexagon/htp/unary-ops.c @@ -234,6 +234,146 @@ static void sqrt_f32(const float * restrict src, } } +static void scale_f16(const _Float16 * restrict src, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float scale = 0.f; + float bias = 0.f; + memcpy(&scale, &op_params[0], sizeof(float)); + memcpy(&bias, &op_params[1], sizeof(float)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_scale_offset_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0, scale, bias); + } +} + +static void clamp_f16(const _Float16 * restrict src, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float min = 0.f; + float max = 0.f; + memcpy(&min, &op_params[0], sizeof(float)); + memcpy(&max, &op_params[1], sizeof(float)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_clamp_scalar_f16(dst_local, src_local, (_Float16) min, (_Float16) max, ne0); + } +} + +static void rms_norm_f16(const _Float16 * restrict src, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float epsilon = 0.f; + memcpy(&epsilon, op_params, sizeof(float)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_fast_rms_norm_f16((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon); + } +} + +static void norm_f16(const _Float16 * restrict src, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float epsilon = 0.f; + memcpy(&epsilon, op_params, sizeof(float)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_fast_norm_f16((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon); + } +} + +static void sqr_f16(const _Float16 * restrict src, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_sqr_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0); + } +} + +static void sqrt_f16(const _Float16 * restrict src, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_sqrt_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0); + } +} + +static void abs_f16(const _Float16 * restrict src, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_abs_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0); + } +} + +static void log_f16(const _Float16 * restrict src, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_log_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0); + } +} + +static void l2_norm_f16(const _Float16 * restrict src, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float epsilon = 0.f; + memcpy(&epsilon, op_params, sizeof(float)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_f = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_f = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_fast_l2_norm_f16((const uint8_t *)src_f, (uint8_t *)dst_f, ne0, epsilon); + } +} + static void neg_f32(const float * restrict src, float * restrict dst, const uint32_t num_rows, @@ -471,8 +611,8 @@ static void log_f32(const float * restrict src, } } -#define DEFINE_UNARY_TASK(NAME, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) \ -static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * data) { \ +#define DEFINE_UNARY_TASK_IMPL(NAME, TYPE, SUFFIX, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) \ +static void unary_task_##SUFFIX##_##NAME(unsigned int nth, unsigned int ith, void * data) { \ const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; \ struct htp_ops_context * octx = uctx->octx; \ const struct htp_tensor * src = octx->src[0]; \ @@ -536,7 +676,7 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat const uint32_t dst_max_block = block_dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1); \ const uint32_t BLOCK = MIN(src0_max_block, dst_max_block); \ if (BLOCK == 0) { \ - FARF(ERROR, "unary-f32 : current VTCM reservation %zu is too small, needed at least %zu\n", \ + FARF(ERROR, "unary-" #SUFFIX " : current VTCM reservation %zu is too small, needed at least %zu\n", \ uctx->vtcm_src0_size_per_thread, src0_row_size_aligned); \ return; \ } \ @@ -578,11 +718,11 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_src0_contig, block_dst_contig, \ ne01, div_ne01); \ \ - float * dst_vtcm = (float *) dma_queue_pop(dma_queue).src; \ - float * src0_vtcm = (float *) dma_queue_pop(dma_queue).dst; \ - float * src1_vtcm = NULL; \ + TYPE * dst_vtcm = (TYPE *) dma_queue_pop(dma_queue).src; \ + TYPE * src0_vtcm = (TYPE *) dma_queue_pop(dma_queue).dst; \ + TYPE * src1_vtcm = NULL; \ if ((IS_RMS_NORM_MUL) && !uctx->broadcast_weight) { \ - src1_vtcm = (float *) dma_queue_pop(dma_queue).dst; \ + src1_vtcm = (TYPE *) dma_queue_pop(dma_queue).dst; \ } \ \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ @@ -625,6 +765,10 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat dma_queue_flush(dma_queue); \ } +// F32 unary task: row-block DMA/VTCM plumbing, float-typed VTCM buffers. +#define DEFINE_UNARY_TASK(NAME, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) \ + DEFINE_UNARY_TASK_IMPL(NAME, float, f32, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) + DEFINE_UNARY_TASK(norm, false, false, norm_f32(src0_vtcm, dst_vtcm, block_size, uctx)) DEFINE_UNARY_TASK(rms_norm, false, false, rms_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx)) DEFINE_UNARY_TASK(rms_norm_mul, true, false, rms_norm_mul_f32(src0_vtcm, uctx->broadcast_weight ? (const float *) src1_vtcm_data : src1_vtcm, dst_vtcm, block_size, uctx)) @@ -644,6 +788,18 @@ DEFINE_UNARY_TASK(unary_log, false, false, log_f32(src0_vtcm, dst_vtcm, blo DEFINE_UNARY_TASK(l2_norm, false, false, l2_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx)) DEFINE_UNARY_TASK(tri, false, true, tri_f32(src0_vtcm, dst_vtcm, block_size, ir, uctx)) +// F16 unary tasks: same DMA/VTCM plumbing as DEFINE_UNARY_TASK, but VTCM buffers are +// _Float16-typed. None of the current F16 ops need RMS_NORM_MUL or TRI support. +DEFINE_UNARY_TASK_IMPL(norm, _Float16, f16, false, false, norm_f16(src0_vtcm, dst_vtcm, block_size, uctx)) +DEFINE_UNARY_TASK_IMPL(rms_norm, _Float16, f16, false, false, rms_norm_f16(src0_vtcm, dst_vtcm, block_size, uctx)) +DEFINE_UNARY_TASK_IMPL(scale, _Float16, f16, false, false, scale_f16(src0_vtcm, dst_vtcm, block_size, uctx)) +DEFINE_UNARY_TASK_IMPL(clamp, _Float16, f16, false, false, clamp_f16(src0_vtcm, dst_vtcm, block_size, uctx)) +DEFINE_UNARY_TASK_IMPL(sqr, _Float16, f16, false, false, sqr_f16(src0_vtcm, dst_vtcm, block_size, uctx)) +DEFINE_UNARY_TASK_IMPL(sqrt, _Float16, f16, false, false, sqrt_f16(src0_vtcm, dst_vtcm, block_size, uctx)) +DEFINE_UNARY_TASK_IMPL(l2_norm, _Float16, f16, false, false, l2_norm_f16(src0_vtcm, dst_vtcm, block_size, uctx)) +DEFINE_UNARY_TASK_IMPL(unary_abs, _Float16, f16, false, false, abs_f16(src0_vtcm, dst_vtcm, block_size, uctx)) +DEFINE_UNARY_TASK_IMPL(unary_log, _Float16, f16, false, false, log_f16(src0_vtcm, dst_vtcm, block_size, uctx)) + // Apply a pointwise unary op to one column tile that is already in VTCM. #define DEFINE_UNARY_TILED_TASK(NAME, IS_TRI, CORE_TILE_EXPR) \ static void unary_task_f32_tiled_##NAME(unsigned int nth, unsigned int ith, void * data) { \ @@ -892,50 +1048,76 @@ DEFINE_UNARY_TILED_TASK(unary_abs, false, hvx_abs_f32_aa(dst_vtcm, src_vtcm DEFINE_UNARY_TILED_TASK(unary_log, false, hvx_log_f32_aa(dst_vtcm, src_vtcm, tw)) DEFINE_UNARY_TILED_TASK(tri, true, tri_apply_tile_f32(src_vtcm, dst_vtcm, tw, col, i01, ne0, tri_ttype)) -static int execute_op_unary_f32(struct htp_ops_context * octx) { +static int execute_op_unary(struct htp_ops_context * octx) { int err = HTP_STATUS_OK; const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; + const bool is_f16 = (src0->type == HTP_TYPE_F16); + const char * op_type = NULL; switch (octx->op) { - case HTP_OP_NORM: op_type = "norm-f32"; break; - case HTP_OP_RMS_NORM: op_type = "rmsnorm-f32"; break; - case HTP_OP_RMS_NORM_MUL: op_type = "rmsnorm-mul-f32"; break; - case HTP_OP_SCALE: op_type = "scale-f32"; break; - case HTP_OP_CLAMP: op_type = "clamp-f32"; break; - case HTP_OP_SQR: op_type = "sqr-f32"; break; - case HTP_OP_SQRT: op_type = "sqrt-f32"; break; - case HTP_OP_UNARY_NEG: op_type = "neg-f32"; break; - case HTP_OP_UNARY_EXP: op_type = "exp-f32"; break; - case HTP_OP_UNARY_SIGMOID: op_type = "sigmoid-f32"; break; - case HTP_OP_UNARY_SILU: op_type = "silu-f32"; break; - case HTP_OP_UNARY_GELU: op_type = "gelu-f32"; break; - case HTP_OP_UNARY_SOFTPLUS: op_type = "softplus-f32"; break; - case HTP_OP_UNARY_TANH: op_type = "tanh-f32"; break; - case HTP_OP_UNARY_ABS: op_type = "abs-f32"; break; - case HTP_OP_UNARY_LOG: op_type = "log-f32"; break; - case HTP_OP_L2_NORM: op_type = "l2norm-f32"; break; - case HTP_OP_TRI: op_type = "tri-f32"; break; + case HTP_OP_NORM: op_type = is_f16 ? "norm-f16" : "norm-f32"; break; + case HTP_OP_RMS_NORM: op_type = is_f16 ? "rmsnorm-f16" : "rmsnorm-f32"; break; + case HTP_OP_RMS_NORM_MUL: op_type = "rmsnorm-mul-f32"; break; + case HTP_OP_SCALE: op_type = is_f16 ? "scale-f16" : "scale-f32"; break; + case HTP_OP_CLAMP: op_type = is_f16 ? "clamp-f16" : "clamp-f32"; break; + case HTP_OP_SQR: op_type = is_f16 ? "sqr-f16" : "sqr-f32"; break; + case HTP_OP_SQRT: op_type = is_f16 ? "sqrt-f16" : "sqrt-f32"; break; + case HTP_OP_UNARY_NEG: op_type = "neg-f32"; break; + case HTP_OP_UNARY_EXP: op_type = "exp-f32"; break; + case HTP_OP_UNARY_SIGMOID: op_type = "sigmoid-f32"; break; + case HTP_OP_UNARY_SILU: op_type = "silu-f32"; break; + case HTP_OP_UNARY_GELU: op_type = "gelu-f32"; break; + case HTP_OP_UNARY_SOFTPLUS: op_type = "softplus-f32"; break; + case HTP_OP_UNARY_TANH: op_type = "tanh-f32"; break; + case HTP_OP_UNARY_ABS: op_type = is_f16 ? "abs-f16" : "abs-f32"; break; + case HTP_OP_UNARY_LOG: op_type = is_f16 ? "log-f16" : "log-f32"; break; + case HTP_OP_L2_NORM: op_type = is_f16 ? "l2norm-f16" : "l2norm-f32"; break; + case HTP_OP_TRI: op_type = "tri-f32"; break; default: FARF(ERROR, "Unsupported unary Op %u\n", octx->op); return HTP_STATUS_NO_SUPPORT; } + // F16 only has row-block kernels for this subset of ops (see the dispatch switch + // below) - reject everything else up front, before touching kparams/VTCM. + if (is_f16) { + switch (octx->op) { + case HTP_OP_NORM: + case HTP_OP_RMS_NORM: + case HTP_OP_SCALE: + case HTP_OP_CLAMP: + case HTP_OP_SQR: + case HTP_OP_SQRT: + case HTP_OP_L2_NORM: + case HTP_OP_UNARY_ABS: + case HTP_OP_UNARY_LOG: + break; + default: + FARF(ERROR, "unary-%s: not supported for F16\n", op_type); + return HTP_STATUS_NO_SUPPORT; + } + } + const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params; const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; const uint32_t n_threads = kparams->n_threads; - const size_t src0_data_row_size = src0->ne[0] * sizeof(float); - const size_t dst_data_row_size = dst->ne[0] * sizeof(float); + const size_t elem_size = is_f16 ? sizeof(_Float16) : sizeof(float); + + const size_t src0_data_row_size = src0->ne[0] * elem_size; + const size_t dst_data_row_size = dst->ne[0] * elem_size; const size_t src0_row_size_aligned = kparams->src0_row_size_aligned; const size_t dst_row_size_aligned = kparams->dst_row_size_aligned; + // Always 0 for F16 - htp_unary_vtcm_layout_build() keeps F16 on the row-block path, + // since only F32 has unary_task_f32_tiled_* kernels. const uint32_t col_tile = kparams->col_tile; size_t src1_data_row_size = 0; @@ -943,6 +1125,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) { bool broadcast_weight = kparams->broadcast_weight; const struct htp_tensor * src1 = NULL; + // RMS_NORM_MUL fusion is F32-only (its weight tensor is always F32; see + // try_fuse_node()'s type guard), so this never triggers when is_f16 is true. if (octx->op == HTP_OP_RMS_NORM_MUL) { src1 = octx->src[1]; src1_data_row_size = src1->ne[0] * sizeof(float); @@ -987,7 +1171,7 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) { .block = kparams->block, .nc = src0->ne[0], - .col_tile = (uint32_t) kparams->col_tile, + .col_tile = col_tile, .broadcast_weight = broadcast_weight, .vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, 0), @@ -1020,6 +1204,19 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) { case HTP_OP_TRI: task_func = unary_task_f32_tiled_tri; break; default: break; } + } else if (is_f16) { + switch (octx->op) { + case HTP_OP_NORM: task_func = unary_task_f16_norm; break; + case HTP_OP_RMS_NORM: task_func = unary_task_f16_rms_norm; break; + case HTP_OP_SCALE: task_func = unary_task_f16_scale; break; + case HTP_OP_CLAMP: task_func = unary_task_f16_clamp; break; + case HTP_OP_SQR: task_func = unary_task_f16_sqr; break; + case HTP_OP_SQRT: task_func = unary_task_f16_sqrt; break; + case HTP_OP_L2_NORM: task_func = unary_task_f16_l2_norm; break; + case HTP_OP_UNARY_ABS: task_func = unary_task_f16_unary_abs; break; + case HTP_OP_UNARY_LOG: task_func = unary_task_f16_unary_log; break; + default: break; + } } else { switch (octx->op) { case HTP_OP_NORM: task_func = unary_task_f32_norm; break; @@ -1047,7 +1244,7 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) { if (task_func) { worker_pool_run_func(octx->ctx->worker_pool, task_func, &uctx, n_threads); } else { - FARF(ERROR, "execute_op_unary_f32: task function is NULL for op %d\n", octx->op); + FARF(ERROR, "execute_op_unary: task function is NULL for op %d\n", octx->op); err = HTP_STATUS_NO_SUPPORT; } } @@ -1058,7 +1255,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) { int op_unary(struct htp_ops_context * octx) { switch (octx->src[0]->type) { case HTP_TYPE_F32: - return execute_op_unary_f32(octx); + case HTP_TYPE_F16: + return execute_op_unary(octx); default: return HTP_STATUS_NO_SUPPORT; diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.h index 458218ff4..116a591c2 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.h +++ b/ggml/src/ggml-hexagon/htp/unary-ops.h @@ -85,17 +85,19 @@ static inline void htp_unary_vtcm_layout_build( bool broadcast_weight, uint32_t n_threads, size_t vtcm_size, + size_t elem_size, uint32_t * out_col_tile, uint32_t * out_vtcm_row_per_thread ) { - const size_t src0_data_row_size = ne00 * sizeof(float); - const size_t dst_data_row_size = ne10 * sizeof(float); + const size_t src0_data_row_size = ne00 * elem_size; + const size_t dst_data_row_size = ne10 * elem_size; const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128); const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, 128); size_t src1_row_size_aligned = 0; if (op == HTP_OP_RMS_NORM_MUL) { + // RMS_NORM_MUL fusion is F32-only; its weight tensor is always F32. const size_t src1_data_row_size = ne11 * sizeof(float); src1_row_size_aligned = hex_round_up(src1_data_row_size, 128); } @@ -125,12 +127,19 @@ static inline void htp_unary_vtcm_layout_build( const bool is_reduction = (op == HTP_OP_NORM || op == HTP_OP_RMS_NORM || op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_L2_NORM); + // The tiled fallback path below only has F32 task functions (unary_task_f32_tiled_*); + // F16 has no tiled kernels, so it must stay on the row-block path like reduction ops. + // NOTE: if F16 ends up with vtcm_row_per_thread == 0 here (row too large for the VTCM + // budget), execute_op_unary() will see BLOCK == 0 and skip computation for that op + // (logged via FARF(ERROR, ...)) since there is no F16 tiled fallback. This is a known + // limitation; supporting it would require adding F16 tiled kernels. + const bool is_f16 = (elem_size == sizeof(_Float16)); uint32_t col_tile = 0; - if (vtcm_row_per_thread == 0 && !is_reduction) { + if (vtcm_row_per_thread == 0 && !is_reduction && !is_f16) { const size_t per_thread_budget = vtcm_size / n_threads; const size_t col_tile_bytes = hex_align_down(per_thread_budget / 4, 128); - col_tile = (uint32_t) (col_tile_bytes / sizeof(float)); + col_tile = (uint32_t) (col_tile_bytes / elem_size); L->src0_bytes = col_tile_bytes * 2; L->dst_bytes = col_tile_bytes * 2; From 9cffdcc801582616250520966699cb5b25d28243 Mon Sep 17 00:00:00 2001 From: Abhiram <78226909+geckguy@users.noreply.github.com> Date: Thu, 3 Sep 2026 01:54:31 +0530 Subject: [PATCH 11/17] server : accept data: URLs for input_video and input_audio (#27735) * server : accept data: URLs for input_video and input_audio input_video and input_audio passed accept_base64_uri=false to handle_media(), so data: URLs got treated as raw base64 strings and failed later with a confusing media probe error (#27724). pass true for these two content types the same way image_url already does, and allow video/audio mime types in the data: url check instead of image only. data URL validation now throws std::invalid_argument so malformed input comes back as 400 instead of 500, matching the other input validation in this file. * server : simplify handle_media and drop unused accept_base64_uri flag * server : update comment and add unit test for invalid data URI MIME --- tools/server/server-common.cpp | 23 +++++++++++----------- tools/server/tests/unit/test_vision_api.py | 1 + 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index c30955e89..2ac98b6fd 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -1062,8 +1062,7 @@ json oaicompat_completion_params_parse(const json & body) { static void handle_media( std::vector & out_files, const std::string & url, - const std::string & media_path, - bool accept_base64_uri) { + const std::string & media_path) { if (!media_path.empty()) { // should already be enforced by arg.cpp, but checking just in case GGML_ASSERT(media_path.back() == DIRECTORY_SEPARATOR); @@ -1104,15 +1103,17 @@ static void handle_media( data.assign((std::istreambuf_iterator(file)), std::istreambuf_iterator()); out_files.push_back(data); - } else if (accept_base64_uri && string_starts_with(url, "data:")) { - // try to decode base64 image + } else if (string_starts_with(url, "data:")) { + // try to decode base64 image, video, or audio std::vector parts = string_split(url, /*separator*/ ','); if (parts.size() != 2) { - throw std::runtime_error("Invalid uri-encoded base64 value"); - } else if (!string_starts_with(parts[0], "data:image/")) { - throw std::runtime_error("Invalid uri format: " + parts[0]); + throw std::invalid_argument("Invalid uri-encoded base64 value"); + } else if (!string_starts_with(parts[0], "data:image/") + && !string_starts_with(parts[0], "data:video/") + && !string_starts_with(parts[0], "data:audio/")) { + throw std::invalid_argument("Invalid uri format: " + parts[0]); } else if (!string_ends_with(parts[0], "base64")) { - throw std::runtime_error("uri must be base64 encoded"); + throw std::invalid_argument("uri must be base64 encoded"); } else { auto base64_data = parts[1]; auto decoded_data = base64_decode(base64_data); @@ -1219,7 +1220,7 @@ json oaicompat_chat_params_parse( json image_url = json_value(p, "image_url", json::object()); std::string url = json_value(image_url, "url", std::string()); - handle_media(out_files, url, opt.media_path, true); + handle_media(out_files, url, opt.media_path); p["type"] = "media_marker"; p["text"] = get_media_marker(); @@ -1234,7 +1235,7 @@ json oaicompat_chat_params_parse( json input_audio = json_value(p, "input_audio", json::object()); std::string url = json_value(input_audio, "data", json_value(input_audio, "url", std::string())); - handle_media(out_files, url, opt.media_path, false); + handle_media(out_files, url, opt.media_path); p["type"] = "media_marker"; p["text"] = get_media_marker(); @@ -1248,7 +1249,7 @@ json oaicompat_chat_params_parse( json input_video = json_value(p, "input_video", json::object()); std::string url = json_value(input_video, "data", json_value(input_video, "url", std::string())); - handle_media(out_files, url, opt.media_path, false); + handle_media(out_files, url, opt.media_path); p["type"] = "media_marker"; p["text"] = get_media_marker(); diff --git a/tools/server/tests/unit/test_vision_api.py b/tools/server/tests/unit/test_vision_api.py index 8b01c5372..3bf868e66 100644 --- a/tools/server/tests/unit/test_vision_api.py +++ b/tools/server/tests/unit/test_vision_api.py @@ -71,6 +71,7 @@ def test_v1_models_supports_multimodal_capability(): ("What is this:\n", "malformed", False, None), ("What is this:\n", "https://google.com/404", False, None), # non-existent image ("What is this:\n", "https://ggml.ai", False, None), # non-image data + ("What is this:\n", "data:text/html;base64,aGVsbG8=", False, None), # unsupported data uri mime # TODO @ngxson : test with multiple images, no images and with audio ] ) From 159b741427337a2e9a58b08121001545d66b5825 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Wed, 2 Sep 2026 23:53:32 +0200 Subject: [PATCH 12/17] finetune: fix no KV cache (#27199) * training: fix no KV cache * apply @ ggerganov suggestion --- examples/training/README.md | 2 ++ ggml/src/ggml.c | 2 +- src/llama-context.cpp | 12 +++++++++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/examples/training/README.md b/examples/training/README.md index df4252792..526ac258f 100644 --- a/examples/training/README.md +++ b/examples/training/README.md @@ -6,6 +6,8 @@ Finetuning of Stories 260K and LLaMA 3.2 1b seems to work with 24 GB of memory. **For CPU training, compile llama.cpp without any additional backends such as CUDA.** **For CUDA training, use the maximum number of GPU layers.** +Flash attention is disabled during training because `FLASH_ATTN_EXT` has no backward pass. + Proof of concept: ``` sh diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 8dc094508..2d5fdb7c1 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -7335,7 +7335,7 @@ void ggml_build_backward_expand( } // inplace operations are currently not supported - GGML_ASSERT(!node->view_src || node->op == GGML_OP_CPY || node->op == GGML_OP_VIEW || + GGML_ASSERT(!node->view_src || node->op == GGML_OP_CPY || node->op == GGML_OP_SET_ROWS || node->op == GGML_OP_VIEW || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_TRANSPOSE); const size_t ihash = ggml_hash_find(&cgraph->visited_hash_set, node); diff --git a/src/llama-context.cpp b/src/llama-context.cpp index f286bd1da..3cc27717e 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -482,7 +482,8 @@ llama_context::~llama_context() { // wait for any pending asynchronous copies into the output buffers before they are freed synchronize(); - if (!model.hparams.no_alloc) { + // when training, ggml_opt allocates extra buffers through the scheduler, so the sizes no longer match the expectation + if (!model.hparams.no_alloc && !opt_ctx) { for (size_t i = 0; i < backend_ptrs.size(); ++i) { ggml_backend_t backend = backend_ptrs[i]; ggml_backend_buffer_type_t buft = backend_buft[i]; @@ -3408,6 +3409,15 @@ void llama_context::opt_init(struct llama_model * model, struct llama_opt_params GGML_ASSERT(model->hparams.n_ctx_train % n_batch == 0); GGML_ASSERT(n_batch % n_ubatch == 0); + if (cparams.flash_attn) { + LLAMA_LOG_INFO("%s: disabling flash attention, FLASH_ATTN_EXT has no backward pass\n", __func__); + cparams.flash_attn = false; + + // the graph changes without flash attention, need to reserve again + sched_need_reserve = true; + sched_reserve(); + } + ggml_opt_params opt_params = ggml_opt_default_params(sched.get(), GGML_OPT_LOSS_TYPE_CROSS_ENTROPY); opt_params.opt_period = n_batch / n_ubatch; opt_params.get_opt_pars = lopt_params.get_opt_pars; From 67a17c17caa95742186f8b1ecadd1b5abd6d5ebb Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Thu, 3 Sep 2026 01:00:57 +0200 Subject: [PATCH 13/17] mtmd: fix idefics3 preproc (#28273) --- tools/mtmd/mtmd-image.cpp | 50 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index 65c24f4d4..890578978 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -980,6 +980,56 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i // // CITE: https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics3/image_processing_idefics3.py#L737 const clip_image_size original_size = img.get_size(); + + // old gguf files have no preprocessor longest size, custom token limits also need the generic size below + if (hparams.image_longest_edge > 0 && hparams.image_min_pixels <= 0 && hparams.image_max_pixels <= 0) { + const int tile_size = hparams.image_size; + const int longest_edge = hparams.image_longest_edge; + const double aspect_ratio = (double) original_size.width / original_size.height; + + clip_image_size resized_size; + if (original_size.width >= original_size.height) { + resized_size.width = longest_edge; + resized_size.height = (int) (longest_edge / aspect_ratio); + resized_size.height += resized_size.height % 2; + } else { + resized_size.height = longest_edge; + resized_size.width = (int) (longest_edge * aspect_ratio); + resized_size.width += resized_size.width % 2; + } + + const int grid_x = (resized_size.width + tile_size - 1) / tile_size; + const int grid_y = (resized_size.height + tile_size - 1) / tile_size; + const clip_image_size refined_size = clip_image_size{grid_x * tile_size, grid_y * tile_size}; + + clip_image_u8 resized_img; + img_tool::resize(img, resized_img, resized_size, hparams.image_resize_algo, PAD_NONE); + + clip_image_u8 refined_img; + img_tool::resize(resized_img, refined_img, refined_size, hparams.image_resize_algo, PAD_NONE); + + clip_image_u8 overview; + img_tool::resize(refined_img, overview, {tile_size, tile_size}, hparams.image_resize_algo, PAD_NONE); + + std::vector slices; + for (int y = 0; y < grid_y; y++) { + for (int x = 0; x < grid_x; x++) { + clip_image_u8 slice; + img_tool::crop(refined_img, slice, x * tile_size, y * tile_size, tile_size, tile_size); + slices.push_back(std::move(slice)); + } + } + + LOG_DBG("%s: grid size: %d x %d (%d tiles) + overview\n", __func__, grid_x, grid_y, grid_x * grid_y); + + mtmd_image_preproc_out output; + output.append_overview(hparams, overview, true); + output.append(hparams, slices, true); + output.grid_x = grid_x; + output.grid_y = grid_y; + return output; + } + const clip_image_size refined_size = img_tool::calc_size_preserved_ratio( original_size, { hparams.image_size, std::max(0, hparams.image_min_pixels), std::max(0, hparams.image_max_pixels), hparams.image_longest_edge }); From c61b98b875eaa5e654a3f5c73b34c310d2c6ab4c Mon Sep 17 00:00:00 2001 From: Yaniss Amazouz Date: Thu, 3 Sep 2026 09:53:08 +0300 Subject: [PATCH 14/17] model: add NVIDIA Nemotron-3-Puzzle-75B-A9B (NemotronHPuzzle) support (#25444) * hparams: add per-layer n_ff_exp/n_expert_used arrays with scalar-or-array loading G1/G2 infrastructure for variable-per-layer expert FFN size and top-k routing (required for Puzzle-75B which has 5 distinct n_ff_exp values and 7 top-k values across its 40 MoE layers). Design: rename scalar members to _impl suffix (following existing convention), add LLAMA_MAX_LAYERS arrays, add n_ff_exp(il)/n_expert_used(il) accessors with scalar fallback. No new GGUF keys: reuses existing expert_feed_forward_length and expert_used_count keys via get_key_or_arr (scalar -> broadcast, array -> per-layer). - llama-hparams.h: n_ff_exp -> n_ff_exp_impl, n_expert_used -> n_expert_used_impl; add n_ff_exp_arr / n_expert_used_arr arrays; add per-layer accessor declarations. - llama-hparams.cpp: implement n_ff_exp(il) and n_expert_used(il); out-of-range il returns impl safely (shared code, no abort). - llama-model.cpp: central n_expert_used load changed to get_key_or_arr; derive impl as max-of-array for validations and backward compat; zero both new arrays; HunyuanVL override also zeroes n_expert_used_arr. - llama-graph.cpp: aggregation loop in build_moe_ffn uses hparams.n_expert_used(il) so per-layer top-k bounds the ggml_view loop correctly. - All other files: mechanical rename hparams.n_{ff_exp,expert_used} -> *_impl. Scalar arches are unaffected (broadcast fills all array slots with the single value). (cherry picked from commit 269a81e03d66e1c353e1a203a0c03a03eb2b1a4e) * nemotron-h: use per-layer n_ff_exp(il) and n_expert_used(il) at MoE call-sites Load n_ff_exp via get_key_or_arr into hparams.n_ff_exp_arr in load_arch_hparams; derive impl as max for existing uniform GGUFs. In load_arch_tensors, compute n_ff_exp_i = hparams.n_ff_exp(i) with fallback to n_ff(i)/n_expert_used(i) for GGUFs that omit expert_feed_forward_length. In build_ffn_layer, pass hparams.n_expert_used(il) to build_moe_ffn so per-layer top-k is used for expert routing selection. All other nemotron-h behaviour (mamba2, attention, shared-exp, latent projection, routed_scaling_factor, expert_weights_norm, sigmoid gating) is unchanged. (cherry picked from commit b1878a101793cd4e59868ac72635a86ea694987c) * arch/*.cpp + gguf-py: mechanical rename n_ff_exp->n_ff_exp_impl, n_expert_used->n_expert_used_impl All non-nemotron arch files continue using the scalar impl member directly. Behaviour is identical: the impl value is the broadcast value from the GGUF scalar. gguf_writer: add_expert_feed_forward_length and add_expert_used_count now accept int | Sequence[int], mirroring add_feed_forward_length, so converters can write per-layer arrays with the same existing GGUF keys. (cherry picked from commit 8f009f54bea5ef9a6a354123bd25e9d5ea2d5e03) * convert: support NemotronHPuzzleForCausalLM (per-block MoE config) Parse block_configs/mtp_block_configs into per-layer arrays (scalar-or-array keys), append the MTP [attention, moe] sub-blocks as blk.88/blk.89 with nextn tensors, accept the backbone.* prefix, and register the arch. Also fix a pre-existing undeclared _experts attribute on NemotronHModel. (cherry picked from commit d1a592f278336e78457454eb6c96bca917135f10) * nemotron-h: distinguish Nemotron 3 Puzzle (75B.A9B) from Super (120B.A12B) Both have 88 layers; the per-layer expert_used_count array (heterogeneous for Puzzle, broadcast-uniform for Super) is the discriminator. (cherry picked from commit f824e09dc812169589cf5662c92d149a4c18c30a) * convert: accept the official Puzzle BF16 checkpoint's tensor naming The officially distributed BF16 checkpoint (NVIDIA-Nemotron-Labs-3-Puzzle- 75B-A9B-BF16) names the trunk model.* (model.layers.*, model.embeddings, model.norm_f) where the original release used the NemotronH-style backbone.*, and spells the router bias e_score_correction_bias instead of e_score_correction.bias. Normalize both at the top of NemotronHPuzzleModel.modify_tensors so either checkpoint converts; every tensor name in the official index (42683 keys, MTP head included) resolves through the tensor map after normalization. (cherry picked from commit 189b67fc2c9d50970416c94b3317a6e7baa49b03) * laguna: use n_ff_exp_impl for the uniform-MoE FFN size Laguna landed after this branch was cut and reads hparams.n_ff_exp as a scalar. This series turns it into a per-layer array with an n_ff_exp(il) accessor, so the three scalar reads no longer compile. Laguna is a uniform MoE, so point them at the scalar fallback n_ff_exp_impl, same as deepseek2/qwen3moe/gemma4 in this series. No behaviour change. (cherry picked from commit dbedc9e19c50dca0acdfb402362e2707bee424ae) * arch: extend the n_ff_exp/n_expert_used rename to archs added upstream kimi-k3, dflash, bailingmoe3, deepseek4, granite-swa and the nemotron-h MTP block still referenced the scalar fields by their old names. n_ff_exp and n_expert_used are accessors now, so those reads no longer compile; point the non-per-layer archs at the _impl scalars and use the indexed form where the call site is per-layer. * convert: keep Puzzle opted out of the NemotronH MTP export path #26725 added MTP export to NemotronHModel, keyed on num_nextn_predict_layers. Puzzle's config carries that key, but NemotronHPuzzleModel bypasses NemotronHModel.__init__ (its per-block config needs a different setup), so _mtp_bid was never assigned and modify_tensors raised AttributeError on any mtp.* tensor. Puzzle's head is also laid out by mtp_block_configs, not the mtp.layers.* form the base maps. Set _mtp_bid to None, drop mtp.* in filter_tensors, and declare supports_mtp_export = False so --mtp / --no-mtp fail at the CLI. * llama: replace n_ff_exp/n_expert_used scalars with per-layer accessors Follow-up to review feedback: the previous revision kept the scalar hparams fields alongside the new per-layer arrays, which duplicated state that get_key_or_arr already handles by broadcasting a scalar value over every layer. Drop both scalars and expose n_ff_exp(il) / n_expert_used(il) built exactly like the existing n_head_kv(il) and n_ff(il) accessors: they index the array and GGML_ABORT out of range, with il defaulting to 0 so genuinely uniform call sites stay a plain n_ff_exp(). Arch loaders now read both keys through get_key_or_arr over n_layer_all, and the n_expert_used validation checks the maximum across layers instead of a single field. * llama: restore per-key required flags on the expert hparam reads The scalar-to-array conversion passed required=false at every call site, which silently made mandatory keys optional. Each read now carries the same required flag it had before the conversion. --- conversion/__init__.py | 1 + conversion/nemotron.py | 87 +++++++++++++++++++++++++++++++++++++ gguf-py/gguf/gguf_writer.py | 14 ++++-- src/llama-graph.cpp | 15 ++++--- src/llama-hparams.cpp | 16 +++++++ src/llama-hparams.h | 11 ++++- src/llama-model-loader.cpp | 4 +- src/llama-model-saver.cpp | 4 +- src/llama-model.cpp | 51 +++++++++++++--------- src/llama-model.h | 3 +- src/models/afmoe.cpp | 4 +- src/models/bailingmoe.cpp | 4 +- src/models/bailingmoe2.cpp | 4 +- src/models/bailingmoe3.cpp | 16 +++---- src/models/bert.cpp | 2 +- src/models/cohere2moe.cpp | 6 +-- src/models/deepseek.cpp | 6 +-- src/models/deepseek2.cpp | 4 +- src/models/deepseek2ocr.cpp | 4 +- src/models/deepseek32.cpp | 6 +-- src/models/deepseek4.cpp | 8 ++-- src/models/dflash.cpp | 6 +-- src/models/dots1.cpp | 4 +- src/models/dots3note.cpp | 4 +- src/models/ernie4-5.cpp | 4 +- src/models/exaone-moe.cpp | 4 +- src/models/gemma4.cpp | 4 +- src/models/glm-dsa.cpp | 6 +-- src/models/glm4-moe.cpp | 6 +-- src/models/granite-swa.cpp | 2 +- src/models/grok.cpp | 4 +- src/models/grovemoe.cpp | 4 +- src/models/hunyuan-moe.cpp | 2 +- src/models/hy-v3.cpp | 4 +- src/models/kimi-k3.cpp | 6 +-- src/models/kimi-linear.cpp | 6 +-- src/models/laguna.cpp | 6 +-- src/models/lfm2.cpp | 6 +-- src/models/lfm2moe.cpp | 8 ++-- src/models/llada-moe.cpp | 4 +- src/models/llama4.cpp | 4 +- src/models/mellum.cpp | 4 +- src/models/mimo2.cpp | 4 +- src/models/minimax-m2.cpp | 2 +- src/models/minimax-m3.cpp | 4 +- src/models/nemotron-h.cpp | 30 ++++++++++--- src/models/openai-moe.cpp | 4 +- src/models/qwen2moe.cpp | 4 +- src/models/qwen35moe.cpp | 6 +-- src/models/qwen3moe.cpp | 4 +- src/models/qwen3next.cpp | 4 +- src/models/qwen3vlmoe.cpp | 4 +- src/models/qwen4exp.cpp | 4 +- src/models/rnd1.cpp | 4 +- src/models/smallthinker.cpp | 4 +- src/models/step35.cpp | 6 +-- 56 files changed, 298 insertions(+), 154 deletions(-) diff --git a/conversion/__init__.py b/conversion/__init__.py index 254a3e6c8..ba73192ef 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -188,6 +188,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "NanbeigeForCausalLM": "nanbeige", "NemotronForCausalLM": "nemotron", "NemotronHForCausalLM": "nemotron", + "NemotronHPuzzleForCausalLM": "nemotron", "NeoBERT": "bert", "NeoBERTForSequenceClassification": "bert", "NeoBERTLMHead": "bert", diff --git a/conversion/nemotron.py b/conversion/nemotron.py index 07fbc6531..c7adb2e27 100644 --- a/conversion/nemotron.py +++ b/conversion/nemotron.py @@ -5,6 +5,7 @@ from typing import Any, Callable, Iterable, TYPE_CHECKING import torch if TYPE_CHECKING: + from pathlib import Path from torch import Tensor from .base import MmprojModel, ModelBase, TextModel, gguf, logger @@ -201,6 +202,7 @@ class NemotronHModel(GraniteHybridModel): model_arch = gguf.MODEL_ARCH.NEMOTRON_H is_moe: bool = False supports_mtp_export = True + _experts: list[dict[str, Tensor]] | None = None _SSM_LAYER_TYPES = {"mamba", "linear_attention"} _ATTN_LAYER_TYPES = {"attention", "full_attention"} @@ -513,3 +515,88 @@ class NemotronHModel(GraniteHybridModel): experts = [k for d in self._experts for k in d.keys()] if len(experts) > 0: raise ValueError(f"Unprocessed experts: {experts}") + + +@ModelBase.register("NemotronHPuzzleForCausalLM") +@ModelBase.example("nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16") +class NemotronHPuzzleModel(NemotronHModel): + """NVIDIA Puzzle: NemotronH with a per-block MoE config (block_configs). + + The checkpoint also ships an MTP draft head (mtp.safetensors). It is skipped + here: there is no Puzzle MTP inference path in tree, and the head is laid out + by mtp_block_configs rather than the mtp.layers.* form NemotronHModel maps.""" + + model_arch = gguf.MODEL_ARCH.NEMOTRON_H_MOE + is_moe: bool = True + supports_mtp_export = False + + def __init__(self, dir_model: "Path", *args, **kwargs): + hparams = dict(kwargs.pop("hparams", None) or ModelBase.load_hparams(dir_model, self.is_mistral_format)) + + self.block_configs: list[dict] = hparams["block_configs"] + self.n_layer_trunk = len(self.block_configs) + + # block_configs carries the per-block MoE shape, and is the authority on the + # block pattern too: the layers_block_type the HF config wrapper computes is + # not sized to it. + hparams["num_hidden_layers"] = self.n_layer_trunk + hparams["layers_block_type"] = [bc["block_type"] for bc in self.block_configs] + + self.model_arch = gguf.MODEL_ARCH.NEMOTRON_H_MOE + + # Bypass NemotronHModel.__init__: it assumes a flat num_experts_per_tok / + # moe_intermediate_size and a layers_block_type sized to block_count, neither + # of which hold for Puzzle's per-block config. + GraniteHybridModel.__init__(self, dir_model, *args, hparams=hparams, **kwargs) + + self.head_dim = self.find_hparam(["head_dim", "attention_head_dim"]) + self.d_inner = self.find_hparam(["num_heads"]) * self.d_model + + # NemotronHModel.__init__ folds an MTP block into block_count when the + # config carries num_nextn_predict_layers; Puzzle's config does, but its + # head has a different layout and no inference path, so stay opted out. + self._mtp_bid = None + + def set_gguf_parameters(self): + GraniteHybridModel.set_gguf_parameters(self) + + head_dim = self.head_dim + if head_dim is None: + raise ValueError("Could not find the attention head dim in config") + self.gguf_writer.add_key_length(head_dim) + self.gguf_writer.add_value_length(head_dim) + + ffn_lengths = [bc.get("moe_intermediate_size") or 0 for bc in self.block_configs] + experts_used = [bc.get("num_experts_per_tok") or 0 for bc in self.block_configs] + + self.gguf_writer.add_feed_forward_length(ffn_lengths) + self.gguf_writer.add_expert_feed_forward_length(ffn_lengths) + self.gguf_writer.add_expert_used_count(experts_used) + + self.gguf_writer.add_expert_shared_feed_forward_length(self.hparams["moe_shared_expert_intermediate_size"]) + self.gguf_writer.add_expert_count(self.hparams["n_routed_experts"]) + self.gguf_writer.add_expert_shared_count(self.hparams["n_shared_experts"]) + self.gguf_writer.add_expert_weights_norm(self.hparams["norm_topk_prob"]) + self.gguf_writer.add_expert_weights_scale(self.hparams["routed_scaling_factor"]) + self.gguf_writer.add_expert_group_count(self.hparams["n_group"]) + self.gguf_writer.add_moe_latent_size(self.hparams["moe_latent_size"]) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # The official BF16 checkpoint (NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16) + # names the trunk "model.*" (model.layers.*, model.embeddings, model.norm_f) + # where the original release used the NemotronH-style "backbone.*", and spells + # the router bias "e_score_correction_bias" instead of "e_score_correction.bias"; + # normalize so both convert identically. + if name.startswith("model."): + name = "backbone." + name[len("model."):] + if name.endswith("mixer.gate.e_score_correction_bias"): + name = name[: -len("e_score_correction_bias")] + "e_score_correction.bias" + + yield from super().modify_tensors(data_torch, name, bid) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + # Drop the MTP head unconditionally; see the class docstring. + if item[0].startswith("mtp."): + return None + return super().filter_tensors(item) diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index d95fe9b1a..689c2fca1 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -733,8 +733,11 @@ class GGUFWriter: else: self.add_array(Keys.LLM.FEED_FORWARD_LENGTH.format(arch=self.arch), length) - def add_expert_feed_forward_length(self, length: int) -> None: - self.add_uint32(Keys.LLM.EXPERT_FEED_FORWARD_LENGTH.format(arch=self.arch), length) + def add_expert_feed_forward_length(self, length: int | Sequence[int]) -> None: + if isinstance(length, int): + self.add_uint32(Keys.LLM.EXPERT_FEED_FORWARD_LENGTH.format(arch=self.arch), length) + else: + self.add_array(Keys.LLM.EXPERT_FEED_FORWARD_LENGTH.format(arch=self.arch), length) def add_expert_shared_feed_forward_length(self, length: int) -> None: self.add_uint32(Keys.LLM.EXPERT_SHARED_FEED_FORWARD_LENGTH.format(arch=self.arch), length) @@ -860,8 +863,11 @@ class GGUFWriter: def add_expert_count(self, count: int) -> None: self.add_uint32(Keys.LLM.EXPERT_COUNT.format(arch=self.arch), count) - def add_expert_used_count(self, count: int) -> None: - self.add_uint32(Keys.LLM.EXPERT_USED_COUNT.format(arch=self.arch), count) + def add_expert_used_count(self, count: int | Sequence[int]) -> None: + if isinstance(count, int): + self.add_uint32(Keys.LLM.EXPERT_USED_COUNT.format(arch=self.arch), count) + else: + self.add_array(Keys.LLM.EXPERT_USED_COUNT.format(arch=self.arch), count) def add_expert_shared_count(self, count: int) -> None: self.add_uint32(Keys.LLM.EXPERT_SHARED_COUNT.format(arch=self.arch), count) diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 6b7eec14f..274a62643 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1466,7 +1466,7 @@ llm_graph_context::llm_graph_context(const llm_graph_params & params) : n_embd_head_v (hparams.n_embd_head_v()), n_embd_v_gqa (hparams.n_embd_v_gqa()), n_expert (hparams.n_expert), - n_expert_used (cparams.warmup ? hparams.n_expert : hparams.n_expert_used), + n_expert_used (cparams.warmup ? hparams.n_expert : hparams.n_expert_used()), freq_base (cparams.rope_freq_base), freq_scale (cparams.rope_freq_scale), ext_factor (cparams.yarn_ext_factor), @@ -2270,25 +2270,26 @@ ggml_tensor * llm_graph_context::build_moe_ffn( assert(n_expert_used > 0); // order the views before the adds - for (uint32_t i = 0; i < hparams.n_expert_used; ++i) { + // Use per-layer n_expert_used to bound the graph even during warmup (avoids + // the large-add-nodes issue for uniform arches; for Puzzle the per-layer + // value is correct). ref: https://github.com/ggml-org/llama.cpp/pull/14753 + const uint32_t n_expert_used_il = hparams.n_expert_used(il); + for (uint32_t i = 0; i < n_expert_used_il; ++i) { cur_experts[i] = ggml_view_2d(ctx0, experts, n_embd, n_tokens, experts->nb[2], i*experts->nb[1]); ggml_build_forward_expand(gf, cur_experts[i]); } // aggregate experts - // note: here we explicitly use hparams.n_expert_used instead of n_expert_used - // to avoid potentially a large number of add nodes during warmup - // ref: https://github.com/ggml-org/llama.cpp/pull/14753 ggml_tensor * moe_out = cur_experts[0]; - for (uint32_t i = 1; i < hparams.n_expert_used; ++i) { + for (uint32_t i = 1; i < n_expert_used_il; ++i) { moe_out = ggml_add(ctx0, moe_out, cur_experts[i]); ggml_build_forward_expand(gf, moe_out); } - if (hparams.n_expert_used == 1) { + if (n_expert_used_il == 1) { // avoid returning a non-contiguous tensor moe_out = ggml_cont(ctx0, moe_out); } diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp index 6a820c61c..7df82ffe2 100644 --- a/src/llama-hparams.cpp +++ b/src/llama-hparams.cpp @@ -71,6 +71,22 @@ uint32_t llama_hparams::n_ff(uint32_t il) const { GGML_ABORT("fatal error"); } +uint32_t llama_hparams::n_ff_exp(uint32_t il) const { + if (il < n_layer_all) { + return n_ff_exp_arr[il]; + } + + GGML_ABORT("fatal error"); +} + +uint32_t llama_hparams::n_expert_used(uint32_t il) const { + if (il < n_layer_all) { + return n_expert_used_arr[il]; + } + + GGML_ABORT("fatal error"); +} + uint32_t llama_hparams::n_gqa(uint32_t il) const { const uint32_t n_head = this->n_head(il); const uint32_t n_head_kv = this->n_head_kv(il); diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 390d3543d..2f238a174 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -62,7 +62,6 @@ struct llama_hparams { // per-token adapter selection. -1 when the model has no such layer. int32_t router_layer = -1; uint32_t n_expert = 0; - uint32_t n_expert_used = 0; uint32_t n_rel_attn_bkts = 0; // TODO: this needs to be reworked @@ -92,10 +91,14 @@ struct llama_hparams { std::array n_head_kv_arr; std::array n_ff_arr; + // per-layer expert feed-forward size + std::array n_ff_exp_arr; + // per-layer top-k expert routing count + std::array n_expert_used_arr; + uint32_t n_layer_dense_lead = 0; uint32_t n_lora_q = 0; uint32_t n_lora_kv = 0; - uint32_t n_ff_exp = 0; uint32_t n_ff_shexp = 0; uint32_t n_ff_chexp = 0; uint32_t n_expert_shared = 0; @@ -385,6 +388,10 @@ struct llama_hparams { uint32_t n_ff(uint32_t il = 0) const; + uint32_t n_ff_exp(uint32_t il = 0) const; + + uint32_t n_expert_used(uint32_t il = 0) const; + uint32_t n_gqa(uint32_t il = 0) const; uint32_t n_rot(uint32_t il = 0) const; diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 7663797ba..d940b1b61 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -951,7 +951,7 @@ static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * w case GGML_OP_MUL_MAT_ID: { // Used for either MoE expert routing or embedded adapter routing - const int n_ids_used = hparams.router_layer >= 0 ? 1 : hparams.n_expert_used; + const int n_ids_used = hparams.router_layer >= 0 ? 1 : hparams.n_expert_used(); GGML_ASSERT(n_ids_used > 0); ggml_tensor * b = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_ids_used, 512); ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_ids_used, 512); @@ -964,7 +964,7 @@ static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * w } break; case GGML_OP_ADD_ID: { - const int n_expert_used = hparams.n_expert_used; + const int n_expert_used = hparams.n_expert_used(); GGML_ASSERT(n_expert_used > 0); ggml_tensor * a = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_expert_used, 512); ggml_tensor * c = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_expert_used, 512); diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 8860bd3f4..919e90ecc 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -222,7 +222,7 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_BLOCK_COUNT, hparams.n_layer_all); add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead); add_kv(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, true); - add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp()); add_kv(LLM_KV_EXPERT_LATENT_LENGTH, hparams.n_expert_latent); add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp); add_kv(LLM_KV_EXPERT_CHUNK_FEED_FORWARD_LENGTH, hparams.n_ff_chexp); @@ -233,7 +233,7 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_USE_PARALLEL_RESIDUAL, hparams.use_par_res); // add_kv(LLM_KV_TENSOR_DATA_LAYOUT, ???); add_kv(LLM_KV_EXPERT_COUNT, hparams.n_expert); - add_kv(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used); + add_kv(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used()); add_kv(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); add_kv(LLM_KV_EXPERT_GROUP_COUNT, hparams.n_expert_groups); add_kv(LLM_KV_EXPERT_GROUP_USED_COUNT, hparams.n_group_used); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index bfce09de0..408e76e32 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -634,7 +634,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str // the FFN is the same for Qwen 3 Next and Qwen 3.5: if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) { - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(il); GGML_ASSERT(tensor->ne[axis] == 2*n_ff_exp); return {{n_ff_exp, 2}}; } @@ -657,7 +657,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str return {{tensor->ne[axis], 1}}; } if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) { - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(il); GGML_ASSERT(tensor->ne[axis] == 2*n_ff_exp); return {{n_ff_exp, 2}}; } @@ -943,6 +943,7 @@ const char * llm_type_name(llm_type type) { case LLM_TYPE_31B_A3_5B: return "31B.A3.5B"; case LLM_TYPE_35B_A3B: return "35B.A3B"; case LLM_TYPE_48B_A3B: return "48B.A3B"; + case LLM_TYPE_75B_A9B: return "75B.A9B"; case LLM_TYPE_80B_A3B: return "80B.A3B"; case LLM_TYPE_A3B: return "A3B"; case LLM_TYPE_100B_A6B: return "100B.A6B"; @@ -1226,14 +1227,15 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false); GGML_ASSERT(hparams.n_layer_nextn <= hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false); - ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false); + std::fill(hparams.n_expert_used_arr.begin(), hparams.n_expert_used_arr.end(), 0); + ml.get_key_or_arr(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used_arr, hparams.n_layer_all, false); ml.get_key(LLM_KV_EXPERT_GROUP_COUNT, hparams.n_expert_groups, false); ml.get_key(LLM_KV_EXPERT_GROUP_USED_COUNT, hparams.n_group_used, false); if (arch == LLM_ARCH_HUNYUAN_VL || arch == LLM_ARCH_HUNYUAN_DENSE) { if (hparams.n_expert <= 1) { - hparams.n_expert = 0; - hparams.n_expert_used = 0; + hparams.n_expert = 0; + std::fill(hparams.n_expert_used_arr.begin(), hparams.n_expert_used_arr.end(), 0); } } @@ -1251,10 +1253,16 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { GGML_ASSERT(hparams.convnext.n_layer <= hparams.n_layer_all); } + // models may route a different number of experts per layer, so validate the maximum + uint32_t n_expert_used_max = 0; + for (uint32_t il = 0; il < hparams.n_layer_all; ++il) { + n_expert_used_max = std::max(n_expert_used_max, hparams.n_expert_used(il)); + } + GGML_ASSERT(hparams.n_expert <= LLAMA_MAX_EXPERTS); - GGML_ASSERT(hparams.n_expert_used <= hparams.n_expert); + GGML_ASSERT(n_expert_used_max <= hparams.n_expert); if (hparams.n_expert > 0) { - GGML_ASSERT(hparams.n_expert_used > 0); + GGML_ASSERT(n_expert_used_max > 0); GGML_ASSERT(hparams.n_expert_groups < hparams.n_expert); if (hparams.n_expert_groups > 1) { GGML_ASSERT(hparams.n_expert % hparams.n_expert_groups == 0); @@ -1262,13 +1270,14 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { GGML_ASSERT(hparams.n_group_used < hparams.n_expert_groups); } } else { - GGML_ASSERT(hparams.n_expert_used == 0); + GGML_ASSERT(n_expert_used_max == 0); GGML_ASSERT(hparams.n_expert_groups == 0); } - std::fill(hparams.n_head_arr.begin(), hparams.n_head_arr.end(), 0); - std::fill(hparams.n_head_kv_arr.begin(), hparams.n_head_kv_arr.end(), 0); - std::fill(hparams.n_ff_arr.begin(), hparams.n_ff_arr.end(), 0); + std::fill(hparams.n_head_arr.begin(), hparams.n_head_arr.end(), 0); + std::fill(hparams.n_head_kv_arr.begin(), hparams.n_head_kv_arr.end(), 0); + std::fill(hparams.n_ff_arr.begin(), hparams.n_ff_arr.end(), 0); + std::fill(hparams.n_ff_exp_arr.begin(), hparams.n_ff_exp_arr.end(), 0); std::fill(hparams.rope_sections.begin(), hparams.rope_sections.end(), 0); std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), 1); @@ -1501,7 +1510,7 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { const auto tn = LLM_TN(arch); const int64_t n_expert = hparams.n_expert; - const int64_t n_expert_used = hparams.n_expert_used; + const int64_t n_expert_used = hparams.n_expert_used(); if (n_expert > 0 && n_expert_used == 0) { throw std::runtime_error("model has expert layers but no expert layers are used"); @@ -1957,7 +1966,7 @@ void llama_model::print_info() const { LLAMA_LOG_INFO("%s: f_attn_value_scale = %.4f\n", __func__, hparams.f_attn_value_scale); LLAMA_LOG_INFO("%s: n_ff = %s\n", __func__, print_f([&](uint32_t il) { return hparams.n_ff(il); }, hparams.n_layer_all).c_str()); LLAMA_LOG_INFO("%s: n_expert = %u\n", __func__, hparams.n_expert); - LLAMA_LOG_INFO("%s: n_expert_used = %u\n", __func__, hparams.n_expert_used); + LLAMA_LOG_INFO("%s: n_expert_used = %u\n", __func__, hparams.n_expert_used()); LLAMA_LOG_INFO("%s: n_expert_groups = %d\n", __func__, hparams.n_expert_groups); LLAMA_LOG_INFO("%s: n_group_used = %d\n", __func__, hparams.n_group_used); LLAMA_LOG_INFO("%s: causal attn = %d\n", __func__, hparams.causal_attn); @@ -2032,7 +2041,7 @@ void llama_model::print_info() const { if (arch == LLM_ARCH_DEEPSEEK) { LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead); - LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp); + LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp()); LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared); LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale); } @@ -2045,7 +2054,7 @@ void llama_model::print_info() const { LLAMA_LOG_INFO("%s: n_lora_kv = %d\n", __func__, hparams.n_lora_kv); LLAMA_LOG_INFO("%s: n_embd_head_k_mla = %d\n", __func__, hparams.n_embd_head_k_mla()); LLAMA_LOG_INFO("%s: n_embd_head_v_mla = %d\n", __func__, hparams.n_embd_head_v_mla()); - LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp); + LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp()); LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared); LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale); LLAMA_LOG_INFO("%s: expert_weights_norm = %d\n", __func__, hparams.expert_weights_norm); @@ -2053,7 +2062,7 @@ void llama_model::print_info() const { } if (arch == LLM_ARCH_QWEN2MOE) { - LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp); + LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp()); LLAMA_LOG_INFO("%s: n_ff_shexp = %d\n", __func__, hparams.n_ff_shexp); } @@ -2063,7 +2072,7 @@ void llama_model::print_info() const { arch == LLM_ARCH_OPENAI_MOE || arch == LLM_ARCH_QWEN3VLMOE || arch == LLM_ARCH_RND1) { - LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp); + LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp()); } if (arch == LLM_ARCH_MINICPM || @@ -2080,7 +2089,7 @@ void llama_model::print_info() const { if (arch == LLM_ARCH_BAILINGMOE) { LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead); - LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp); + LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp()); LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared); LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale); LLAMA_LOG_INFO("%s: expert_weights_norm = %d\n", __func__, hparams.expert_weights_norm); @@ -2088,7 +2097,7 @@ void llama_model::print_info() const { if (arch == LLM_ARCH_BAILINGMOE2 || arch == LLM_ARCH_BAILINGMOE3) { LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead); - LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp); + LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp()); LLAMA_LOG_INFO("%s: n_ff_shexp = %d\n", __func__, hparams.n_ff_shexp); LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared); LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale); @@ -2098,12 +2107,12 @@ void llama_model::print_info() const { } if (arch == LLM_ARCH_SMALLTHINKER || arch == LLM_ARCH_LFM2MOE) { - LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp); + LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp()); LLAMA_LOG_INFO("%s: expert_gating_func = %s\n", __func__, llama_expert_gating_func_name((llama_expert_gating_func_type) hparams.expert_gating_func)); } if (arch == LLM_ARCH_GROVEMOE) { - LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp); + LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp()); LLAMA_LOG_INFO("%s: n_ff_chexp = %d\n", __func__, hparams.n_ff_chexp); LLAMA_LOG_INFO("%s: n_group_experts = %d\n", __func__, hparams.n_group_experts); LLAMA_LOG_INFO("%s: expert_group_scale = %.2f\n", __func__, hparams.expert_group_scale); diff --git a/src/llama-model.h b/src/llama-model.h index ee6bb5ac3..4c4a30e01 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -128,6 +128,7 @@ enum llm_type { LLM_TYPE_31B_A3_5B, LLM_TYPE_35B_A3B, // Qwen3.5 LLM_TYPE_48B_A3B, // Kimi Linear + LLM_TYPE_75B_A9B, // Nemotron 3 Puzzle LLM_TYPE_80B_A3B, // Qwen3 Next LLM_TYPE_A3B, // Qwen3.8 Flash Next LLM_TYPE_100B_A6B, @@ -839,7 +840,7 @@ const char * llm_type_name(llm_type type); const int64_t n_token_types = vocab.n_token_types(); GGML_UNUSED(n_token_types); \ const int64_t n_rot = hparams.n_rot(); GGML_UNUSED(n_rot); \ const int64_t n_expert = hparams.n_expert; GGML_UNUSED(n_expert); \ - const int64_t n_expert_used = hparams.n_expert_used; GGML_UNUSED(n_expert_used); \ + const int64_t n_expert_used = hparams.n_expert_used(); GGML_UNUSED(n_expert_used); \ const int64_t n_ctx_train = hparams.n_ctx_train; GGML_UNUSED(n_ctx_train); // For internal test use diff --git a/src/models/afmoe.cpp b/src/models/afmoe.cpp index 063b21425..cf0220367 100644 --- a/src/models/afmoe.cpp +++ b/src/models/afmoe.cpp @@ -3,7 +3,7 @@ void llama_model_afmoe::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); @@ -52,7 +52,7 @@ void llama_model_afmoe::load_arch_tensors(llama_model_loader &) { output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); } - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); for (int i = 0; i < n_layer; ++i) { auto & layer = layers[i]; diff --git a/src/models/bailingmoe.cpp b/src/models/bailingmoe.cpp index 7faf73c83..9d1073ae1 100644 --- a/src/models/bailingmoe.cpp +++ b/src/models/bailingmoe.cpp @@ -3,7 +3,7 @@ void llama_model_bailingmoe::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); @@ -19,7 +19,7 @@ void llama_model_bailingmoe::load_arch_tensors(llama_model_loader &) { LLAMA_LOAD_LOCALS; const int64_t n_expert_shared = hparams.n_expert_shared; - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); diff --git a/src/models/bailingmoe2.cpp b/src/models/bailingmoe2.cpp index 8fc0ea752..24fc4e022 100644 --- a/src/models/bailingmoe2.cpp +++ b/src/models/bailingmoe2.cpp @@ -3,7 +3,7 @@ void llama_model_bailingmoe2::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); @@ -21,7 +21,7 @@ void llama_model_bailingmoe2::load_arch_tensors(llama_model_loader &) { LLAMA_LOAD_LOCALS; const int64_t n_expert_shared = hparams.n_expert_shared; - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); diff --git a/src/models/bailingmoe3.cpp b/src/models/bailingmoe3.cpp index 5ebedaecb..1f2592cfa 100644 --- a/src/models/bailingmoe3.cpp +++ b/src/models/bailingmoe3.cpp @@ -15,7 +15,7 @@ void llama_model_bailingmoe3::load_arch_hparams(llama_model_loader & ml) { hparams.kda_safe_gate = true; } ml.get_key(LLM_KV_KDA_GATE_LOWER_BOUND, hparams.kda_gate_lower_bound); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead); @@ -26,7 +26,7 @@ void llama_model_bailingmoe3::load_arch_hparams(llama_model_loader & ml) { ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer_all, false); if (hparams.n_ff_shexp == 0) { - hparams.n_ff_shexp = hparams.n_ff_exp * std::max(1u, hparams.n_expert_shared); + hparams.n_ff_shexp = hparams.n_ff_exp() * std::max(1u, hparams.n_expert_shared); } GGML_ASSERT(hparams.kda_safe_gate); @@ -115,9 +115,9 @@ void llama_model_bailingmoe3::load_arch_tensors(llama_model_loader & ml) { } else { layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", il), { n_embd, n_expert }, trunk_flags); layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", il), { n_expert }, trunk_flags); - layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", il), { n_embd, hparams.n_ff_exp, n_expert }, trunk_flags); - layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", il), { n_embd, hparams.n_ff_exp, n_expert }, trunk_flags); - layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), { hparams.n_ff_exp, n_embd, n_expert }, trunk_flags); + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", il), { n_embd, hparams.n_ff_exp(), n_expert }, trunk_flags); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", il), { n_embd, hparams.n_ff_exp(), n_expert }, trunk_flags); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), { hparams.n_ff_exp(), n_embd, n_expert }, trunk_flags); layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", il), { n_embd, hparams.n_ff_shexp }, trunk_flags); layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", il), { n_embd, hparams.n_ff_shexp }, trunk_flags); layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", il), { hparams.n_ff_shexp, n_embd }, trunk_flags); @@ -145,9 +145,9 @@ void llama_model_bailingmoe3::load_arch_tensors(llama_model_loader & ml) { layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", il), { n_embd }, flags); layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", il), { n_embd, n_expert }, flags); layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", il), { n_expert }, flags); - layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", il), { n_embd, hparams.n_ff_exp, n_expert }, flags); - layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", il), { n_embd, hparams.n_ff_exp, n_expert }, flags); - layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), { hparams.n_ff_exp, n_embd, n_expert }, flags); + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", il), { n_embd, hparams.n_ff_exp(), n_expert }, flags); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", il), { n_embd, hparams.n_ff_exp(), n_expert }, flags); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), { hparams.n_ff_exp(), n_embd, n_expert }, flags); layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", il), { n_embd, hparams.n_ff_shexp }, flags); layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", il), { n_embd, hparams.n_ff_shexp }, flags); layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", il), { hparams.n_ff_shexp, n_embd }, flags); diff --git a/src/models/bert.cpp b/src/models/bert.cpp index 53ce29f23..ca0281d30 100644 --- a/src/models/bert.cpp +++ b/src/models/bert.cpp @@ -182,7 +182,7 @@ llama_model_bert::graph::graph(const llama_model & model, const llm_graph_params nullptr, model.layers[il].ffn_down_exps, nullptr, - hparams.n_expert, hparams.n_expert_used, + hparams.n_expert, hparams.n_expert_used(), LLM_FFN_GELU, false, hparams.expert_weights_scale, LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, diff --git a/src/models/cohere2moe.cpp b/src/models/cohere2moe.cpp index c50910edc..5e02cd56e 100644 --- a/src/models/cohere2moe.cpp +++ b/src/models/cohere2moe.cpp @@ -13,7 +13,7 @@ void llama_model_cohere2moe::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale); ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared, false); ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); @@ -89,7 +89,7 @@ void llama_model_cohere2moe::load_arch_tensors(llama_model_loader & ml) { layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd }, flags); layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), { n_embd, n_ff }, flags); } else { - const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff; + const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff; layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert }, flags); layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff_exp, n_embd, n_expert }, flags); @@ -113,7 +113,7 @@ void llama_model_cohere2moe::load_arch_tensors(llama_model_loader & ml) { create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_gqa, n_embd_gqa, flags); layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, flags); - const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff; + const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff; // Routed experts layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert }, flags); diff --git a/src/models/deepseek.cpp b/src/models/deepseek.cpp index f52ec9518..a47a9c3da 100644 --- a/src/models/deepseek.cpp +++ b/src/models/deepseek.cpp @@ -3,11 +3,11 @@ void llama_model_deepseek::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); - switch (hparams.n_ff_exp) { + switch (hparams.n_ff_exp()) { case 1408: type = LLM_TYPE_16B; break; case 1792: type = LLM_TYPE_20B; break; default: type = LLM_TYPE_UNKNOWN; @@ -19,7 +19,7 @@ void llama_model_deepseek::load_arch_tensors(llama_model_loader &) { const int64_t n_expert_shared = hparams.n_expert_shared; - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); diff --git a/src/models/deepseek2.cpp b/src/models/deepseek2.cpp index 3a76187aa..4628ff4da 100644 --- a/src/models/deepseek2.cpp +++ b/src/models/deepseek2.cpp @@ -15,7 +15,7 @@ void llama_model_deepseek2::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv); ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA, hparams.n_embd_head_k_mla_impl, false); ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl, false); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); @@ -79,7 +79,7 @@ void llama_model_deepseek2::load_arch_tensors(llama_model_loader & ml) { const int64_t q_lora_rank = hparams.n_lora_q; const int64_t kv_lora_rank = hparams.n_lora_kv; - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); diff --git a/src/models/deepseek2ocr.cpp b/src/models/deepseek2ocr.cpp index 65d31c31b..1c5c452e9 100644 --- a/src/models/deepseek2ocr.cpp +++ b/src/models/deepseek2ocr.cpp @@ -4,7 +4,7 @@ void llama_model_deepseek2ocr::load_arch_hparams(llama_model_loader & ml) { // similar to deepseek2, but without MLA ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); @@ -25,7 +25,7 @@ void llama_model_deepseek2ocr::load_arch_tensors(llama_model_loader &) { const int64_t n_expert_shared = hparams.n_expert_shared; // similar to deepseek2, but without MLA - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); diff --git a/src/models/deepseek32.cpp b/src/models/deepseek32.cpp index 079bdfc30..60cc17c49 100644 --- a/src/models/deepseek32.cpp +++ b/src/models/deepseek32.cpp @@ -4,7 +4,7 @@ #include "llama-kv-cache-dsa.h" void llama_model_deepseek32::load_arch_hparams(llama_model_loader & ml) { - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); hparams.f_norm_eps = 1e-6; // eps for layer norm ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, false); @@ -20,7 +20,7 @@ void llama_model_deepseek32::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv); ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA, hparams.n_embd_head_k_mla_impl, false); ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl, false); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); // DSA parameters @@ -71,7 +71,7 @@ void llama_model_deepseek32::load_arch_tensors(llama_model_loader & ml) { const int64_t q_lora_rank = hparams.n_lora_q; const int64_t kv_lora_rank = hparams.n_lora_kv; - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); const int64_t n_expert_shared = hparams.n_expert_shared; tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp index 680516b23..5bdf14b48 100644 --- a/src/models/deepseek4.cpp +++ b/src/models/deepseek4.cpp @@ -29,7 +29,7 @@ void llama_model_deepseek4::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q); ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale); ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm); @@ -83,7 +83,7 @@ void llama_model_deepseek4::load_arch_tensors(llama_model_loader & ml) { LLAMA_LOAD_LOCALS; const int64_t q_lora_rank = hparams.n_lora_q; - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); const int64_t n_expert_shared = hparams.n_expert_shared; const int64_t n_embd_head = hparams.n_embd_head_k(); @@ -1298,7 +1298,7 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p layer.ffn_gate_exps, layer.ffn_down_exps, exp_probs_b, - n_expert, hparams.n_expert_used, + n_expert, hparams.n_expert_used(), LLM_FFN_SILU, hparams.expert_weights_norm, hparams.expert_weights_scale, (llama_expert_gating_func_type) hparams.expert_gating_func, @@ -1455,7 +1455,7 @@ llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llm layer.ffn_gate_exps, layer.ffn_down_exps, layer.ffn_exp_probs_b, - n_expert, hparams.n_expert_used, + n_expert, hparams.n_expert_used(), LLM_FFN_SILU, hparams.expert_weights_norm, hparams.expert_weights_scale, (llama_expert_gating_func_type) hparams.expert_gating_func, diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp index 036bcc14a..da84f30b6 100644 --- a/src/models/dflash.cpp +++ b/src/models/dflash.cpp @@ -40,7 +40,7 @@ void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) { if (hparams.dsv4_hc_mult > 0) { ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q); ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale); ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm); @@ -159,7 +159,7 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) { if (hparams.dsv4_hc_mult > 0) { const int64_t q_lora_rank = hparams.n_lora_q; - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); const int64_t n_expert_shared = hparams.n_expert_shared; const int64_t n_embd_head = hparams.n_embd_head_k(); const int64_t o_groups = hparams.dsv4_o_group_count; @@ -948,7 +948,7 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_ layer.ffn_gate_exps, layer.ffn_down_exps, layer.ffn_exp_probs_b, - n_expert, hparams.n_expert_used, + n_expert, hparams.n_expert_used(), LLM_FFN_SILU, hparams.expert_weights_norm, hparams.expert_weights_scale, (llama_expert_gating_func_type) hparams.expert_gating_func, diff --git a/src/models/dots1.cpp b/src/models/dots1.cpp index 07d6ab1b7..a3a85748e 100644 --- a/src/models/dots1.cpp +++ b/src/models/dots1.cpp @@ -3,7 +3,7 @@ void llama_model_dots1::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); @@ -19,7 +19,7 @@ void llama_model_dots1::load_arch_tensors(llama_model_loader &) { LLAMA_LOAD_LOCALS; const int64_t n_expert_shared = hparams.n_expert_shared; - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); diff --git a/src/models/dots3note.cpp b/src/models/dots3note.cpp index 7656562b0..0991c488e 100644 --- a/src/models/dots3note.cpp +++ b/src/models/dots3note.cpp @@ -11,7 +11,7 @@ void llama_model_dots3note::load_arch_hparams(llama_model_loader & ml) { // MoE parameters ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); @@ -56,7 +56,7 @@ void llama_model_dots3note::load_arch_tensors(llama_model_loader & ml) { const int64_t n_embd_head_qk_rope = hparams.n_rot(); const int64_t q_lora_rank = hparams.n_lora_q; - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); const int64_t n_expert_shared = hparams.n_expert_shared; tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); diff --git a/src/models/ernie4-5.cpp b/src/models/ernie4-5.cpp index 895cf690b..7bf7be648 100644 --- a/src/models/ernie4-5.cpp +++ b/src/models/ernie4-5.cpp @@ -6,7 +6,7 @@ void llama_model_ernie4_5::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); if (arch == LLM_ARCH_ERNIE4_5_MOE) { - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); ml.get_key(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, hparams.n_moe_layer_step); ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); @@ -47,7 +47,7 @@ void llama_model_ernie4_5::load_arch_tensors(llama_model_loader &) { layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); if (arch == LLM_ARCH_ERNIE4_5_MOE && static_cast(i) >= hparams.n_layer_dense_lead) { // MoE layers - int n_ff_exp = hparams.n_ff_exp; + int n_ff_exp = hparams.n_ff_exp(); layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED); diff --git a/src/models/exaone-moe.cpp b/src/models/exaone-moe.cpp index 86e5a3a98..976ee050a 100644 --- a/src/models/exaone-moe.cpp +++ b/src/models/exaone-moe.cpp @@ -13,7 +13,7 @@ void llama_model_exaone_moe::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared, false); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); @@ -30,7 +30,7 @@ void llama_model_exaone_moe::load_arch_hparams(llama_model_loader & ml) { void llama_model_exaone_moe::load_arch_tensors(llama_model_loader &) { LLAMA_LOAD_LOCALS; - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); const int64_t n_ff_shexp = hparams.n_ff_shexp > 0 ? hparams.n_ff_shexp : n_ff_exp; const int64_t head_dim = hparams.n_embd_head_k(); const int64_t n_qo_dim = n_head * head_dim; diff --git a/src/models/gemma4.cpp b/src/models/gemma4.cpp index aa518c6df..c6dd7d1bf 100644 --- a/src/models/gemma4.cpp +++ b/src/models/gemma4.cpp @@ -11,7 +11,7 @@ void llama_model_gemma4::load_arch_hparams(llama_model_loader & ml) { hparams.f_attention_scale = 1.0f; // Gemma4 uses self.scaling = 1.0 (no pre-attn scaling) ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false); ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, hparams.n_embd_per_layer); @@ -32,7 +32,7 @@ void llama_model_gemma4::load_arch_tensors(llama_model_loader &) { LLAMA_LOAD_LOCALS; const uint32_t n_embd_per_layer = hparams.n_embd_per_layer; - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); if (n_embd_head_k != n_embd_head_v) { throw std::runtime_error("Gemma 4 requires n_embd_head_k == n_embd_head_v"); diff --git a/src/models/glm-dsa.cpp b/src/models/glm-dsa.cpp index 543b15cf3..44d883274 100644 --- a/src/models/glm-dsa.cpp +++ b/src/models/glm-dsa.cpp @@ -27,7 +27,7 @@ const std::array GLM_5_2_DEFAULT_INDEXER_TYPES = { }; void llama_model_glm_dsa::load_arch_hparams(llama_model_loader & ml) { - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, false); @@ -42,7 +42,7 @@ void llama_model_glm_dsa::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv); ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA, hparams.n_embd_head_k_mla_impl, false); ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl, false); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); // DSA parameters @@ -104,7 +104,7 @@ void llama_model_glm_dsa::load_arch_tensors(llama_model_loader & ml) { const int64_t q_lora_rank = hparams.n_lora_q; const int64_t kv_lora_rank = hparams.n_lora_kv; - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); diff --git a/src/models/glm4-moe.cpp b/src/models/glm4-moe.cpp index 1d2ac65fd..d6ae5783c 100644 --- a/src/models/glm4-moe.cpp +++ b/src/models/glm4-moe.cpp @@ -1,7 +1,7 @@ #include "models.h" void llama_model_glm4_moe::load_arch_hparams(llama_model_loader & ml) { - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, false); @@ -40,7 +40,7 @@ void llama_model_glm4_moe::load_arch_tensors(llama_model_loader & ml) { } GGML_ASSERT(hparams.n_expert > 0 && "n_expert must be > 0 for GLM4_MOE MoE layers"); - GGML_ASSERT(hparams.n_expert_used > 0 && "n_expert_used must be > 0 for GLM4_MOE MoE layers"); + GGML_ASSERT(hparams.n_expert_used() > 0 && "n_expert_used must be > 0 for GLM4_MOE MoE layers"); tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0); @@ -82,7 +82,7 @@ void llama_model_glm4_moe::load_arch_tensors(llama_model_loader & ml) { layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), { n_expert }, flags); // MoE branch - const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used; + const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used; layer.ffn_gate_exps = create_tensor( tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert }, flags); diff --git a/src/models/granite-swa.cpp b/src/models/granite-swa.cpp index 3aa2b63b2..08d9e8a54 100644 --- a/src/models/granite-swa.cpp +++ b/src/models/granite-swa.cpp @@ -11,7 +11,7 @@ void llama_model_granite_swa::load_arch_hparams(llama_model_loader & ml) { // MoE expert configuration ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false); - ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false); + ml.get_key_or_arr(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used_arr, hparams.n_layer_all, false); // iSWA configuration ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); diff --git a/src/models/grok.cpp b/src/models/grok.cpp index 42f38af67..cb6afc3a7 100644 --- a/src/models/grok.cpp +++ b/src/models/grok.cpp @@ -12,7 +12,7 @@ void llama_model_grok::load_arch_hparams(llama_model_loader & ml) { hparams.f_final_logit_softcapping = 0.0f; ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false); ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale, false); ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale, false); ml.get_key(LLM_KV_ATTENTION_OUTPUT_SCALE, hparams.f_attn_out_scale, false); @@ -50,7 +50,7 @@ void llama_model_grok::load_arch_tensors(llama_model_loader &) { output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); } - const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff/* / n_expert_used*/; // grok-1 n_ff_exp == n_ff + const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff/* / n_expert_used*/; // grok-1 n_ff_exp == n_ff for (int i = 0; i < n_layer; ++i) { auto & layer = layers[i]; diff --git a/src/models/grovemoe.cpp b/src/models/grovemoe.cpp index 643a448e5..f32f3e9ed 100644 --- a/src/models/grovemoe.cpp +++ b/src/models/grovemoe.cpp @@ -1,7 +1,7 @@ #include "models.h" void llama_model_grovemoe::load_arch_hparams(llama_model_loader & ml) { - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_CHUNK_FEED_FORWARD_LENGTH, hparams.n_ff_chexp, false); ml.get_key(LLM_KV_EXPERT_GROUP_SCALE, hparams.expert_group_scale); ml.get_key(LLM_KV_EXPERTS_PER_GROUP, hparams.n_group_experts); @@ -46,7 +46,7 @@ void llama_model_grovemoe::load_arch_tensors(llama_model_loader &) { layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); // MoE branch - const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used; + const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used; const int64_t n_ff_chexp = hparams.n_ff_chexp ? hparams.n_ff_chexp : n_embd_head_k; const int64_t n_chunk_expert = n_expert / hparams.n_group_experts; diff --git a/src/models/hunyuan-moe.cpp b/src/models/hunyuan-moe.cpp index 4d55f5e7f..cedc3b53e 100644 --- a/src/models/hunyuan-moe.cpp +++ b/src/models/hunyuan-moe.cpp @@ -2,7 +2,7 @@ void llama_model_hunyuan_moe::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); switch (hparams.n_layer()) { diff --git a/src/models/hy-v3.cpp b/src/models/hy-v3.cpp index 3c45331b1..f6b72d843 100644 --- a/src/models/hy-v3.cpp +++ b/src/models/hy-v3.cpp @@ -2,7 +2,7 @@ void llama_model_hy_v3::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); @@ -45,7 +45,7 @@ void llama_model_hy_v3::load_arch_tensors(llama_model_loader & ml) { auto load_block = [&](int i, int flags) { auto & layer = layers[i]; - const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / (n_expert_used > 0 ? n_expert_used : 1); + const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / (n_expert_used > 0 ? n_expert_used : 1); const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff_exp; layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, flags); diff --git a/src/models/kimi-k3.cpp b/src/models/kimi-k3.cpp index 7b46bccdb..b061093eb 100644 --- a/src/models/kimi-k3.cpp +++ b/src/models/kimi-k3.cpp @@ -30,7 +30,7 @@ void llama_model_kimi_k3::load_arch_hparams(llama_model_loader & ml) { hparams.is_recr_impl[i] = hparams.n_head_kv(i) == 0; } - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); @@ -139,7 +139,7 @@ void llama_model_kimi_k3::load_arch_tensors(llama_model_loader &) { layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); } else { - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0); @@ -584,7 +584,7 @@ ggml_tensor * llama_model_kimi_k3::graph::build_latent_moe( layer.ffn_down_exps, layer.ffn_exp_probs_b, hparams.n_expert, - hparams.n_expert_used, + hparams.n_expert_used(), LLM_FFN_SITU, hparams.expert_weights_norm, hparams.expert_weights_scale, (llama_expert_gating_func_type) hparams.expert_gating_func, diff --git a/src/models/kimi-linear.cpp b/src/models/kimi-linear.cpp index bda3cd9b0..601d1d9be 100644 --- a/src/models/kimi-linear.cpp +++ b/src/models/kimi-linear.cpp @@ -19,7 +19,7 @@ void llama_model_kimi_linear::load_arch_hparams(llama_model_loader & ml) { } // MoE parameters - Kimi uses moe_intermediate_size = 1024 - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); @@ -137,7 +137,7 @@ void llama_model_kimi_linear::load_arch_tensors(llama_model_loader &) { layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); // MoE intermediate size (different from dense FFN) - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); // Kimi uses n_layer_dense_lead to determine which layers use dense FFN vs MoE // first_k_dense_replace = 1 means layer 0 uses dense FFN, layers 1+ use MoE @@ -504,7 +504,7 @@ llama_model_kimi_linear::graph::graph(const llama_model & model, const llm_graph layer.ffn_down_exps, layer.ffn_exp_probs_b, hparams.n_expert, - hparams.n_expert_used, + hparams.n_expert_used(), LLM_FFN_SILU, true, hparams.expert_weights_scale, (llama_expert_gating_func_type) hparams.expert_gating_func, diff --git a/src/models/laguna.cpp b/src/models/laguna.cpp index 82c9a9538..556400bfc 100644 --- a/src/models/laguna.cpp +++ b/src/models/laguna.cpp @@ -9,7 +9,7 @@ void llama_model_laguna::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); @@ -24,7 +24,7 @@ void llama_model_laguna::load_arch_hparams(llama_model_loader & ml) { // Weightless fixtures (test-llama-archs) omit this key; derive a nonzero // size so the shared expert is still built. Real GGUFs always carry the // exact value (routed and shared FF lengths may differ). - hparams.n_ff_shexp = hparams.n_ff_exp * hparams.n_expert_shared; + hparams.n_ff_shexp = hparams.n_ff_exp() * hparams.n_expert_shared; } // Sliding-window attention is OPTIONAL. XS.2 is hybrid (full / SWA / SWA / @@ -76,7 +76,7 @@ void llama_model_laguna::load_arch_tensors(llama_model_loader & ml) { output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); } - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); const int64_t n_ff_shexp = hparams.n_ff_shexp; for (int i = 0; i < n_layer; ++i) { diff --git a/src/models/lfm2.cpp b/src/models/lfm2.cpp index 9a4295557..07b71ccd3 100644 --- a/src/models/lfm2.cpp +++ b/src/models/lfm2.cpp @@ -53,9 +53,9 @@ void llama_model_lfm2::load_arch_tensors(llama_model_loader &) { if (is_moe_layer) { GGML_ASSERT(n_expert && n_expert_used); layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); - layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, hparams.n_ff_exp, n_expert}, 0); - layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {hparams.n_ff_exp, n_embd, n_expert}, 0); - layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, hparams.n_ff_exp, n_expert}, 0); + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, hparams.n_ff_exp(), n_expert}, 0); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {hparams.n_ff_exp(), n_embd, n_expert}, 0); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, hparams.n_ff_exp(), n_expert}, 0); layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0); } else { // dense layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); diff --git a/src/models/lfm2moe.cpp b/src/models/lfm2moe.cpp index 490f5c223..f8d47f9b8 100644 --- a/src/models/lfm2moe.cpp +++ b/src/models/lfm2moe.cpp @@ -6,7 +6,7 @@ void llama_model_lfm2moe::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_SHORTCONV_L_CACHE, hparams.n_shortconv_l_cache); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func); for (uint32_t il = 0; il < hparams.n_layer(); ++il) { @@ -42,9 +42,9 @@ void llama_model_lfm2moe::load_arch_tensors(llama_model_loader &) { if (is_moe_layer) { GGML_ASSERT(n_expert && n_expert_used); layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); - layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, hparams.n_ff_exp, n_expert}, 0); - layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {hparams.n_ff_exp, n_embd, n_expert}, 0); - layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, hparams.n_ff_exp, n_expert}, 0); + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, hparams.n_ff_exp(), n_expert}, 0); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {hparams.n_ff_exp(), n_embd, n_expert}, 0); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, hparams.n_ff_exp(), n_expert}, 0); layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0); } else { // dense layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); diff --git a/src/models/llada-moe.cpp b/src/models/llada-moe.cpp index 2ae893864..0ee9ce1be 100644 --- a/src/models/llada-moe.cpp +++ b/src/models/llada-moe.cpp @@ -1,7 +1,7 @@ #include "models.h" void llama_model_llada_moe::load_arch_hparams(llama_model_loader & ml) { - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); // diffusion language model uses non-causal attention @@ -39,7 +39,7 @@ void llama_model_llada_moe::load_arch_tensors(llama_model_loader &) { layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); - const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used; + const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used; layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0); layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0); diff --git a/src/models/llama4.cpp b/src/models/llama4.cpp index 7194c72a5..8a812beff 100644 --- a/src/models/llama4.cpp +++ b/src/models/llama4.cpp @@ -2,7 +2,7 @@ void llama_model_llama4::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, hparams.n_moe_layer_step); const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); @@ -75,7 +75,7 @@ void llama_model_llama4::load_arch_tensors(llama_model_loader &) { layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0)); if (is_moe_layer) { - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0); diff --git a/src/models/mellum.cpp b/src/models/mellum.cpp index 28823018b..872a9c8f5 100644 --- a/src/models/mellum.cpp +++ b/src/models/mellum.cpp @@ -2,7 +2,7 @@ void llama_model_mellum::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); if (hparams.n_swa > 0) { @@ -61,7 +61,7 @@ void llama_model_mellum::load_arch_tensors(llama_model_loader &) { throw std::runtime_error("n_expert_used must be > 0 for Mellum"); } - const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used; + const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used; layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0); layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0); diff --git a/src/models/mimo2.cpp b/src/models/mimo2.cpp index 1dc554220..8772319f4 100644 --- a/src/models/mimo2.cpp +++ b/src/models/mimo2.cpp @@ -5,7 +5,7 @@ void llama_model_mimo2::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); @@ -62,7 +62,7 @@ void llama_model_mimo2::load_arch_tensors(llama_model_loader & ml) { layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED | flags); // MoE branch - int64_t n_ff_exp = hparams.n_ff_exp; + int64_t n_ff_exp = hparams.n_ff_exp(); layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, TENSOR_NOT_REQUIRED | flags); layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, TENSOR_NOT_REQUIRED | flags); layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, TENSOR_NOT_REQUIRED | flags); diff --git a/src/models/minimax-m2.cpp b/src/models/minimax-m2.cpp index 86a8ae2b1..c2e69bfaa 100644 --- a/src/models/minimax-m2.cpp +++ b/src/models/minimax-m2.cpp @@ -2,7 +2,7 @@ void llama_model_minimax_m2::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); switch (hparams.n_layer()) { diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cpp index 1ba699d01..80260a629 100644 --- a/src/models/minimax-m3.cpp +++ b/src/models/minimax-m3.cpp @@ -13,7 +13,7 @@ void llama_model_minimax_m3::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); @@ -36,7 +36,7 @@ void llama_model_minimax_m3::load_arch_hparams(llama_model_loader & ml) { void llama_model_minimax_m3::load_arch_tensors(llama_model_loader &) { LLAMA_LOAD_LOCALS; const int64_t n_expert_shared = hparams.n_expert_shared; - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); diff --git a/src/models/nemotron-h.cpp b/src/models/nemotron-h.cpp index 55640c996..d2c48f125 100644 --- a/src/models/nemotron-h.cpp +++ b/src/models/nemotron-h.cpp @@ -1,5 +1,7 @@ #include "models.h" +#include // std::max + void llama_model_nemotron_h::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv); ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner); @@ -16,7 +18,8 @@ void llama_model_nemotron_h::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); // MTP head final_layernorm - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); + // Puzzle models set a different expert FFN size per layer + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false); ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared, false); ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); @@ -26,7 +29,17 @@ void llama_model_nemotron_h::load_arch_hparams(llama_model_loader & ml) { switch (hparams.n_layer()) { case 52: type = LLM_TYPE_31B_A3_5B; break; // Nemotron-H_MOE 31B case 56: type = LLM_TYPE_9B; break; - case 88: type = LLM_TYPE_120B_A12B; break; + case 88: + { + // Nemotron 3 Super (uniform MoE) and Nemotron 3 Puzzle (per-layer + // heterogeneous MoE) both have 88 layers; the per-layer top-k array + // is the discriminator. + bool heterogeneous = false; + for (uint32_t i = 1; i < hparams.n_layer(); ++i) { + heterogeneous |= hparams.n_expert_used_arr[i] != hparams.n_expert_used_arr[0]; + } + type = heterogeneous ? LLM_TYPE_75B_A9B : LLM_TYPE_120B_A12B; + } break; default: type = LLM_TYPE_UNKNOWN; } } @@ -94,7 +107,10 @@ void llama_model_nemotron_h::load_arch_tensors(llama_model_loader & ml) { layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); } else { if (n_expert != 0) { - const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used; + // Use per-layer n_ff_exp; fall back to n_ff/n_expert_used if absent (existing GGUFs). + const int64_t n_ff_exp_i = hparams.n_ff_exp(i) + ? (int64_t)hparams.n_ff_exp(i) + : hparams.n_ff(i) / (int64_t)hparams.n_expert_used(i); const int64_t n_ff_shexp = hparams.n_ff_shexp; layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert}, trunk_flags); @@ -104,8 +120,8 @@ void llama_model_nemotron_h::load_arch_tensors(llama_model_loader & ml) { layer.ffn_latent_down = create_tensor(tn(LLM_TENSOR_FFN_LATENT_DOWN, "weight", i), {n_embd, moe_n_embd}, TENSOR_NOT_REQUIRED); layer.ffn_latent_up = create_tensor(tn(LLM_TENSOR_FFN_LATENT_UP, "weight", i), {moe_n_embd, n_embd}, TENSOR_NOT_REQUIRED); - layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, moe_n_embd, n_expert}, trunk_flags); - layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {moe_n_embd, n_ff_exp, n_expert}, trunk_flags); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp_i, moe_n_embd, n_expert}, trunk_flags); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {moe_n_embd, n_ff_exp_i, n_expert}, trunk_flags); // Shared expert branch layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, trunk_flags); @@ -129,7 +145,7 @@ void llama_model_nemotron_h::load_arch_tensors(llama_model_loader & ml) { const int64_t n_head_i = hparams.n_head(i); const int64_t n_embd_k_gqa_i = hparams.n_embd_k_gqa(i); const int64_t n_embd_v_gqa_i = hparams.n_embd_v_gqa(i); - const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used; + const int64_t n_ff_exp = hparams.n_ff_exp(i) ? (int64_t)hparams.n_ff_exp(i) : n_ff / (int64_t)hparams.n_expert_used(i); const int64_t n_ff_shexp = hparams.n_ff_shexp; // NextN input-fusion tensors @@ -280,7 +296,7 @@ ggml_tensor * llama_model_nemotron_h::graph::build_ffn_layer(ggml_tensor * cur, nullptr, // no gate model.layers[il].ffn_down_exps, model.layers[il].ffn_exp_probs_b, - n_expert, n_expert_used, + n_expert, (int64_t)hparams.n_expert_used(il), LLM_FFN_RELU_SQR, hparams.expert_weights_norm, hparams.expert_weights_scale, LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID, diff --git a/src/models/openai-moe.cpp b/src/models/openai-moe.cpp index c91bae1c3..c9f9b677d 100644 --- a/src/models/openai-moe.cpp +++ b/src/models/openai-moe.cpp @@ -2,7 +2,7 @@ void llama_model_openai_moe::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; @@ -24,7 +24,7 @@ void llama_model_openai_moe::load_arch_hparams(llama_model_loader & ml) { void llama_model_openai_moe::load_arch_tensors(llama_model_loader &) { LLAMA_LOAD_LOCALS; - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); diff --git a/src/models/qwen2moe.cpp b/src/models/qwen2moe.cpp index e831ed11a..8bcb1017b 100644 --- a/src/models/qwen2moe.cpp +++ b/src/models/qwen2moe.cpp @@ -1,7 +1,7 @@ #include "models.h" void llama_model_qwen2moe::load_arch_hparams(llama_model_loader & ml) { - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false); ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); @@ -42,7 +42,7 @@ void llama_model_qwen2moe::load_arch_tensors(llama_model_loader &) { } // MoE branch - const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used; + const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used; layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0); layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0); diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cpp index 9bf4ea432..ed4083f12 100644 --- a/src/models/qwen35moe.cpp +++ b/src/models/qwen35moe.cpp @@ -2,7 +2,7 @@ #include "llama-memory-recurrent.h" void llama_model_qwen35moe::load_arch_hparams(llama_model_loader & ml) { - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false); ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); @@ -54,7 +54,7 @@ void llama_model_qwen35moe::load_arch_tensors(llama_model_loader & ml) { auto load_block_trunk = [&](int il, int flags) { auto & layer = layers[il]; - const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used; + const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used; const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff; // Calculate dimensions from hyperparameters @@ -106,7 +106,7 @@ void llama_model_qwen35moe::load_arch_tensors(llama_model_loader & ml) { auto load_block_mtp = [&](int il) { auto & layer = layers[il]; - const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used; + const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used; const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff; // MTP block looks like a full-attention Qwen3.5 decoder block with MoE FFN. diff --git a/src/models/qwen3moe.cpp b/src/models/qwen3moe.cpp index 6f6df5390..a6a3381e5 100644 --- a/src/models/qwen3moe.cpp +++ b/src/models/qwen3moe.cpp @@ -1,7 +1,7 @@ #include "models.h" void llama_model_qwen3moe::load_arch_hparams(llama_model_loader & ml) { - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); switch (hparams.n_layer()) { @@ -47,7 +47,7 @@ void llama_model_qwen3moe::load_arch_tensors(llama_model_loader &) { } // MoE branch - const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used; + const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used; layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0); layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0); diff --git a/src/models/qwen3next.cpp b/src/models/qwen3next.cpp index b2b8809c7..eb823b8ea 100644 --- a/src/models/qwen3next.cpp +++ b/src/models/qwen3next.cpp @@ -2,7 +2,7 @@ #include "llama-memory-recurrent.h" void llama_model_qwen3next::load_arch_hparams(llama_model_loader & ml) { - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false); ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); @@ -50,7 +50,7 @@ void llama_model_qwen3next::load_arch_tensors(llama_model_loader & ml) { output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, TENSOR_DUPLICATED); } - const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used; + const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used; // Calculate dimensions from hyperparameters const int64_t head_k_dim = hparams.ssm_d_state; diff --git a/src/models/qwen3vlmoe.cpp b/src/models/qwen3vlmoe.cpp index 7c41592f7..e7a81e32c 100644 --- a/src/models/qwen3vlmoe.cpp +++ b/src/models/qwen3vlmoe.cpp @@ -3,7 +3,7 @@ void llama_model_qwen3vlmoe::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_NUM_DEEPSTACK_LAYERS, hparams.n_deepstack_layers, false); ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, true); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); switch (hparams.n_layer()) { @@ -49,7 +49,7 @@ void llama_model_qwen3vlmoe::load_arch_tensors(llama_model_loader &) { } // MoE branch - const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used; + const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used; layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0); layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0); diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index 8f0e47b1f..773204a5a 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -24,7 +24,7 @@ static void qwen4exp_require_arr_len(llama_model_loader & ml, llm_kv kid, uint32 } void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) { - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false); ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); @@ -191,7 +191,7 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) { for (int il = 0; il < n_layer; ++il) { auto & layer = layers[il]; - const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used; + const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used; const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff; const int64_t head_k_dim = hparams.ssm_d_state; diff --git a/src/models/rnd1.cpp b/src/models/rnd1.cpp index fc276ce59..553a75730 100644 --- a/src/models/rnd1.cpp +++ b/src/models/rnd1.cpp @@ -1,7 +1,7 @@ #include "models.h" void llama_model_rnd1::load_arch_hparams(llama_model_loader & ml) { - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); switch (hparams.n_layer()) { @@ -49,7 +49,7 @@ void llama_model_rnd1::load_arch_tensors(llama_model_loader &) { } // MoE branch - const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used; + const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used; layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0); layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0); diff --git a/src/models/smallthinker.cpp b/src/models/smallthinker.cpp index a8e3d957f..680ffb8fd 100644 --- a/src/models/smallthinker.cpp +++ b/src/models/smallthinker.cpp @@ -18,7 +18,7 @@ void llama_model_smallthinker::load_arch_hparams(llama_model_loader & ml) { hparams.n_no_rope_layer_step = hparams.n_layer(); } - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); @@ -57,7 +57,7 @@ void llama_model_smallthinker::load_arch_tensors(llama_model_loader &) { GGML_ASSERT(n_expert_used > 0 && "n_expert_used must be > 0 for SMALLTHINKER"); // MoE branch - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert }, 0); layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert }, 0); layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff_exp, n_embd, n_expert }, 0); diff --git a/src/models/step35.cpp b/src/models/step35.cpp index d101d115e..53f3179c6 100644 --- a/src/models/step35.cpp +++ b/src/models/step35.cpp @@ -9,7 +9,7 @@ void llama_model_step35::load_arch_hparams(llama_model_loader & ml) { hparams.n_rot_full = hparams.n_rot_full / 2; // MoE + SWA parameters - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); @@ -99,7 +99,7 @@ void llama_model_step35::load_arch_tensors(llama_model_loader & ml) { layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED); // MoE routed experts + selection bias (router_bias) - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, TENSOR_NOT_REQUIRED); layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, TENSOR_NOT_REQUIRED); layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, TENSOR_NOT_REQUIRED); @@ -150,7 +150,7 @@ void llama_model_step35::load_arch_tensors(llama_model_loader & ml) { layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED); // MoE routed experts + selection bias (router_bias) - const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_exp = hparams.n_ff_exp(); layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, TENSOR_NOT_REQUIRED); layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, TENSOR_NOT_REQUIRED); layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, TENSOR_NOT_REQUIRED); From 4aa6ffba259ae69f0eb5f6f2c917a20986c15cb7 Mon Sep 17 00:00:00 2001 From: Eurekatic Date: Thu, 3 Sep 2026 08:59:06 +0200 Subject: [PATCH 15/17] sycl: reduce redundant work in Q4_K multi-column MMVQ (#27062) * sycl: Q4_K Weight unpack optimization and reuse between destination Columns * sycl: Q4_K small N (N=2..4) + two output rows by subgroup reuse of activation between two rows. * sycl: gate Q4_K two-row reuse for small N=2 * sycl: Fix on magic number now uses Q4_K_MMVQ_ROW_PAIR_MIN_NROWS=6272 for it, added tests for coverage around Q4_K_MMVQ_ROW_PAIR_MIN_NROWS with perf support to test Q4_K MUL_MAT, applied the same reuse pattern to the activation as the weights. Assisted-by: GPT-5.6 Sol --------- Co-authored-by: RaulAbejonDelgado --- ggml/src/ggml-sycl/mmvq.cpp | 197 ++++++++++++++++++++++++++------- ggml/src/ggml-sycl/vecdotq.hpp | 135 +++++++++++++++------- tests/test-backend-ops.cpp | 41 +++++++ 3 files changed, 293 insertions(+), 80 deletions(-) diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index 220663d5a..933bc77d2 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -6,6 +6,24 @@ #include "quants.hpp" #include "vecdotq.hpp" +// Minimum weight-row count at which the Q4_K multi-column MMVQ kernel handles two output rows per +// subgroup (rows_per_sg == 2) instead of one, when ncols_dst == 2. +// +// Pairing rows lets a subgroup load each activation block once and apply it to two rows, at the cost +// of halving the number of subgroups in the launch. With only two destination columns there is too +// little work per row to hide that loss of parallelism, so pairing only pays off once there are +// enough rows to keep the device occupied. This is a measured performance crossover, not a +// correctness or hardware limit - both variants compute the same result for any nrows. +// +// Derived on Intel Arc Pro B70 with `test-backend-ops perf -o MUL_MAT` (Q4_K, ncols_dst == 2), +// sweeping nrows over 5120..6912 at ncols 17408 and 19968: one row per subgroup was up to 9% faster +// below the crossover, two rows per subgroup 8-15% faster above it, and the crossover fell inside +// (6144, 6272] for both ncols with no measurable ncols dependence. A later 32-row granularity sweep +// narrowed it to (6144, 6176], so 6272 is a conservative gate rather than the exact crossover. +// ncols_dst >= 3 amortizes the activation loads over more columns and is faster with two rows at +// every row count, so it does not consult this threshold. +static constexpr int Q4_K_MMVQ_ROW_PAIR_MIN_NROWS = 6272; + template static void mul_mat_vec_q_reorder(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols, const int nrows, const sycl::nd_item<3> & nd_item) { @@ -59,7 +77,7 @@ static void mul_mat_vec_q_reorder(const void * __restrict__ vx, const void * __r // With has_fusion, `vgate` is a second weight matrix sharing vx's shape, stride and reorder // layout: one pass computes both row dot products and the epilogue writes glu(gate, up). -template +template static void mul_mat_vec_q_reorder_ncols(const void * __restrict__ vx, const void * __restrict__ vgate, const void * __restrict__ vy, float * __restrict__ dst, const int ncols, const int nrows, const int stride_col_y_bytes, const int stride_col_dst, @@ -71,14 +89,17 @@ static void mul_mat_vec_q_reorder_ncols(const void * __restrict__ vx, const void const int sg_range = sg.get_group_linear_range(); const int workgroup_id = nd_item.get_group_linear_id(); const int sg_id = sg.get_group_linear_id(); - const int row = workgroup_id * sg_range + sg_id; + const int row0 = (workgroup_id * sg_range + sg_id) * rows_per_sg; // row is sub-group uniform, so this retires whole sub-groups and the collectives below // stay convergent - if (row >= nrows) { + if (row0 >= nrows) { return; } + static_assert(rows_per_sg == 1 || + reorder_vec_dot_shared_activations::value); + const int blocks_per_row = ncols / block_traits::qk; constexpr int blocks_per_subgroup = ceil_div(block_traits::vdr_mmvq * WARP_SIZE, block_traits::qi); constexpr int block_elements_per_subgroup = block_traits::qi / block_traits::vdr_mmvq; @@ -87,34 +108,96 @@ static void mul_mat_vec_q_reorder_ncols(const void * __restrict__ vx, const void static_assert(blocks_per_subgroup > 0); static_assert(block_elements_per_subgroup > 0); - float partial_sum[ncols_dst] = { 0.0f }; + float partial_sum[ncols_dst][rows_per_sg] = {}; // sized 1 rather than 0 when unused: zero-length arrays are not standard C++, and the // array is dead and eliminated in that case - [[maybe_unused]] float partial_gate[has_fusion ? ncols_dst : 1] = { 0.0f }; + [[maybe_unused]] float partial_gate[has_fusion ? ncols_dst : 1][has_fusion ? rows_per_sg : 1] = {}; for (int i = sg.get_local_linear_id() / block_elements_per_subgroup; i < blocks_per_row; i += blocks_per_subgroup) { - const int ibx = row * blocks_per_row + i; - - // the offsets depend only on the block index and the matrix shape, never on the base - // pointer, which is what lets vgate reuse them - const auto bx_offset = block_type::get_block_offset(ibx, nblocks); - const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx); const int iby = i * block_type::block_to_q8_1_ratio(); #pragma unroll for (int elem = 0; elem < block_elements_per_subgroup; elem += WARP_SIZE) { const int iqs = elem + block_traits::vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup); + if constexpr (rows_per_sg > 1) { + typename reorder_vec_dot_q_sycl::weights wx[rows_per_sg]; + [[maybe_unused]] typename reorder_vec_dot_q_sycl::weights wg[rows_per_sg]; #pragma unroll - for (int j = 0; j < ncols_dst; ++j) { - const char * vy_j = (const char *) vy + j * stride_col_y_bytes; - const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1; - const sycl::half2 * q8_1_ds_ptr = (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2)); - - partial_sum[j] += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs); - + for (int r = 0; r < rows_per_sg; ++r) { + const int row = sycl::min(row0 + r, nrows - 1); + const int ibx = row * blocks_per_row + i; + const auto bx_offset = block_type::get_block_offset(ibx, nblocks); + const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx); + wx[r] = reorder_vec_dot_q_sycl::load(vx, bx_offset, d_offset, iqs); + if constexpr (has_fusion) { + wg[r] = reorder_vec_dot_q_sycl::load(vgate, bx_offset, d_offset, iqs); + } + } +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + const char * vy_j = (const char *) vy + j * stride_col_y_bytes; + const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1; + const sycl::half2 * q8_1_ds_ptr = + (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2)); + const auto a = reorder_vec_dot_q_sycl::load_activations(q8_1_quant_ptr, q8_1_ds_ptr, iqs); +#pragma unroll + for (int r = 0; r < rows_per_sg; ++r) { + partial_sum[j][r] += reorder_vec_dot_q_sycl::apply(wx[r], a); + if constexpr (has_fusion) { + partial_gate[j][r] += reorder_vec_dot_q_sycl::apply(wg[r], a); + } + } + } + } else if constexpr (reorder_vec_dot_shared_weights::value) { + const int ibx = row0 * blocks_per_row + i; + const auto bx_offset = block_type::get_block_offset(ibx, nblocks); + const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx); + const auto wx = reorder_vec_dot_q_sycl::load(vx, bx_offset, d_offset, iqs); if constexpr (has_fusion) { - partial_gate[j] += - reorder_vec_dot_q_sycl()(vgate, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs); + const auto wg = reorder_vec_dot_q_sycl::load(vgate, bx_offset, d_offset, iqs); + +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + const char * vy_j = (const char *) vy + j * stride_col_y_bytes; + const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1; + const sycl::half2 * q8_1_ds_ptr = + (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2)); + + // up and gate share the activation, so load it once and apply it twice + const auto a = reorder_vec_dot_q_sycl::load_activations(q8_1_quant_ptr, q8_1_ds_ptr, iqs); + + partial_sum[j][0] += reorder_vec_dot_q_sycl::apply(wx, a); + partial_gate[j][0] += reorder_vec_dot_q_sycl::apply(wg, a); + } + } else { +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + const char * vy_j = (const char *) vy + j * stride_col_y_bytes; + const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1; + const sycl::half2 * q8_1_ds_ptr = + (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2)); + + partial_sum[j][0] += reorder_vec_dot_q_sycl::dot(wx, q8_1_quant_ptr, q8_1_ds_ptr, iqs); + } + } + } else { + const int ibx = row0 * blocks_per_row + i; + const auto bx_offset = block_type::get_block_offset(ibx, nblocks); + const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx); +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + const char * vy_j = (const char *) vy + j * stride_col_y_bytes; + const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1; + const sycl::half2 * q8_1_ds_ptr = + (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2)); + + partial_sum[j][0] += + reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs); + + if constexpr (has_fusion) { + partial_gate[j][0] += + reorder_vec_dot_q_sycl()(vgate, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs); + } } } } @@ -122,17 +205,20 @@ static void mul_mat_vec_q_reorder_ncols(const void * __restrict__ vx, const void #pragma unroll for (int j = 0; j < ncols_dst; ++j) { - float sum = sycl::reduce_over_group(nd_item.get_sub_group(), partial_sum[j], std::plus<>()); +#pragma unroll + for (int r = 0; r < rows_per_sg; ++r) { + float sum = sycl::reduce_over_group(nd_item.get_sub_group(), partial_sum[j][r], std::plus<>()); - if constexpr (has_fusion) { - const float gate = sycl::reduce_over_group(nd_item.get_sub_group(), partial_gate[j], std::plus<>()); + if constexpr (has_fusion) { + const float gate = sycl::reduce_over_group(nd_item.get_sub_group(), partial_gate[j][r], std::plus<>()); - // uniform across the launch; the launcher only instantiates SWIGLU and GEGLU - sum *= glu_op == GGML_GLU_OP_SWIGLU ? op_silu(gate) : op_gelu(gate); - } + // uniform across the launch; the launcher only instantiates SWIGLU and GEGLU + sum *= glu_op == GGML_GLU_OP_SWIGLU ? op_silu(gate) : op_gelu(gate); + } - if (sg.leader()) { - dst[j * stride_col_dst + row] = sum; + if (sg.leader() && row0 + r < nrows) { + dst[j * stride_col_dst + row0 + r] = sum; + } } } } @@ -1671,8 +1757,8 @@ static void reorder_mul_mat_vec_q4_k_q8_1_sycl(const void * vx, const void * vy, }); } -template -static void reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols( +template +static void reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int stride_col_y_bytes, const int stride_col_dst, @@ -1680,20 +1766,31 @@ static void reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols( GGML_ASSERT(ncols % QK_K == 0); constexpr size_t num_subgroups = WARP_SIZE; - const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups * rows_per_sg); const sycl::range<3> block_nums(1, 1, block_num_y); const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); stream->submit([&](sycl::handler & cgh) { cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_reorder_ncols, ncols_dst>( + mul_mat_vec_q_reorder_ncols, ncols_dst, + /*has_fusion=*/ false, rows_per_sg>( vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item); }); }); } +template +static void reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + constexpr int rows_per_sg = ncols_dst >= 3 && ncols_dst <= 4 ? 2 : 1; + reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); +} + static void reorder_mul_mat_vec_q4_k_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, @@ -1701,7 +1798,13 @@ static void reorder_mul_mat_vec_q4_k_q8_1_sycl_switch_ncols( dpct::queue_ptr stream) { switch (ncols_dst) { case 1: reorder_mul_mat_vec_q4_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 2: + if (nrows >= Q4_K_MMVQ_ROW_PAIR_MIN_NROWS) { + reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl<2, 2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); + } else { + reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl<2, 1>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); + } + break; case 3: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; case 4: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; case 5: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; @@ -2839,8 +2942,8 @@ bool ggml_sycl_mul_mat_vec_q_id_reorder( } } -template -static void launch_mul_mat_vec_q_reorder_glu(const void * vx, const void * vgate, const void * vy, float * dst, +template +static void launch_mul_mat_vec_q_reorder_glu_impl(const void * vx, const void * vgate, const void * vy, float * dst, const int ncols, const int nrows, const int stride_col_y_bytes, const int stride_col_dst, const ggml_glu_op glu_op, dpct::queue_ptr stream) { @@ -2848,20 +2951,33 @@ static void launch_mul_mat_vec_q_reorder_glu(const void * vx, const void * vgate constexpr size_t num_subgroups = WARP_SIZE; - const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups * rows_per_sg); const sycl::range<3> block_nums(1, 1, block_num_y); const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); stream->submit([&](sycl::handler & cgh) { cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_reorder_ncols( + mul_mat_vec_q_reorder_ncols( vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, nd_item); }); }); } +template +static void launch_mul_mat_vec_q_reorder_glu(const void * vx, const void * vgate, const void * vy, float * dst, + const int ncols, const int nrows, const int stride_col_y_bytes, + const int stride_col_dst, const ggml_glu_op glu_op, + dpct::queue_ptr stream) { + constexpr int rows_per_sg = + reorder_vec_dot_shared_activations::value && ncols_dst >= 3 && ncols_dst <= 4 + ? 2 + : 1; + launch_mul_mat_vec_q_reorder_glu_impl(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream); +} + bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glu_op glu_op, const void * vx, const void * vgate, const void * vy, float * dst, int ncols, int nrows, int ncols_dst, int stride_col_y_bytes, int stride_col_dst, @@ -2881,8 +2997,11 @@ bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glu stride_col_dst, glu_op, stream); return true; case 2: - launch_mul_mat_vec_q_reorder_glu(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, - stride_col_dst, glu_op, stream); + if (nrows >= Q4_K_MMVQ_ROW_PAIR_MIN_NROWS) { + launch_mul_mat_vec_q_reorder_glu_impl(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream); + } else { + launch_mul_mat_vec_q_reorder_glu_impl(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream); + } return true; case 3: launch_mul_mat_vec_q_reorder_glu(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, diff --git a/ggml/src/ggml-sycl/vecdotq.hpp b/ggml/src/ggml-sycl/vecdotq.hpp index 3ad4cee93..ed5fd7de8 100644 --- a/ggml/src/ggml-sycl/vecdotq.hpp +++ b/ggml/src/ggml-sycl/vecdotq.hpp @@ -351,6 +351,25 @@ template struct reorder_vec_dot_q_sycl { static_assert(T != T, "ggml_type for reorder vecdot not implemented"); }; +// For some types the weight side of the dot product does not depend on the destination column, so a +// multi-column mul_mat_vec can unpack it once per block instead of once per column. Such a type adds +// load() and dot() next to operator() and opts in here. See reorder_vec_dot_q_sycl. +template struct reorder_vec_dot_shared_weights { + static constexpr bool value = false; +}; + +template <> struct reorder_vec_dot_shared_weights { + static constexpr bool value = true; +}; + +template struct reorder_vec_dot_shared_activations { + static constexpr bool value = false; +}; + +template <> struct reorder_vec_dot_shared_activations { + static constexpr bool value = true; +}; + template <> struct reorder_vec_dot_q_sycl { static constexpr ggml_type gtype = GGML_TYPE_Q4_0; @@ -540,50 +559,84 @@ template <> struct reorder_vec_dot_q_sycl { using q4_k_block = ggml_sycl_reordered::block_q_t; using q4_k_traits = typename q4_k_block::traits; + struct weights { + int v[2]; + uint16_t aux[2]; + ggml_half2 dm; + int bq8_offset; + }; + + struct activations { + int u[2 * QR4_K]; + float d8[QR4_K]; + }; + + __dpct_inline__ static weights load(const void * __restrict__ vbq, const std::pair ibx_offset, + const std::pair d_offset, const int & iqs) { + const uint8_t * base = static_cast(vbq); + const uint8_t * qs = base + ibx_offset.first; + const uint8_t * scs = base + d_offset.first; + const ggml_half2 * dms = reinterpret_cast(base + d_offset.second); + + weights w; + w.bq8_offset = QR4_K * ((iqs / 2) / (QI8_1 / 2)); + + const int * q4 = (const int *) (qs + 16 * w.bq8_offset + 4 * ((iqs / 2) % 4)); + const uint16_t * scales = (const uint16_t *) scs; + + w.v[0] = q4[0]; + w.v[1] = q4[4]; + + const int j = (QR4_K * ((iqs / 2) / (QI8_1 / 2))) / 2; + if (j < 2) { + w.aux[0] = scales[j + 0] & 0x3f3f; + w.aux[1] = scales[j + 2] & 0x3f3f; + } else { + w.aux[0] = ((scales[j + 2] >> 0) & 0x0f0f) | ((scales[j - 2] & 0xc0c0) >> 2); + w.aux[1] = ((scales[j + 2] >> 4) & 0x0f0f) | ((scales[j - 0] & 0xc0c0) >> 2); + } + + w.dm = *dms; + + return w; + } + + __dpct_inline__ static activations load_activations(const int8_t * q8_1_quant_ptr, + const sycl::half2 * q8_1_ds, const int & iqs) { + activations a; + const int bq8_offset = QR4_K * ((iqs / 2) / (QI8_1 / 2)); + for (int i = 0; i < QR4_K; ++i) { + const int8_t * quant_base_ptr = q8_1_quant_ptr + (bq8_offset + i) * QK8_1; + sycl::half2 ds_values = *(q8_1_ds + bq8_offset + i); + + a.d8[i] = ds_values[0]; + + const int * q8 = (const int *) quant_base_ptr + ((iqs / 2) % 4); + a.u[2 * i + 0] = q8[0]; + a.u[2 * i + 1] = q8[4]; + } + + return a; + } + + __dpct_inline__ static float apply(const weights & w, const activations & a) { + const uint8_t * sc = (const uint8_t *) w.aux; + const uint8_t * m = sc + 2; + + return vec_dot_q4_K_q8_1_impl_vmmq(w.v, a.u, sc, m, w.dm, a.d8); + } + + __dpct_inline__ static float dot(const weights & w, const int8_t * q8_1_quant_ptr, + const sycl::half2 * q8_1_ds, const int & iqs) { + const auto a = load_activations(q8_1_quant_ptr, q8_1_ds, iqs); + + return apply(w, a); + } + __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair ibx_offset, const std::pair d_offset, const int8_t * q8_1_quant_ptr, const sycl::half2 * q8_1_ds, const int & iqs) { - const uint8_t * base = static_cast(vbq); - const uint8_t * qs = base + ibx_offset.first; - const uint8_t * scs = base + d_offset.first; - const ggml_half2 * dms = reinterpret_cast(base + d_offset.second); - - const int bq8_offset = QR4_K * ((iqs / 2) / (QI8_1 / 2)); - const int * q4 = (const int *) (qs + 16 * bq8_offset + 4 * ((iqs / 2) % 4)); - const uint16_t * scales = (const uint16_t *) scs; - - int v[2]; - int u[2 * QR4_K]; - float d8[QR4_K]; - - v[0] = q4[0]; - v[1] = q4[4]; - - uint16_t aux[2]; - const int j = (QR4_K * ((iqs / 2) / (QI8_1 / 2))) / 2; - if (j < 2) { - aux[0] = scales[j + 0] & 0x3f3f; - aux[1] = scales[j + 2] & 0x3f3f; - } else { - aux[0] = ((scales[j + 2] >> 0) & 0x0f0f) | ((scales[j - 2] & 0xc0c0) >> 2); - aux[1] = ((scales[j + 2] >> 4) & 0x0f0f) | ((scales[j - 0] & 0xc0c0) >> 2); - } - - const uint8_t * sc = (const uint8_t *) aux; - const uint8_t * m = sc + 2; - - for (int i = 0; i < QR4_K; ++i) { - const int8_t* quant_base_ptr = q8_1_quant_ptr + (bq8_offset + i) * QK8_1; - sycl::half2 ds_values = *(q8_1_ds + bq8_offset + i); - - d8[i] = ds_values[0]; - - const int * q8 = (const int *) quant_base_ptr + ((iqs / 2) % 4); - u[2 * i + 0] = q8[0]; - u[2 * i + 1] = q8[4]; - } - - return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, *dms, d8); + return dot(load(vbq, ibx_offset, d_offset, iqs), q8_1_quant_ptr, q8_1_ds, iqs); } }; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 1b0eaca8f..154146dda 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9435,6 +9435,21 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 16, 8, 16*256, { 1, 1}, {1, 1})); } + // Multi-column MMVQ coverage for the Q4_K weight-reuse path and a Q5_K control. + for (ggml_type type_a : { GGML_TYPE_Q4_K, GGML_TYPE_Q5_K }) { + for (int n = 1; n <= 8; ++n) { + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 4096, n, 1024, { 1, 1 }, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1023, n, 4096, { 1, 1 }, { 1, 1 })); + } + } + + // The SYCL backend picks between one and two output rows per subgroup by row count when there + // are two destination columns (Q4_K_MMVQ_ROW_PAIR_MIN_NROWS in ggml-sycl/mmvq.cpp). Cover both + // sides of that boundary, including an odd row count above it for the row-pair tail. + for (int64_t m : {6271, 6272, 6273}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, m, 2, 1024, { 1, 1 }, { 1, 1 })); + } + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_MXFP4, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); @@ -10364,6 +10379,22 @@ static std::vector> make_test_cases_eval() { true, 16, 8, b, false, true, false)); } + // Fused row-pair coverage: minimum rows, an even pair, and an odd tail. + for (ggml_glu_op glu_op : { GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU }) { + for (int64_t m_batch : { 2, 3, 4 }) { + for (int64_t rows : { 1, 2, 3 }) { + test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q4_K, glu_op, m_batch, rows, 256, + false, 16, 8, false, false, true, false, { 1, 1 })); + } + } + } + + // Both sides of the same row-count boundary as above, on the fused path. + for (int64_t rows : {6271, 6272, 6273}) { + test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 2, rows, 256, + false, 16, 8, false, false, true, false, { 1, 1 })); + } + for (auto gate : {GATING_FUNC_SOFTMAX, GATING_FUNC_SIGMOID, GATING_FUNC_SOFTMAX_WEIGHT, GATING_FUNC_SQRT_SOFTPLUS}) { for (bool with_norm : {false, true}) { for (bool bias_probs : {false, true}) { @@ -10651,6 +10682,16 @@ static std::vector> make_test_cases_perf() { } } + // Q4_K multi-column mat-vec, at ffn_up/ffn_gate geometry (k = n_embd, m = n_ff): n sweeps the + // per-column specializations used for short prompts and speculative/MTP verify, and m brackets + // the row count at which the SYCL backend switches to two output rows per subgroup + // (Q4_K_MMVQ_ROW_PAIR_MIN_NROWS in ggml-sycl/mmvq.cpp), so both sides of it can be measured. + for (int64_t m : {4096, 6144, 6272, 14336}) { + for (int bs : {1, 2, 3, 4, 8}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, m, bs, 4096, {1, 1}, {1, 1})); + } + } + // qwen3-30b-a3b for (int bs : {1, 4, 8, 32, 64, 128, 256, 512}) { for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS}) { From 5ec4eab69edbfaa4b01bcf1ee1123bb885b8d892 Mon Sep 17 00:00:00 2001 From: Tarek Dakhran Date: Thu, 3 Sep 2026 09:32:24 +0200 Subject: [PATCH 16/17] misc : prevent RAM peaking at model loading stage (#27483) --- src/llama-model-loader.cpp | 22 ++++++++++++++++++++-- src/llama-model.cpp | 8 ++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index d940b1b61..49f3c4f8e 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -1497,7 +1497,6 @@ bool llama_model_loader::load_all_data( } GGML_ASSERT(size_data != 0 && "call init_mappings() first"); - std::vector> read_buf; std::vector>> validation_result; // 4 staging buffers for async uploads, each sized 1MB seems to be a good default for single NVMe drives. @@ -1598,7 +1597,25 @@ bool llama_model_loader::load_all_data( ggml_backend_name(upload_backend)); } + std::vector tensors; for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur != NULL; cur = ggml_get_next_tensor(ctx, cur)) { + tensors.push_back(cur); + } + + // without mmap, tensors in non-host buffers are staged through a temporary buffer sized like the tensor + // load them biggest-first so the largest staging buffer is allocated while the fewest weights are resident + if (!use_mmap) { + std::stable_sort(tensors.begin(), tensors.end(), [](const ggml_tensor * a, const ggml_tensor * b) { + const bool staged_a = a->buffer && !ggml_backend_buffer_is_host(a->buffer); + const bool staged_b = b->buffer && !ggml_backend_buffer_is_host(b->buffer); + if (staged_a != staged_b) { + return staged_a; + } + return staged_a && ggml_nbytes(a) > ggml_nbytes(b); + }); + } + + for (struct ggml_tensor * cur : tensors) { const auto * weight = get_weight(ggml_get_name(cur)); if (weight == nullptr) { // this can happen with split experts models @@ -1711,7 +1728,8 @@ bool llama_model_loader::load_all_data( buffer_idx %= n_buffers; } } else { - read_buf.resize(n_size); + // scoped to one tensor so only one staging buffer is alive at a time + std::vector> read_buf(n_size); file->seek(weight->offs, SEEK_SET); file->read_raw(read_buf.data(), n_size); ggml_backend_tensor_set(cur, read_buf.data(), 0, n_size); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 408e76e32..6344f2d8a 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1816,6 +1816,14 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { return true; } + // without mmap, load non-host buffers first: their tensors go through a staging buffer, which is cheapest while the fewest weights are resident + if (!ml.use_mmap) { + std::stable_partition(ctx_buf_maps.begin(), ctx_buf_maps.end(), [](const auto & ctx_buf_map) { + const auto & buf_map = ctx_buf_map.second; + return !buf_map.empty() && !ggml_backend_buffer_is_host(buf_map.begin()->second); + }); + } + // load tensor data for (auto & [ctx, buf_map] : ctx_buf_maps) { if (!ml.load_all_data(ctx, buf_map, use_mlock ? &pimpl->mlock_mmaps : NULL, params.progress_callback, params.progress_callback_user_data)) { From d646c9d15500a702425e8a90c19d2400deecbd0c Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 3 Sep 2026 10:37:23 +0300 Subject: [PATCH 17/17] convert : skip bias_vl tensor in DeepSeek-V4 DSpark conversion (#28294) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * convert : skip bias_vl tensor in DeepSeek-V4 DSpark conversion The DFLASH arch does not include FFN_EXP_PROBS_B_VL, so the DSpark conversion failed when it tried to write the mtmd-only hash routing tensor ffn.gate.bias_vl. Drop it like the tid2eid tensor; the DFLASH draft only consumes ffn.gate.bias via FFN_EXP_PROBS_B. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * cont : fix Co-authored-by: Sigbjørn Skjæret --------- Co-authored-by: Sigbjørn Skjæret --- conversion/deepseek.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/conversion/deepseek.py b/conversion/deepseek.py index abaed7880..817eb7612 100644 --- a/conversion/deepseek.py +++ b/conversion/deepseek.py @@ -1007,6 +1007,13 @@ class DeepseekV4DSparkModel(DeepseekV4Model): return self._DSPARK_ROOT_MAP[name] return super()._map_dsv4_tensor_name(name, bid) + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # the DFlash draft uses the plain exp-probs bias (ffn.gate.bias -> FFN_EXP_PROBS_B); + # the mtmd-only hash routing tensors (bias_vl, tid2eid) are not part of the DFLASH arch + if name.endswith(".ffn.gate.bias_vl"): + return + yield from super().modify_tensors(data_torch, name, bid) + def set_vocab(self): if self.target_model_dir is None: raise ValueError("DeepSeek-V4 DSpark requires --target-model-dir with the target tokenizer")