diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py
index 2fbd020b9..54ff7cdac 100755
--- a/convert_hf_to_gguf.py
+++ b/convert_hf_to_gguf.py
@@ -895,8 +895,8 @@ class TextModel(ModelBase):
# ref: https://huggingface.co/JetBrains/Mellum-4b-base
res = "mellum"
if chkhsh == "9b1be57e70d20d9501b2b3186e792d81181ae36ada3903c26f9fea418cf87206":
- # ref: https://huggingface.co/inclusionAI/LLaDA-MoE-7B-A1B-Base
- res = "llada-moe"
+ # ref: https://huggingface.co/inclusionAI/Ling-mini-base-2.0
+ res = "bailingmoe2"
if chkhsh == "53e325976a6e142379c19b09afcae354f2f496f147afa8f9e189a33fe4e3024e":
# ref: https://huggingface.co/ibm-granite/granite-docling-258M
res = "granite-docling"
@@ -8060,6 +8060,103 @@ class BailingMoeModel(TextModel):
raise ValueError(f"Unprocessed experts: {experts}")
+@ModelBase.register("BailingMoeV2ForCausalLM")
+class BailingMoeV2Model(TextModel):
+ model_arch = gguf.MODEL_ARCH.BAILINGMOE2
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ if nextn_layers := self.hparams.get("num_nextn_predict_layers", 0):
+ self.block_count = self.hparams["num_hidden_layers"] + nextn_layers
+ self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
+
+ def set_vocab(self):
+ self._set_vocab_gpt2()
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ hparams = self.hparams
+ if (rope_dim := hparams.get("head_dim")) is None:
+ rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]
+
+ self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.hparams.get("partial_rotary_factor", 0.5)))
+ rope_scaling = self.hparams.get("rope_scaling") or {}
+ if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:
+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)
+ self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
+ self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])
+ else:
+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)
+ self.gguf_writer.add_leading_dense_block_count(hparams["first_k_dense_replace"])
+ self.gguf_writer.add_vocab_size(hparams["vocab_size"])
+ self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])
+ self.gguf_writer.add_expert_shared_feed_forward_length(hparams.get("moe_shared_expert_intermediate_size", hparams["moe_intermediate_size"] * hparams["num_shared_experts"]))
+ self.gguf_writer.add_expert_weights_scale(hparams["routed_scaling_factor"])
+ self.gguf_writer.add_expert_count(hparams["num_experts"])
+ self.gguf_writer.add_expert_shared_count(hparams["num_shared_experts"])
+ self.gguf_writer.add_expert_group_count(hparams["n_group"])
+ self.gguf_writer.add_expert_group_used_count(hparams["topk_group"])
+ self.gguf_writer.add_expert_weights_norm(hparams["norm_topk_prob"])
+
+ if hparams["score_function"] == "sigmoid":
+ self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
+ elif hparams["score_function"] == "softmax":
+ self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SOFTMAX)
+ else:
+ raise ValueError(f"Unsupported score_function value: {hparams['score_function']}")
+
+ if (nextn_layers := self.hparams.get("num_nextn_predict_layers")) is not None:
+ self.gguf_writer.add_nextn_predict_layers(nextn_layers)
+
+ _experts: list[dict[str, Tensor]] | None = None
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if "mlp.experts" in name:
+ n_experts = self.hparams["num_experts"]
+ assert bid is not None
+
+ tensors: list[tuple[str, Tensor]] = []
+
+ if self._experts is None:
+ self._experts = [{} for _ in range(self.block_count)]
+
+ self._experts[bid][name] = data_torch
+
+ if len(self._experts[bid]) >= n_experts * 3:
+ # merge the experts into a single 3d tensor
+ for w_name in ["down_proj", "gate_proj", "up_proj"]:
+ datas: list[Tensor] = []
+
+ for xid in range(n_experts):
+ ename = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight"
+ datas.append(self._experts[bid][ename])
+ del self._experts[bid][ename]
+
+ data_torch = torch.stack(datas, dim=0)
+
+ merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"
+
+ new_name = self.map_tensor_name(merged_name)
+
+ tensors.append((new_name, data_torch))
+
+ return tensors
+
+ if name.endswith(".expert_bias"):
+ name = name.replace(".expert_bias", ".expert_bias.bias")
+
+ return [(self.map_tensor_name(name), data_torch)]
+
+ def prepare_tensors(self):
+ super().prepare_tensors()
+
+ if self._experts is not None:
+ # flatten `list[dict[str, Tensor]]` into `list[str]`
+ experts = [k for d in self._experts for k in d.keys()]
+ if len(experts) > 0:
+ raise ValueError(f"Unprocessed experts: {experts}")
+
+
@ModelBase.register("GroveMoeForCausalLM", "modeling_grove_moe.GroveMoeForCausalLM")
class GroveMoeModel(TextModel):
model_arch = gguf.MODEL_ARCH.GROVEMOE
diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py
index 28002f766..0ebc1b160 100755
--- a/convert_hf_to_gguf_update.py
+++ b/convert_hf_to_gguf_update.py
@@ -139,7 +139,7 @@ models = [
{"name": "lfm2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LiquidAI/LFM2-Tokenizer"},
{"name": "exaone4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B", },
{"name": "mellum", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/JetBrains/Mellum-4b-base", },
- {"name": "llada-moe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/inclusionAI/LLaDA-MoE-7B-A1B-Base", },
+ {"name": "bailingmoe2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/inclusionAI/Ling-mini-base-2.0", },
{"name": "granite-docling", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ibm-granite/granite-docling-258M", },
]
diff --git a/docs/backend/zDNN.md b/docs/backend/zDNN.md
deleted file mode 100644
index 8d2e11177..000000000
--- a/docs/backend/zDNN.md
+++ /dev/null
@@ -1,61 +0,0 @@
-# llama.cpp for IBM zDNN Accelerator
-
-## Background
-
-IBM zDNN (Z Deep Neural Network) is a hardware acceleration library designed specifically to leverage the IBM NNPA (Neural Network Processor Assist) accelerator located within IBM Telum I and II processors. It provides significant performance improvements for neural network inference operations.
-
-### Llama.cpp + IBM zDNN
-
-The llama.cpp zDNN backend is designed to enable llama.cpp on IBM z17 and later systems via the IBM zDNN hardware acceleration library.
-
-## Software & Hardware Support
-
-| Hardware Level | Status | Verified |
-| -------------------- | ------------- | -------------------------- |
-| IBM z17 / LinuxONE 5 | Supported | RHEL 9.6, IBM z17, 40 IFLs |
-| IBM z16 / LinuxONE 4 | Not Supported | |
-
-## Data Types Supported
-
-| Data Type | Status |
-| --------- | --------- |
-| F32 | Supported |
-| F16 | Supported |
-| BF16 | Supported |
-
-## CMake Options
-
-The IBM zDNN backend has the following CMake options that control the behaviour of the backend.
-
-| CMake Option | Default Value | Description |
-| ------------ | ------------- | ----------------------------------- |
-| `GGML_ZDNN` | `OFF` | Compile llama.cpp with zDNN support |
-| `ZDNN_ROOT` | `""` | Override zDNN library lookup |
-
-## 1. Install zDNN Library
-
-Note: Using the zDNN library provided via `apt` or `yum` may not work correctly as reported in [#15772](https://github.com/ggml-org/llama.cpp/issues/15772). It is preferred that you compile from source.
-
-```sh
-git clone --recurse-submodules https://github.com/IBM/zDNN
-cd zDNN
-
-autoreconf .
-./configure --prefix=/opt/zdnn-libs
-
-make build
-sudo make install
-```
-
-## 2. Build llama.cpp
-
-```sh
-git clone https://github.com/ggml-org/llama.cpp
-cd llama.cpp
-
-cmake -S . -G Ninja -B build \
- -DCMAKE_BUILD_TYPE=Release \
- -DGGML_ZDNN=ON \
- -DZDNN_ROOT=/opt/zdnn-libs
-cmake --build build --config Release -j$(nproc)
-```
diff --git a/docs/build-riscv64-spacemit.md b/docs/build-riscv64-spacemit.md
deleted file mode 100644
index eaa653254..000000000
--- a/docs/build-riscv64-spacemit.md
+++ /dev/null
@@ -1,89 +0,0 @@
-> [!IMPORTANT]
-> This build documentation is specific only to RISC-V SpacemiT SOCs.
-
-## Build llama.cpp locally (for riscv64)
-
-1. Prepare Toolchain For RISCV
-~~~
-wget https://archive.spacemit.com/toolchain/spacemit-toolchain-linux-glibc-x86_64-v1.1.2.tar.xz
-~~~
-
-2. Build
-Below is the build script: it requires utilizing RISC-V vector instructions for acceleration. Ensure the `GGML_CPU_RISCV64_SPACEMIT` compilation option is enabled. The currently supported optimization version is `RISCV64_SPACEMIT_IME1`, corresponding to the `RISCV64_SPACEMIT_IME_SPEC` compilation option. Compiler configurations are defined in the `riscv64-spacemit-linux-gnu-gcc.cmake` file. Please ensure you have installed the RISC-V compiler and set the environment variable via `export RISCV_ROOT_PATH={your_compiler_path}`.
-```bash
-
-cmake -B build \
- -DCMAKE_BUILD_TYPE=Release \
- -DGGML_CPU_RISCV64_SPACEMIT=ON \
- -DLLAMA_CURL=OFF \
- -DGGML_RVV=ON \
- -DGGML_RV_ZFH=ON \
- -DGGML_RV_ZICBOP=ON \
- -DRISCV64_SPACEMIT_IME_SPEC=RISCV64_SPACEMIT_IME1 \
- -DCMAKE_TOOLCHAIN_FILE=${PWD}/cmake/riscv64-spacemit-linux-gnu-gcc.cmake \
- -DCMAKE_INSTALL_PREFIX=build/installed
-
-cmake --build build --parallel $(nproc) --config Release
-
-pushd build
-make install
-popd
-```
-
-## Simulation
-You can use QEMU to perform emulation on non-RISC-V architectures.
-
-1. Download QEMU
-~~~
-wget https://archive.spacemit.com/spacemit-ai/qemu/jdsk-qemu-v0.0.14.tar.gz
-~~~
-
-2. Run Simulation
-After build your llama.cpp, you can run the executable file via QEMU for simulation, for example:
-~~~
-export QEMU_ROOT_PATH={your QEMU file path}
-export RISCV_ROOT_PATH_IME1={your RISC-V compiler path}
-
-${QEMU_ROOT_PATH}/bin/qemu-riscv64 -L ${RISCV_ROOT_PATH_IME1}/sysroot -cpu max,vlen=256,elen=64,vext_spec=v1.0 ${PWD}/build/bin/llama-cli -m ${PWD}/models/Qwen2.5-0.5B-Instruct-Q4_0.gguf -t 1
-~~~
-## Performance
-#### Quantization Support For Matrix
-~~~
-model name : Spacemit(R) X60
-isa : rv64imafdcv_zicbom_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zfhmin_zca_zcd_zba_zbb_zbc_zbs_zkt_zve32f_zve32x_zve64d_zve64f_zve64x_zvfh_zvfhmin_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt
-mmu : sv39
-uarch : spacemit,x60
-mvendorid : 0x710
-marchid : 0x8000000058000001
-~~~
-
-Q4_0
-| Model | Size | Params | backend | threads | test | t/s |
-| -----------| -------- | ------ | ------- | ------- | ---- |------|
-Qwen2.5 0.5B |403.20 MiB|630.17 M| cpu | 4 | pp512|64.12 ± 0.26|
-Qwen2.5 0.5B |403.20 MiB|630.17 M| cpu | 4 | tg128|10.03 ± 0.01|
-Qwen2.5 1.5B |1011.16 MiB| 1.78 B | cpu | 4 | pp512|24.16 ± 0.02|
-Qwen2.5 1.5B |1011.16 MiB| 1.78 B | cpu | 4 | tg128|3.83 ± 0.06|
-Qwen2.5 3B | 1.86 GiB | 3.40 B | cpu | 4 | pp512|12.08 ± 0.02|
-Qwen2.5 3B | 1.86 GiB | 3.40 B | cpu | 4 | tg128|2.23 ± 0.02|
-
-Q4_1
-| Model | Size | Params | backend | threads | test | t/s |
-| -----------| -------- | ------ | ------- | ------- | ---- |------|
-Qwen2.5 0.5B |351.50 MiB|494.03 M| cpu | 4 | pp512|62.07 ± 0.12|
-Qwen2.5 0.5B |351.50 MiB|494.03 M| cpu | 4 | tg128|9.91 ± 0.01|
-Qwen2.5 1.5B |964.06 MiB| 1.54 B | cpu | 4 | pp512|22.95 ± 0.25|
-Qwen2.5 1.5B |964.06 MiB| 1.54 B | cpu | 4 | tg128|4.01 ± 0.15|
-Qwen2.5 3B | 1.85 GiB | 3.09 B | cpu | 4 | pp512|11.55 ± 0.16|
-Qwen2.5 3B | 1.85 GiB | 3.09 B | cpu | 4 | tg128|2.25 ± 0.04|
-
-
-Q4_K
-| Model | Size | Params | backend | threads | test | t/s |
-| -----------| -------- | ------ | ------- | ------- | ---- |------|
-Qwen2.5 0.5B |462.96 MiB|630.17 M| cpu | 4 | pp512|9.29 ± 0.05|
-Qwen2.5 0.5B |462.96 MiB|630.17 M| cpu | 4 | tg128|5.67 ± 0.04|
-Qwen2.5 1.5B | 1.04 GiB | 1.78 B | cpu | 4 | pp512|10.38 ± 0.10|
-Qwen2.5 1.5B | 1.04 GiB | 1.78 B | cpu | 4 | tg128|3.17 ± 0.08|
-Qwen2.5 3B | 1.95 GiB | 3.40 B | cpu | 4 | pp512|4.23 ± 0.04|
-Qwen2.5 3B | 1.95 GiB | 3.40 B | cpu | 4 | tg128|1.73 ± 0.00|
diff --git a/embd_res/klite.embd b/embd_res/klite.embd
index 0e503f34d..46ac65fa8 100644
--- a/embd_res/klite.embd
+++ b/embd_res/klite.embd
@@ -27427,7 +27427,8 @@ Current version indicated by LITEVER below.