Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-18 16:55:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source 
# Conflicts:
#	.github/actions/windows-setup-cuda/action.yml
#	.github/workflows/build-3rd-party.yml
#	.github/workflows/build-android.yml
#	.github/workflows/build-apple.yml
#	.github/workflows/build-cann.yml
#	.github/workflows/build-cuda-ubuntu.yml
#	.github/workflows/build-ibm.yml
#	.github/workflows/build-msys.yml
#	.github/workflows/build-opencl.yml
#	.github/workflows/build-openvino.yml
#	.github/workflows/build-riscv.yml
#	.github/workflows/build-rpc.yml
#	.github/workflows/build-sanitize.yml
#	.github/workflows/build-self-hosted.yml
#	.github/workflows/build-sycl.yml
#	.github/workflows/build-vulkan.yml
#	.github/workflows/build-webgpu.yml
#	.github/workflows/hip-quality-check.yml
#	.github/workflows/release.yml
#	.github/workflows/server-sanitize.yml
#	.github/workflows/server-self-hosted.yml
#	.github/workflows/server.yml
#	.github/workflows/ui-self-hosted.yml
#	.github/workflows/ui.yml
#	CONTRIBUTING.md
#	ci/run.sh
#	docs/autoparser.md
#	docs/multimodal/granitevision.md
#	examples/model-conversion/README.md
#	ggml/src/ggml-hexagon/ggml-hexagon.cpp
#	ggml/src/ggml-hexagon/htp/CMakeLists.txt
#	ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c
#	ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c
#	ggml/src/ggml-hexagon/htp/htp-ops.h
#	ggml/src/ggml-hexagon/htp/main.c
#	ggml/src/ggml-hexagon/htp/matmul-ops.c
#	ggml/src/ggml-opencl/CMakeLists.txt
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	ggml/src/ggml-vulkan/CMakeLists.txt
#	ggml/src/ggml-vulkan/ggml-vulkan.cpp
#	ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt
#	ggml/src/ggml-webgpu/ggml-webgpu.cpp
#	ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl
#	ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl
#	ggml/src/ggml-zendnn/ggml-zendnn.cpp
#	pyproject.toml
#	scripts/snapdragon/adb/run-completion.sh
#	scripts/snapdragon/adb/run-tool.sh
#	scripts/sync_vendor.py
#	tests/gguf-model-data.cpp
#	tools/cli/README.md
#	tools/completion/README.md
#	tools/server/README.md
This commit is contained in:
Concedo
2026-05-28 15:28:29 +08:00
parent b6f56b4431 491c4d7d2e
commit ca942eb172
20 changed files with 1618 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-vocab.cpp
+11
View File
@@ -736,6 +736,14 @@ struct llm_tokenizer_bpe : llm_tokenizer {
};
byte_encode = false;
break;
case LLAMA_VOCAB_PRE_TYPE_MINICPM5:
regex_exprs = {
// original regex from tokenizer.json (openbmb/MiniCPM5-1B)
"\\p{N}{1,3}",
// "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}+| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}+| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
};
break;
default:
// default regex for BPE tokenization pre-processing
regex_exprs = {
@@ -2275,6 +2283,9 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
} else if (tokenizer_pre == "default") {
pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
} else if (tokenizer_pre == "minicpm5") {
pre_type = LLAMA_VOCAB_PRE_TYPE_MINICPM5;
ignore_merges = true;
} else if (
tokenizer_pre == "llama3" ||
tokenizer_pre == "llama-v3" ||
src/llama-vocab.h
+1
View File
@@ -61,6 +61,7 @@ enum llama_vocab_pre_type {
LLAMA_VOCAB_PRE_TYPE_JAIS2 = 49,
LLAMA_VOCAB_PRE_TYPE_GEMMA4 = 50,
LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE = 51,
LLAMA_VOCAB_PRE_TYPE_MINICPM5 = 52,
};
struct LLM_KV;