Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 17:25:07 +02:00
Code Issues Packages Projects Releases Wiki Activity

vocab : add ufakzeka pre-tokenizer (#29033)

Browse Source 
* vocab : add ufakzeka pre-tokenizer

* vocab : move ufakzeka to the models list and regenerate the hash mapping
This commit is contained in:
Sait Furkan Teke
2026-09-18 11:45:11 +03:00
committed by GitHub
parent 8ed1a55efc
commit dc85f89c7e
4 changed files with 15 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conversion/base.py
+3
View File
@@ -1861,6 +1861,9 @@ class TextModel(ModelBase):
if chkhsh == "972da7b59cec44d1f0a490a86c96df53859e486e481563e5dddac155013d87ac":
# ref: https://huggingface.co/poolside/Laguna-XS.2
res = "laguna"
if chkhsh == "653660222fb704f61cbf2b618a8ae6502b7f8b20c980f9a5de07ed78e13319cd":
# ref: https://huggingface.co/ufakai/ufakzeka-1
res = "ufakzeka"
if res is None:
logger.warning("\n")
convert_hf_to_gguf_update.py
+1
View File
@@ -163,6 +163,7 @@ models = [
{"name": "granite-embed-multi-311m", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ibm-granite/granite-embedding-311m-multilingual-r2", },
{"name": "mellum2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Base"},
{"name": "laguna", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/poolside/Laguna-XS.2", },
{"name": "ufakzeka", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ufakai/ufakzeka-1", },
]
# some models are known to be broken upstream, so we will skip them as exceptions
src/llama-vocab.cpp
+10
View File
@@ -488,6 +488,12 @@ struct llm_tokenizer_bpe : llm_tokenizer {
"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1}| ?[^\\s\\p{L}\\p{N}\\r\\n]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
};
break;
case LLAMA_VOCAB_PRE_TYPE_UFAKZEKA:
regex_exprs = {
// Qwen2 pattern without the English contraction group, so Turkish apostrophe suffixes stay attached
"[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
};
break;
case LLAMA_VOCAB_PRE_TYPE_GROK_2:
regex_exprs = {
// original regex from tokenizer.json
@@ -2376,6 +2382,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
tokenizer_pre == "kimi-k2") {
pre_type = LLAMA_VOCAB_PRE_TYPE_KIMI_K2;
clean_spaces = false;
} else if (
tokenizer_pre == "ufakzeka") {
pre_type = LLAMA_VOCAB_PRE_TYPE_UFAKZEKA;
clean_spaces = false;
} else if (
tokenizer_pre == "grok-2") {
pre_type = LLAMA_VOCAB_PRE_TYPE_GROK_2;
src/llama-vocab.h
+1
View File
@@ -67,6 +67,7 @@ enum llama_vocab_pre_type {
LLAMA_VOCAB_PRE_TYPE_LAGUNA = 56,
LLAMA_VOCAB_PRE_TYPE_HY_V4 = 57,
LLAMA_VOCAB_PRE_TYPE_SPARK2_5 = 58,
LLAMA_VOCAB_PRE_TYPE_UFAKZEKA = 59,
};
struct LLM_KV;