Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 09:15:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge commit '506200cf8b5c8419ce97d16dc8c50f4634e21ebe' into concedo_experimental

Browse Source 
# Conflicts:
#	docs/multimodal.md
#	scripts/compare-llama-bench.py
#	src/llama-vocab.cpp
#	tools/llama-bench/README.md
#	tools/llama-bench/llama-bench.cpp
This commit is contained in:
Concedo
2026-04-07 14:58:36 +08:00
parent 5e16453f0c 506200cf8b
commit 15d269197e
6 changed files with 170 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-vocab.cpp
+7 -2
View File
@@ -3064,8 +3064,9 @@ uint8_t llama_vocab::impl::token_to_byte(llama_token id) const {
return strtol(buf.c_str(), NULL, 16);
}
case LLAMA_VOCAB_TYPE_BPE: {
GGML_ASSERT_CONTINUE(false);
return unicode_utf8_to_byte(token_data.text); // TODO: why is this here after GGML_ASSERT?
// Gemma4 uses BPE with SPM-style byte fallback tokens (<0xXX>)
auto buf = token_data.text.substr(3, 2);
return strtol(buf.c_str(), NULL, 16);
}
case LLAMA_VOCAB_TYPE_WPM: {
GGML_ABORT("fatal error");
@@ -3578,6 +3579,10 @@ int32_t llama_vocab::impl::token_to_piece(llama_token token, char * buf, int32_t
std::string result = llama_decode_text(token_text);
return _try_copy(result.data(), result.size());
}
if (attr & LLAMA_TOKEN_ATTR_BYTE) {
char byte = (char) token_to_byte(token);
return _try_copy((char*) &byte, 1);
}
break;
}
case LLAMA_VOCAB_TYPE_RWKV: {