Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-13 02:10:45 +02:00
Code Issues Packages Projects Releases Wiki Activity

llama : towards llama3 tokenization support (wip)

Browse Source
This commit is contained in:
Georgi Gerganov
2024-04-26 14:55:03 +03:00
parent ed42711b90
commit 4907e41aa7
8 changed files with 298 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-tokenizer-0-deepseek-coder.cpp
+2 -2
View File
@@ -89,8 +89,8 @@ int main(int argc, char **argv) {
}
}
if (llama_vocab_type(model) != LLAMA_VOCAB_TYPE_DEEPSEEKCODER) {
fprintf(stderr, "%s : error: vocab type is not DEEPSEEKCODER\n", __func__);
if (llama_vocab_type(model) != LLAMA_VOCAB_TYPE_BPE) {
fprintf(stderr, "%s : error: vocab type is not BPE\n", __func__);
llama_free_model(model);
llama_free(ctx);
return 2;