Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 17:24:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

llama : towards llama3 tokenization support (wip)

Browse Source
This commit is contained in:
Georgi Gerganov
2024-04-26 14:55:03 +03:00
parent ed42711b90
commit 4907e41aa7
8 changed files with 298 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-tokenizer-0-llama.cpp
+1 -1
View File
@@ -126,7 +126,7 @@ int main(int argc, char **argv) {
}
printf("\n");
bool correct = res_nobos.size() == test_kv.second.size() && res_bos.size() == res_nobos.size() + 1 && res_bos[0] == 1;
bool correct = res_nobos.size() == test_kv.second.size() && res_bos.size() == res_nobos.size() + 1 && res_bos[0] == llama_token_bos(model);
for (int i = 0; i < (int) res_nobos.size() && correct; ++i) {
if (test_kv.second[i] != res_bos[i + 1]) {