Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 01:04:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

llama : use new pre-tokenizer type

Browse Source
This commit is contained in:
Georgi Gerganov
2024-04-26 20:08:28 +03:00
parent 9b4d63ae53
commit 43e12ce8e5
12 changed files with 87 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-tokenizer-0-llama.cpp
+2
View File
@@ -17,6 +17,8 @@ static const std::map<std::string, std::vector<llama_token>> & k_tests() {
{ " " , { 268, }, },
{ "\t" , { 29871, 12, }, },
{ "\n" , { 29871, 13, }, },
{ "\n\n" , { 29871, 13, 13, }, },
{ "\n\n\n" , { 29871, 13, 13, 13, }, },
{ "\t\n" , { 29871, 12, 13, }, },
{ "Hello world" , { 15043, 3186, }, },
{ " Hello world" , { 29871, 15043, 3186, }, },