Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 01:04:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

tests : add tokenizer tests for numbers

Browse Source
This commit is contained in:
Georgi Gerganov
2024-04-26 13:21:28 +03:00
parent c56e19db4b
commit 7a44e44342
4 changed files with 46 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-tokenizer-0-llama.cpp
+13 -1
View File
@@ -36,7 +36,19 @@ static const std::map<std::string, std::vector<llama_token>> & k_tests() {
{ " Hello" , { 1678, 15043, }, },
{ " Hello" , { 268, 15043, }, },
{ " Hello\n Hello" , { 268, 15043, 13, 1678, 15043, }, },
{ " (" , { 29871, 313, }, },
{ " (" , { 29871, 313, }, },
{ "\n =" , { 29871, 13, 353, }, },
{ "' era" , { 525, 3152, }, },
{ "Hello, y'all! How are you 😁 ?我想在apple工作1314151天～", { 15043, 29892, 343, 29915, 497, 29991, 1128, 526, 366, 29871, 243, 162, 155, 132, 1577, 30672, 31522, 30505, 11548, 31041, 30732, 29896, 29941, 29896, 29946, 29896, 29945, 29896, 30408, 30739, }, },
{ "3" , { 29871, 29941, }, },
{ "33" , { 29871, 29941, 29941, }, },
{ "333" , { 29871, 29941, 29941, 29941, }, },
{ "3333" , { 29871, 29941, 29941, 29941, 29941, }, },
{ "33333" , { 29871, 29941, 29941, 29941, 29941, 29941, }, },
{ "333333" , { 29871, 29941, 29941, 29941, 29941, 29941, 29941, }, },
{ "3333333" , { 29871, 29941, 29941, 29941, 29941, 29941, 29941, 29941, }, },
{ "33333333" , { 29871, 29941, 29941, 29941, 29941, 29941, 29941, 29941, 29941, }, },
{ "333333333" , { 29871, 29941, 29941, 29941, 29941, 29941, 29941, 29941, 29941, 29941, }, },
};
return _k_tests;