Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-18 16:55:05 +02:00
Code Issues Packages Projects Releases Wiki Activity

tests : add tokenizer tests for numbers

Browse Source
This commit is contained in:
Georgi Gerganov
2024-04-26 13:21:28 +03:00
parent c56e19db4b
commit 7a44e44342
4 changed files with 46 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-tokenizer-0-falcon.cpp
+10
View File
@@ -36,9 +36,19 @@ static const std::map<std::string, std::vector<llama_token>> & k_tests() {
{ " Hello" , { 258, 23090, }, },
{ " Hello" , { 466, 23090, }, },
{ " Hello\n Hello" , { 466, 23090, 742, 23090, }, },
{ " (" , { 204, 19, }, },
{ "\n =" , { 1212, 40, }, },
{ "' era" , { 18, 4932, }, },
{ "Hello, y'all! How are you 😁 ?我想在apple工作1314151天～", { 9856, 23, 291, 18, 436, 12, 1265, 362, 299, 8196, 207, 204, 42, 50087, 123, 2727, 20300, 32022, 133, 234, 17419, 30137, 28, 7858, 181, 133, 236, }, },
{ "3" , { 30, }, },
{ "33" , { 3138, }, },
{ "333" , { 22287, }, },
{ "3333" , { 22287, 30, }, },
{ "33333" , { 22287, 3138, }, },
{ "333333" , { 22287, 22287, }, },
{ "3333333" , { 22287, 22287, 30, }, },
{ "33333333" , { 22287, 22287, 3138, }, },
{ "333333333" , { 22287, 22287, 22287, }, },
};
return _k_tests;