llama : use new pre-tokenizer type

2026-09-18 16:55:05 +02:00 · 2024-04-26 20:08:28 +03:00
parent 9b4d63ae53
commit 43e12ce8e5
12 changed files with 87 additions and 44 deletions
@@ -17,6 +17,8 @@ static const std::map<std::string, std::vector<llama_token>> & k_tests() {
        { "   "                   , {     466, }, },
        { "\t"                    , {     192, }, },
        { "\n"                    , {     193, }, },
+        { "\n\n"                  , {    1001, }, },
+        { "\n\n\n"                , {   11331, }, },
        { "\t\n"                  , {   19125, }, },
        { "Hello world"           , {    9856,   1079, }, },
        { " Hello world"          , {   23090,   1079, }, },