merged the changes from deepseeker models to main branch

This commit is contained in:
Jaggzh
2024-02-12 04:04:34 -08:00
committed by Georgi Gerganov
parent 83b72cb086
commit 6fbab2dbc8
15 changed files with 886 additions and 151 deletions
+1 -1
View File
@@ -38,6 +38,7 @@ static const std::map<std::string, std::vector<llama_token>> & k_tests() {
{ " Hello\n Hello" , { 466, 23090, 742, 23090, }, },
{ "\n =" , { 1212, 40, }, },
{ "' era" , { 18, 4932, }, },
{ "Hello, y'all! How are you 😁 ?我想在apple工作1314151天~", { 9856, 23, 291, 18, 436, 12, 1265, 362, 299, 8196, 207, 204, 42, 50087, 123, 2727, 20300, 32022, 133, 234, 17419, 30137, 28, 7858, 181, 133, 236, }, },
};
return _k_tests;
@@ -115,7 +116,6 @@ int main(int argc, char **argv) {
printf("\n");
bool correct = res.size() == test_kv.second.size();
for (int i = 0; i < (int) res.size() && correct; ++i) {
if (test_kv.second[i] != res[i]) {
correct = false;