Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-18 16:55:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

added skip bos for tokenize endpoint

Browse Source
This commit is contained in:
Concedo
2024-06-05 10:49:11 +08:00
parent 5789417802
commit 10b148f4c2
5 changed files with 11 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
expose.cpp
+2 -2
View File
@@ -277,11 +277,11 @@ extern "C"
}
static std::vector<int> toks; //just share a static object for token counting
token_count_outputs token_count(const char * input)
token_count_outputs token_count(const char * input, bool addbos)
{
std::string inputstr = input;
token_count_outputs output;
toks = gpttype_get_token_arr(inputstr);
toks = gpttype_get_token_arr(inputstr,addbos);
output.count = toks.size();
output.ids = toks.data(); //this may be slightly unsafe
return output;