Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-08-25 22:21:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

detokenize add special token ids

Browse Source
This commit is contained in:
Concedo
2026-05-17 17:04:47 +08:00
parent c1514e328b
commit de2c5f1cef
3 changed files with 20 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
expose.cpp
+2 -2
View File
@@ -362,14 +362,14 @@ extern "C"
}
static std::string detokenized_str = ""; //just share a static object for detokenizing
const char * detokenize(const token_count_outputs input)
const char * detokenize(const detokenize_inputs input)
{
std::vector<int> input_arr;
for(int i=0;i<input.count;++i)
{
input_arr.push_back(input.ids[i]);
}
detokenized_str = gpttype_detokenize(input_arr,false);
detokenized_str = gpttype_detokenize(input_arr,input.special);
return detokenized_str.c_str();
}