Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-02 11:01:20 +02:00
Code Issues Packages Projects Releases Wiki Activity

workaround for deepseek not working

Browse Source
This commit is contained in:
Concedo
2024-07-05 22:39:53 +08:00
parent 6b0756506b
commit 388a2aff00
3 changed files with 17 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/unicode.cpp
+11 -1
View File
@@ -189,9 +189,19 @@ static std::unordered_map<std::string, uint8_t> unicode_utf8_to_byte_map() {
return map;
}
static bool unicode_wstring_from_utf8_failed_once = false;
static inline std::wstring unicode_wstring_from_utf8(const std::string & s) {
std::wstring_convert<std::codecvt_utf8<wchar_t>> conv;
return conv.from_bytes(s);
try {
return conv.from_bytes(s);
} catch(const std::exception & e) {
if(!unicode_wstring_from_utf8_failed_once)
{
unicode_wstring_from_utf8_failed_once = true;
printf("\nunicode_wstring_from_utf8 failed: %s\n", e.what());
}
return L"";
}
}
static std::vector<std::string> unicode_byte_encoding_process(const std::vector<std::string> & bpe_words) {