Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-08-31 17:40:52 +02:00
Code Issues Packages Projects Releases Wiki Activity

rope fixes

Browse Source
This commit is contained in:
Concedo
2023-11-02 20:41:16 +08:00
parent bc4ff72317
commit 42eabf2f2f
4 changed files with 15 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
llama.cpp
+9 -4
View File
@@ -1245,10 +1245,15 @@ struct llama_vocab {
id special_eot_id = 32010;
int find_bpe_rank(std::string token_left, std::string token_right) const {
GGML_ASSERT(token_left.find(" ") == std::string::npos);
GGML_ASSERT(token_left.find("\n") == std::string::npos);
GGML_ASSERT(token_right.find(" ") == std::string::npos);
GGML_ASSERT(token_right.find("\n") == std::string::npos);
// GGML_ASSERT(token_left.find(" ") == std::string::npos);
// GGML_ASSERT(token_left.find("\n") == std::string::npos);
// GGML_ASSERT(token_right.find(" ") == std::string::npos);
// GGML_ASSERT(token_right.find("\n") == std::string::npos);
//the above breaks gguf v1 falcons
replace_all(token_left, " ", "\u0120");
replace_all(token_left, "\n", "\u010A");
replace_all(token_right, " ", "\u0120");
replace_all(token_right, "\n", "\u010A");
auto it = bpe_ranks.find(std::make_pair(token_left, token_right));
if (it == bpe_ranks.end()) {