mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-08-31 09:30:50 +02:00
unicode : include '~' in collapsed symbol class (#26972)
The collapsed \p{S} class was missing '~', which split " ~" into
separate pre-tokens and prevented the Ġ~ BPE merge used by DeepSeek V4.
This caused re-tokenized prompts to diverge from sampled tokens and
broke KV cache reuse.
Assisted-by: Codex
This commit is contained in:
@@ -0,0 +1,24 @@
|
||||
#include "../src/unicode.h"
|
||||
|
||||
#include <cstdio>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
int main() {
|
||||
const std::vector<std::string> regex_exprs = {
|
||||
"[~][A-Za-z]+| ?[\\p{S}]+|\\s+",
|
||||
};
|
||||
const std::vector<std::string> expected = { " ~", "foo" };
|
||||
const auto actual = unicode_regex_split(" ~foo", regex_exprs, false);
|
||||
|
||||
if (actual != expected) {
|
||||
fprintf(stderr, "unexpected split:");
|
||||
for (const auto & piece : actual) {
|
||||
fprintf(stderr, " [%s]", piece.c_str());
|
||||
}
|
||||
fprintf(stderr, "\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user