mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-08-26 06:31:13 +02:00
Thiago Padilha
afd439df1f
The collapsed \p{S} class was missing '~', which split " ~" into
separate pre-tokens and prevented the Ġ~ BPE merge used by DeepSeek V4.
This caused re-tokenized prompts to diverge from sampled tokens and
broke KV cache reuse.
Assisted-by: Codex
25 lines
589 B
C++
25 lines
589 B
C++
#include "../src/unicode.h"
|
|
|
|
#include <cstdio>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
int main() {
|
|
const std::vector<std::string> regex_exprs = {
|
|
"[~][A-Za-z]+| ?[\\p{S}]+|\\s+",
|
|
};
|
|
const std::vector<std::string> expected = { " ~", "foo" };
|
|
const auto actual = unicode_regex_split(" ~foo", regex_exprs, false);
|
|
|
|
if (actual != expected) {
|
|
fprintf(stderr, "unexpected split:");
|
|
for (const auto & piece : actual) {
|
|
fprintf(stderr, " [%s]", piece.c_str());
|
|
}
|
|
fprintf(stderr, "\n");
|
|
return 1;
|
|
}
|
|
|
|
return 0;
|
|
}
|