diff --git a/src/unicode.cpp b/src/unicode.cpp index b02ecdc93..93996f9dd 100644 --- a/src/unicode.cpp +++ b/src/unicode.cpp @@ -1241,7 +1241,7 @@ std::vector unicode_regex_split(const std::string & text, const std { unicode_cpt_flags::LETTER, "\x41-\x5A\x61-\x7A" }, // A-Za-z { unicode_cpt_flags::PUNCTUATION, "\x21-\x23\x25-\x2A\x2C-\x2F\x3A-\x3B\x3F-\x40\\\x5B-\\\x5D\x5F\\\x7B\\\x7D" }, // !-#%-*,-/:-;?-@\[-\]_\{\} { unicode_cpt_flags::ACCENT_MARK, "" }, // no sub-128 codepoints - { unicode_cpt_flags::SYMBOL, "\\\x24\\\x2B\x3C-\x3E\x5E\x60\\\x7C" }, // $+<=>^`| + { unicode_cpt_flags::SYMBOL, "\\\x24\\\x2B\x3C-\x3E\x5E\x60\\\x7C\\\x7E" }, // $+<=>^`|~ }; // compute collapsed codepoints only if needed by at least one regex diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3ee51b519..0db7fd9ef 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -116,6 +116,8 @@ function(llama_build_and_test source) set_property(TEST ${TEST_TARGET} PROPERTY LABELS ${LLAMA_TEST_LABEL}) endfunction() +llama_build_and_test(test-unicode.cpp) + # build test-tokenizer-0 target once and add many tests llama_build(test-tokenizer-0.cpp) diff --git a/tests/test-unicode.cpp b/tests/test-unicode.cpp new file mode 100644 index 000000000..2347d9000 --- /dev/null +++ b/tests/test-unicode.cpp @@ -0,0 +1,24 @@ +#include "../src/unicode.h" + +#include +#include +#include + +int main() { + const std::vector regex_exprs = { + "[~][A-Za-z]+| ?[\\p{S}]+|\\s+", + }; + const std::vector expected = { " ~", "foo" }; + const auto actual = unicode_regex_split(" ~foo", regex_exprs, false); + + if (actual != expected) { + fprintf(stderr, "unexpected split:"); + for (const auto & piece : actual) { + fprintf(stderr, " [%s]", piece.c_str()); + } + fprintf(stderr, "\n"); + return 1; + } + + return 0; +}