Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-20 01:31:31 +02:00
Code Issues Packages Projects Releases Wiki Activity

unicode : clean-up

Browse Source
This commit is contained in:
Georgi Gerganov
2024-04-28 18:01:59 +03:00
parent d63cc9068b
commit e972e6cbf8
3 changed files with 53 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
unicode-data.h
+2 -5
View File
@@ -2,8 +2,6 @@
#include <cstdint>
#include <map>
#include <set>
#include <string>
#include <utility>
#include <vector>
@@ -14,6 +12,5 @@ extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_accent_ma
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_punctuation;
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_symbol;
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_control;
extern const std::multimap<uint32_t, uint32_t> unicode_map_nfd;
extern const std::map<char32_t, char32_t> unicode_map_lowercase;
extern const std::set<std::string> unicode_regex_with_custom_preprocessor;
extern const std::multimap<uint32_t, uint32_t> unicode_map_nfd;
extern const std::map<char32_t, char32_t> unicode_map_lowercase;