Merge branch 'master' into concedo

# Conflicts:
#	.github/workflows/build.yml
#	CMakeLists.txt
#	Makefile
#	README.md
#	main.cpp
This commit is contained in:
Concedo
2023-03-22 22:31:45 +08:00
25 changed files with 3028 additions and 1944 deletions
+6 -1
View File
@@ -11,4 +11,9 @@
#include <string>
#include <vector>
std::vector<gpt_vocab::id> legacy_llama_tokenize(const gpt_vocab & vocab, const std::string & text, bool bos);
#include "llama.h"
int check_file_format(const std::string & fname);
std::vector<llama_token> legacy_llama_tokenize(struct llama_context * ctx, const std::string & text, bool add_bos);