Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-19 09:15:04 +02:00
Code Issues Packages Projects Releases Wiki Activity

convert : use a string for the SentencePiece tokenizer path

Browse Source
This commit is contained in:
Francis Couture-Harpin
2024-05-01 13:07:10 -04:00
parent 3870164f47
commit dcd8dfa1b5
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
convert.py
+1 -1
View File
@@ -463,7 +463,7 @@ class SentencePieceVocab(Vocab):
raise FileNotFoundError('Cannot find tokenizer.model')
self.sentencepiece_tokenizer = SentencePieceProcessor()
self.sentencepiece_tokenizer.LoadFromFile(fname_tokenizer)
self.sentencepiece_tokenizer.LoadFromFile(str(fname_tokenizer))
vocab_size = self.sentencepiece_tokenizer.vocab_size()
new_tokens = {id: piece for piece, id in added_tokens.items() if id >= vocab_size}