Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-18 16:54:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

handle unknown tokens

Browse Source
This commit is contained in:
Vladimir Mandic
2024-05-21 16:59:16 -04:00
parent 934cc71f06
commit 49c997e20f
1 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/prompt_parser_diffusers.py
+5 -1
View File
@@ -120,7 +120,11 @@ def get_tokens(msg, prompt):
ids = getattr(ids, 'input_ids', [])
tokens = []
for i in ids:
tokens.append(list(token_dict.keys())[list(token_dict.values()).index(i)])
try:
key = list(token_dict.keys())[list(token_dict.values()).index(i)]
tokens.append(key)
except Exception:
tokens.append(f'UNK_{i}')
token_count = len(ids) - int(has_bos_token) - int(has_eos_token)
shared.log.trace(f'Prompt tokenizer: type={msg} tokens={token_count} {tokens}')