From 49c997e20fdba9e28a8e4d8c9eebbeffa82e6cf5 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 21 May 2024 16:59:16 -0400 Subject: [PATCH] handle unknown tokens --- modules/prompt_parser_diffusers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 72602d4fd..542857913 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -120,7 +120,11 @@ def get_tokens(msg, prompt): ids = getattr(ids, 'input_ids', []) tokens = [] for i in ids: - tokens.append(list(token_dict.keys())[list(token_dict.values()).index(i)]) + try: + key = list(token_dict.keys())[list(token_dict.values()).index(i)] + tokens.append(key) + except Exception: + tokens.append(f'UNK_{i}') token_count = len(ids) - int(has_bos_token) - int(has_eos_token) shared.log.trace(f'Prompt tokenizer: type={msg} tokens={token_count} {tokens}')