From f1a4faadb8782b4309275fef08312523b0574551 Mon Sep 17 00:00:00 2001 From: awsr <43862868+awsr@users.noreply.github.com> Date: Fri, 20 Feb 2026 02:27:26 -0800 Subject: [PATCH 1/9] Add optional detailed token counts --- modules/ui_common.py | 57 ++++++++++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/modules/ui_common.py b/modules/ui_common.py index 3b7530e08..82ca944a1 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -1,6 +1,7 @@ import json import html import os +import re import shutil import platform import subprocess @@ -427,32 +428,52 @@ def connect_reuse_seed(seed: gr.Number, reuse_seed_btn: gr.Button, generation_in reuse_seed_btn.click(fn=copy_seed, _js="(x, y) => [x, selected_gallery_index()]", show_progress='hidden', inputs=[generation_info, dummy_component], outputs=[seed, dummy_component, subseed_strength]) -def update_token_counter(text): - token_count = 0 - max_length = 75 +def update_token_counter(text: str): if shared.state.job_count > 0: log.debug('Tokenizer busy') - return f"{token_count}/{max_length}" - from modules import extra_networks - if isinstance(text, list): - prompt, _ = extra_networks.parse_prompts(text) - else: - prompt, _ = extra_networks.parse_prompt(text) + return gr.update(value="--/--", visible=True) + + from modules.extra_networks import parse_prompt + + token_counts = [0] + max_length = 75 + count_formatted = '' + visible = False + + prompt, _ = parse_prompt(text) + prompt_list = [prompt] + ids = [] if shared.sd_loaded and hasattr(shared.sd_model, 'tokenizer') and shared.sd_model.tokenizer is not None: + if shared.opts.prompt_detailed_tokens: + p_split = re.compile(r'\bBREAK\b|\n' if shared.opts.sd_textencder_linebreak else r'\bBREAK\b') + prompt_list = re.split(p_split, prompt) + tokenizer = shared.sd_model.tokenizer # For multi-modal processors (e.g., PixtralProcessor), use the underlying text tokenizer if hasattr(tokenizer, 'tokenizer') and tokenizer.tokenizer is not None: tokenizer = tokenizer.tokenizer - has_bos_token = hasattr(tokenizer, 'bos_token_id') and tokenizer.bos_token_id is not None - has_eos_token = hasattr(tokenizer, 'eos_token_id') and tokenizer.eos_token_id is not None - try: - ids = tokenizer(prompt) - ids = getattr(ids, 'input_ids', []) - except Exception: - ids = [] - token_count = len(ids) - int(has_bos_token) - int(has_eos_token) + has_bos_token = getattr(tokenizer, 'bos_token_id', None) is not None + has_eos_token = getattr(tokenizer, 'eos_token_id', None) is not None model_max_length = getattr(tokenizer, 'model_max_length', 0) max_length = model_max_length - int(has_bos_token) - int(has_eos_token) if max_length is None or max_length < 0 or max_length > 10000: max_length = 0 - return gr.update(value=f"{token_count}/{max_length}", visible=token_count > 0) + + try: + try: + ids = getattr(tokenizer(prompt_list), 'input_ids', []) + except TypeError: + for p in prompt_list: + ids.append(getattr(tokenizer(p), 'input_ids', [])) + except Exception as e: + shared.log.warning("Token counter:", e) + return gr.update(value=f"??/{max_length}", visible=True) + + token_counts = [len(group) - int(has_bos_token) - int(has_eos_token) for group in ids] + if len(token_counts) > 1: + visible = True + count_formatted = f"{token_counts} {sum(token_counts)}" if shared.opts.prompt_detailed_tokens else str(sum(token_counts)) + elif len(token_counts) == 1 and token_counts[0] > 0: + visible = True + count_formatted = str(token_counts[0]) + return gr.update(value=f"{count_formatted}/{max_length}", visible=visible) From 897925da2a1e37a23a1dac74d92ffbf7c714de64 Mon Sep 17 00:00:00 2001 From: awsr <43862868+awsr@users.noreply.github.com> Date: Fri, 20 Feb 2026 03:23:17 -0800 Subject: [PATCH 2/9] Minor cleanup --- modules/ui_common.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/ui_common.py b/modules/ui_common.py index 82ca944a1..99a8a2d22 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -435,9 +435,7 @@ def update_token_counter(text: str): from modules.extra_networks import parse_prompt - token_counts = [0] - max_length = 75 - count_formatted = '' + count_formatted = '0' visible = False prompt, _ = parse_prompt(text) From 0d27d9ea090f8300677d4c2571dedf2b62ac4fb5 Mon Sep 17 00:00:00 2001 From: awsr <43862868+awsr@users.noreply.github.com> Date: Fri, 27 Feb 2026 18:06:01 -0800 Subject: [PATCH 3/9] Use manual looping - I'm assuming there won't be much of a difference in performance. If it ends up being too slow, this can always be reverted. --- modules/ui_common.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/modules/ui_common.py b/modules/ui_common.py index 99a8a2d22..0231676c6 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -458,11 +458,8 @@ def update_token_counter(text: str): max_length = 0 try: - try: - ids = getattr(tokenizer(prompt_list), 'input_ids', []) - except TypeError: - for p in prompt_list: - ids.append(getattr(tokenizer(p), 'input_ids', [])) + for p in prompt_list: + ids.append(getattr(tokenizer(p), 'input_ids', [])) except Exception as e: shared.log.warning("Token counter:", e) return gr.update(value=f"??/{max_length}", visible=True) From f43b459e9b3a56cbddcfbbb3260c23c0e9e80941 Mon Sep 17 00:00:00 2001 From: awsr <43862868+awsr@users.noreply.github.com> Date: Fri, 27 Feb 2026 20:11:10 -0800 Subject: [PATCH 4/9] Revert "Use manual looping" This reverts commit e5fdb0015727edc68938a8a394a83b3c70f9833a. --- modules/ui_common.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/ui_common.py b/modules/ui_common.py index 0231676c6..99a8a2d22 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -458,8 +458,11 @@ def update_token_counter(text: str): max_length = 0 try: - for p in prompt_list: - ids.append(getattr(tokenizer(p), 'input_ids', [])) + try: + ids = getattr(tokenizer(prompt_list), 'input_ids', []) + except TypeError: + for p in prompt_list: + ids.append(getattr(tokenizer(p), 'input_ids', [])) except Exception as e: shared.log.warning("Token counter:", e) return gr.update(value=f"??/{max_length}", visible=True) From 09c82064e127f97305a63eb5a3593fb6da09978c Mon Sep 17 00:00:00 2001 From: awsr <43862868+awsr@users.noreply.github.com> Date: Mon, 2 Mar 2026 15:30:28 -0800 Subject: [PATCH 5/9] Only enable detailed with native prompt parsing --- modules/ui_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ui_common.py b/modules/ui_common.py index 99a8a2d22..64fce5c3e 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -442,7 +442,7 @@ def update_token_counter(text: str): prompt_list = [prompt] ids = [] if shared.sd_loaded and hasattr(shared.sd_model, 'tokenizer') and shared.sd_model.tokenizer is not None: - if shared.opts.prompt_detailed_tokens: + if shared.opts.prompt_detailed_tokens and shared.opts.prompt_attention == 'native': p_split = re.compile(r'\bBREAK\b|\n' if shared.opts.sd_textencder_linebreak else r'\bBREAK\b') prompt_list = re.split(p_split, prompt) From 686ac0375de25406e8e548b1f6e80ce65dd8d1a6 Mon Sep 17 00:00:00 2001 From: awsr <43862868+awsr@users.noreply.github.com> Date: Thu, 12 Mar 2026 23:48:33 -0700 Subject: [PATCH 6/9] Always enabled --- modules/ui_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ui_common.py b/modules/ui_common.py index 64fce5c3e..e9c254263 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -442,7 +442,7 @@ def update_token_counter(text: str): prompt_list = [prompt] ids = [] if shared.sd_loaded and hasattr(shared.sd_model, 'tokenizer') and shared.sd_model.tokenizer is not None: - if shared.opts.prompt_detailed_tokens and shared.opts.prompt_attention == 'native': + if shared.opts.prompt_attention == 'native': p_split = re.compile(r'\bBREAK\b|\n' if shared.opts.sd_textencder_linebreak else r'\bBREAK\b') prompt_list = re.split(p_split, prompt) From 041029d184082c156d5d395bcc98f94b6c45fe9d Mon Sep 17 00:00:00 2001 From: awsr <43862868+awsr@users.noreply.github.com> Date: Fri, 13 Mar 2026 00:03:05 -0700 Subject: [PATCH 7/9] Restore max length of 75 (77 for model) Default model max length for CLIP is 77 including BOS and EOS. --- modules/ui_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ui_common.py b/modules/ui_common.py index e9c254263..a1d4481c3 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -452,7 +452,7 @@ def update_token_counter(text: str): tokenizer = tokenizer.tokenizer has_bos_token = getattr(tokenizer, 'bos_token_id', None) is not None has_eos_token = getattr(tokenizer, 'eos_token_id', None) is not None - model_max_length = getattr(tokenizer, 'model_max_length', 0) + model_max_length = getattr(tokenizer, 'model_max_length', 77) max_length = model_max_length - int(has_bos_token) - int(has_eos_token) if max_length is None or max_length < 0 or max_length > 10000: max_length = 0 From 9a7f1e7978e2cab7a62bae9760d751d74c1954e1 Mon Sep 17 00:00:00 2001 From: awsr <43862868+awsr@users.noreply.github.com> Date: Fri, 13 Mar 2026 00:15:54 -0700 Subject: [PATCH 8/9] Typing update --- modules/ui_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ui_common.py b/modules/ui_common.py index a1d4481c3..c771a0ad9 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -459,7 +459,7 @@ def update_token_counter(text: str): try: try: - ids = getattr(tokenizer(prompt_list), 'input_ids', []) + ids: list = getattr(tokenizer(prompt_list), 'input_ids', []) except TypeError: for p in prompt_list: ids.append(getattr(tokenizer(p), 'input_ids', [])) From 51a10320c7c4cd810278723daad7f25ef2733f5e Mon Sep 17 00:00:00 2001 From: awsr <43862868+awsr@users.noreply.github.com> Date: Fri, 13 Mar 2026 01:06:09 -0700 Subject: [PATCH 9/9] Implement updated Warn Once handling --- modules/ui_common.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/ui_common.py b/modules/ui_common.py index c771a0ad9..133136d8d 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -5,6 +5,7 @@ import re import shutil import platform import subprocess +from weakref import WeakSet import gradio as gr from modules import paths, call_queue, shared, errors, ui_sections, ui_symbols, ui_components, generation_parameters_copypaste, images, scripts_manager, script_callbacks, infotext, processing from modules.logger import log @@ -14,6 +15,8 @@ folder_symbol = ui_symbols.folder debug = log.trace if os.environ.get('SD_PASTE_DEBUG', None) is not None else lambda *args, **kwargs: None debug('Trace: PASTE') +warn_once_set = WeakSet() + def gr_show(visible=True): return {"visible": visible, "__type__": "update"} @@ -464,7 +467,9 @@ def update_token_counter(text: str): for p in prompt_list: ids.append(getattr(tokenizer(p), 'input_ids', [])) except Exception as e: - shared.log.warning("Token counter:", e) + if tokenizer not in warn_once_set: + log.warning(f"Token counter: {e}") + warn_once_set.add(tokenizer) return gr.update(value=f"??/{max_length}", visible=True) token_counts = [len(group) - int(has_bos_token) - int(has_eos_token) for group in ids]