add setting for prompt linebreaks

Signed-off-by: Vladimir Mandic <mandic00@live.com>

modules/prompt_parser.py

@@ -326,7 +326,7 @@ def parse_prompt_attention(text):
         whitespace = ''
     else:
         re_attention = re_attention_v1
-        if native:
+        if native and opts.sd_textencder_linebreak:
             text = text.replace('\n', ' BREAK ')
         else:
             text = text.replace('\n', ' ')

modules/prompt_parser_diffusers.py

@@ -354,13 +354,28 @@ def get_prompts_with_weights(pipe, prompt: str):
     if shared.opts.prompt_mean_norm:
         texts_and_weights = normalize_prompt(texts_and_weights)
     texts, text_weights = zip(*texts_and_weights)
-    if debug_enabled:
+    avg_weight = 0
+    min_weight = 1
+    max_weight = 0
+    sections = 0
+
+    try:
         all_tokens = 0
-        for text in texts:
+        for text, weight in zip(texts, text_weights):
             tokens = get_tokens(pipe, 'section', text)
             all_tokens += tokens
-        debug(f'Prompt tokenizer: parser={shared.opts.prompt_attention} tokens={all_tokens}')
+            avg_weight += tokens*weight
+            min_weight = min(min_weight, weight)
+            max_weight = max(max_weight, weight)
+            if text != 'BREAK':
+                sections += 1
+        if all_tokens > 0:
+            avg_weight = avg_weight / all_tokens
+            shared.log.debug(f'Prompt tokenizer: parser={shared.opts.prompt_attention} len={len(prompt)} sections={sections} tokens={all_tokens} weights={min_weight:.2f}/{avg_weight:.2f}/{max_weight:.2f}')
+    except Exception:
+        pass
     debug(f'Prompt: weights={texts_and_weights} time={(time.time() - t0):.3f}')
+
     return texts, text_weights
 
 

modules/shared.py

@@ -499,6 +499,7 @@ options_templates.update(options_section(('text_encoder', "Text Encoder"), {
     "sd_textencoder_cache": OptionInfo(True, "Cache text encoder results", gr.Checkbox, {"visible": False}),
     "sd_textencoder_cache_size": OptionInfo(4, "Text encoder cache size", gr.Slider, {"minimum": 0, "maximum": 16, "step": 1}),
     "comma_padding_backtrack": OptionInfo(20, "Prompt padding", gr.Slider, {"minimum": 0, "maximum": 74, "step": 1, "visible": not native }),
+    "sd_textencder_linebreak": OptionInfo(True, "Use line break as prompt segment marker", gr.Checkbox),
     "diffusers_zeros_prompt_pad": OptionInfo(False, "Use zeros for prompt padding", gr.Checkbox),
     "diffusers_pooled": OptionInfo("default", "Diffusers SDXL pooled embeds", gr.Radio, {"choices": ['default', 'weighted']}),
 }))