Merge pull request #4584 from awsr/processing-updates

Processing updates

modules/extra_networks.py

@@ -151,33 +151,30 @@ def deactivate(p, extra_network_data=None, force=shared.opts.lora_force_reload):
 re_extra_net = re.compile(r"<(\w+):([^>]+)>")
 
 
-def parse_prompt(prompt):
-    res = defaultdict(list)
+def parse_prompt(prompt: str | None) -> tuple[str, defaultdict[str, list[ExtraNetworkParams]]]:
+    res: defaultdict[str, list[ExtraNetworkParams]] = defaultdict(list)
     if prompt is None:
-        return prompt, res
+        return "", res
+    if isinstance(prompt, list):
+        shared.log.warning("parse_prompt was called with a list instead of a string", prompt)
+        return parse_prompts(prompt)
 
-    def found(m):
-        name = m.group(1)
-        args = m.group(2)
+    def found(m: re.Match[str]):
+        name, args = m.group(1, 2)
         res[name].append(ExtraNetworkParams(items=args.split(":")))
         return ""
-    if isinstance(prompt, list):
-        prompt = [re.sub(re_extra_net, found, p) for p in prompt]
-    else:
-        prompt = re.sub(re_extra_net, found, prompt)
-    return prompt, res
+
+    updated_prompt = re.sub(re_extra_net, found, prompt)
+    return updated_prompt, res
 
 
-def parse_prompts(prompts):
-    res = []
-    extra_data = None
-    if prompts is None:
-        return prompts, extra_data
-
+def parse_prompts(prompts: list[str]):
+    updated_prompt_list: list[str] = []
+    extra_data: defaultdict[str, list[ExtraNetworkParams]] = defaultdict(list)
     for prompt in prompts:
         updated_prompt, parsed_extra_data = parse_prompt(prompt)
-        if extra_data is None:
+        if not extra_data:
             extra_data = parsed_extra_data
-        res.append(updated_prompt)
+        updated_prompt_list.append(updated_prompt)
 
-    return res, extra_data
+    return updated_prompt_list, extra_data

modules/face/faceid.py

@@ -205,7 +205,7 @@ def face_id(
             ip_model_dict["faceid_embeds"] = face_embeds # overwrite placeholder
             faceid_model.set_scale(scale)
 
-            if p.all_prompts is None or len(p.all_prompts) == 0:
+            if not p.all_prompts:
                 processing.process_init(p)
                 p.init(p.all_prompts, p.all_seeds, p.all_subseeds)
             for n in range(p.n_iter):

modules/face/instantid.py

@@ -63,7 +63,7 @@ def instant_id(p: processing.StableDiffusionProcessing, app, source_images, stre
     sd_models.move_model(shared.sd_model, devices.device) # move pipeline to device
 
     # pipeline specific args
-    if p.all_prompts is None or len(p.all_prompts) == 0:
+    if not p.all_prompts:
         processing.process_init(p)
         p.init(p.all_prompts, p.all_seeds, p.all_subseeds)
     orig_prompt_attention = shared.opts.prompt_attention
@@ -73,8 +73,8 @@ def instant_id(p: processing.StableDiffusionProcessing, app, source_images, stre
     p.task_args['controlnet_conditioning_scale'] = float(conditioning)
     p.task_args['ip_adapter_scale'] = float(strength)
     shared.log.debug(f"InstantID args: {p.task_args}")
-    p.task_args['prompt'] = p.all_prompts[0] if p.all_prompts is not None else p.prompt
-    p.task_args['negative_prompt'] = p.all_negative_prompts[0] if p.all_negative_prompts is not None else p.negative_prompt
+    p.task_args['prompt'] = p.all_prompts[0] if p.all_prompts else p.prompt
+    p.task_args['negative_prompt'] = p.all_negative_prompts[0] if p.all_negative_prompts else p.negative_prompt
     p.task_args['image_embeds'] = face_embeds[0] # overwrite placeholder
 
     # run processing

modules/face/photomaker.py

@@ -34,7 +34,7 @@ def photo_maker(p: processing.StableDiffusionProcessing, app, model: str, input_
         return None
 
     # validate prompt
-    if p.all_prompts is None or len(p.all_prompts) == 0:
+    if not p.all_prompts:
         processing.process_init(p)
         p.init(p.all_prompts, p.all_seeds, p.all_subseeds)
     trigger_ids = shared.sd_model.tokenizer.encode(trigger) + shared.sd_model.tokenizer_2.encode(trigger)
@@ -61,7 +61,7 @@ def photo_maker(p: processing.StableDiffusionProcessing, app, model: str, input_
     shared.opts.data['prompt_attention'] = 'fixed' # otherwise need to deal with class_tokens_mask
     p.task_args['input_id_images'] = input_images
     p.task_args['start_merge_step'] = int(start * p.steps)
-    p.task_args['prompt'] = p.all_prompts[0] if p.all_prompts is not None else p.prompt
+    p.task_args['prompt'] = p.all_prompts[0] if p.all_prompts else p.prompt
 
     is_v2 = 'v2' in model
     if is_v2:

modules/images.py

@@ -311,7 +311,7 @@ def parse_novelai_metadata(data: dict):
     return geninfo
 
 
-def read_info_from_image(image: Image.Image, watermark: bool = False):
+def read_info_from_image(image: Image.Image, watermark: bool = False) -> tuple[str, dict]:
     if image is None:
         return '', {}
     if isinstance(image, str):
@@ -322,9 +322,11 @@ def read_info_from_image(image: Image.Image, watermark: bool = False):
             return '', {}
     items = image.info or {}
     geninfo = items.pop('parameters', None) or items.pop('UserComment', None) or ''
-    if geninfo is not None and len(geninfo) > 0:
+    if isinstance(geninfo, dict):
         if 'UserComment' in geninfo:
-            geninfo = geninfo['UserComment']
+            geninfo = geninfo['UserComment'] # Info was nested
+        else:
+            geninfo = '' # Unknown format. Ignore contents
         items['UserComment'] = geninfo
 
     if "exif" in items:
@@ -342,7 +344,7 @@ def read_info_from_image(image: Image.Image, watermark: bool = False):
                         val = round(val[0] / val[1], 2)
                     if val is not None and key in ExifTags.TAGS: # add known tags
                         if ExifTags.TAGS[key] == 'UserComment': # add geninfo from UserComment
-                            geninfo = val
+                            geninfo = str(val)
                             items['parameters'] = val
                         else:
                             items[ExifTags.TAGS[key]] = val

modules/ipadapter.py

@@ -5,14 +5,18 @@ Lightweight IP-Adapter applied to existing pipeline in Diffusers
 - IP adapters: https://huggingface.co/h94/IP-Adapter
 """
 
+from __future__ import annotations
 import os
 import time
 import json
+from typing import TYPE_CHECKING
 from PIL import Image
-import diffusers
 import transformers
 from modules import processing, shared, devices, sd_models, errors, model_quant
 
+if TYPE_CHECKING:
+    from diffusers import DiffusionPipeline
+
 
 clip_loaded = None
 adapters_loaded = []
@@ -160,7 +164,7 @@ def unapply(pipe, unload: bool = False): # pylint: disable=arguments-differ
         pass
 
 
-def load_image_encoder(pipe: diffusers.DiffusionPipeline, adapter_names: list[str]):
+def load_image_encoder(pipe: DiffusionPipeline, adapter_names: list[str]):
     global clip_loaded # pylint: disable=global-statement
     for adapter_name in adapter_names:
         # which clip to use

modules/pag/__init__.py

@@ -15,7 +15,7 @@ def apply(p: processing.StableDiffusionProcessing): # pylint: disable=arguments-
         cls = unapply()
     if p.pag_scale == 0:
         return
-    if 'PAG' in cls.__name__:
+    if cls is not None and 'PAG' in cls.__name__:
         pass
     elif detect.is_sd15(cls):
         if sd_models.get_diffusers_task(shared.sd_model) != sd_models.DiffusersTaskType.TEXT_2_IMAGE:

modules/processing.py

@@ -243,13 +243,13 @@ def process_init(p: StableDiffusionProcessing):
     seed = get_fixed_seed(p.seed)
     subseed = get_fixed_seed(p.subseed)
     reset_prompts = False
-    if p.all_prompts is None:
+    if not p.all_prompts:
         p.all_prompts = p.prompt if isinstance(p.prompt, list) else p.batch_size * p.n_iter * [p.prompt]
         reset_prompts = True
-    if p.all_negative_prompts is None:
+    if not p.all_negative_prompts:
         p.all_negative_prompts = p.negative_prompt if isinstance(p.negative_prompt, list) else p.batch_size * p.n_iter * [p.negative_prompt]
         reset_prompts = True
-    if p.all_seeds is None:
+    if not p.all_seeds:
         reset_prompts = True
         if type(seed) == list:
             p.all_seeds = [int(s) for s in seed]
@@ -262,7 +262,7 @@ def process_init(p: StableDiffusionProcessing):
                     for i in range(len(p.all_prompts)):
                         seed = get_fixed_seed(p.seed)
                         p.all_seeds.append(int(seed) + (i if p.subseed_strength == 0 else 0))
-    if p.all_subseeds is None:
+    if not p.all_subseeds:
         if type(subseed) == list:
             p.all_subseeds = [int(s) for s in subseed]
         else:
@@ -270,8 +270,8 @@ def process_init(p: StableDiffusionProcessing):
     if reset_prompts:
         if not hasattr(p, 'keep_prompts'):
             p.all_prompts, p.all_negative_prompts = shared.prompt_styles.apply_styles_to_prompts(p.all_prompts, p.all_negative_prompts, p.styles, p.all_seeds)
-            p.prompts = p.all_prompts[p.iteration * p.batch_size:(p.iteration+1) * p.batch_size]
-            p.negative_prompts = p.all_negative_prompts[p.iteration * p.batch_size:(p.iteration+1) * p.batch_size]
+            p.prompts = p.all_prompts[(p.iteration * p.batch_size):((p.iteration+1) * p.batch_size)]
+            p.negative_prompts = p.all_negative_prompts[(p.iteration * p.batch_size):((p.iteration+1) * p.batch_size)]
         p.prompts, _ = extra_networks.parse_prompts(p.prompts)
 
 
@@ -427,13 +427,13 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
                 continue
 
             if not hasattr(p, 'keep_prompts'):
-                p.prompts = p.all_prompts[n * p.batch_size:(n+1) * p.batch_size]
-                p.negative_prompts = p.all_negative_prompts[n * p.batch_size:(n+1) * p.batch_size]
-            p.seeds = p.all_seeds[n * p.batch_size:(n+1) * p.batch_size]
-            p.subseeds = p.all_subseeds[n * p.batch_size:(n+1) * p.batch_size]
+                p.prompts = p.all_prompts[(n * p.batch_size):((n+1) * p.batch_size)]
+                p.negative_prompts = p.all_negative_prompts[(n * p.batch_size):((n+1) * p.batch_size)]
+            p.seeds = p.all_seeds[(n * p.batch_size):((n+1) * p.batch_size)]
+            p.subseeds = p.all_subseeds[(n * p.batch_size):((n+1) * p.batch_size)]
             if p.scripts is not None and isinstance(p.scripts, scripts_manager.ScriptRunner):
                 p.scripts.before_process_batch(p, batch_number=n, prompts=p.prompts, seeds=p.seeds, subseeds=p.subseeds)
-            if len(p.prompts) == 0:
+            if not p.prompts:
                 break
             p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts)
             if p.scripts is not None and isinstance(p.scripts, scripts_manager.ScriptRunner):
@@ -469,8 +469,8 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
             if p.scripts is not None and isinstance(p.scripts, scripts_manager.ScriptRunner):
                 p.scripts.postprocess_batch(p, samples, batch_number=n)
             if p.scripts is not None and isinstance(p.scripts, scripts_manager.ScriptRunner):
-                p.prompts = p.all_prompts[n * p.batch_size:(n+1) * p.batch_size]
-                p.negative_prompts = p.all_negative_prompts[n * p.batch_size:(n+1) * p.batch_size]
+                p.prompts = p.all_prompts[(n * p.batch_size):((n+1) * p.batch_size)]
+                p.negative_prompts = p.all_negative_prompts[(n * p.batch_size):((n+1) * p.batch_size)]
                 batch_params = scripts_manager.PostprocessBatchListArgs(list(samples))
                 p.scripts.postprocess_batch_list(p, batch_params, batch_number=n)
                 samples = batch_params.images

modules/processing_class.py

@@ -308,15 +308,14 @@ class StableDiffusionProcessing:
             shared.log.error(f'Override: {override_settings} {e}')
             self.override_settings = {}
 
-        # null items initialized later
-        self.prompts = None
-        self.negative_prompts = None
-        self.all_prompts = None
-        self.all_negative_prompts = None
+        self.prompts = []
+        self.negative_prompts = []
+        self.all_prompts = []
+        self.all_negative_prompts = []
         self.seeds = []
         self.subseeds = []
-        self.all_seeds = None
-        self.all_subseeds = None
+        self.all_seeds = []
+        self.all_subseeds = []
 
         # a1111 compatibility items
         self.seed_enable_extras: bool = True

modules/processing_diffusers.py

@@ -563,9 +563,9 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
         shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.INPAINTING) # force pipeline
         if len(getattr(p, 'init_images', [])) == 0:
             p.init_images = [TF.to_pil_image(torch.rand((3, getattr(p, 'height', 512), getattr(p, 'width', 512))))]
-    if p.prompts is None or len(p.prompts) == 0:
+    if not p.prompts:
         p.prompts = p.all_prompts[p.iteration * p.batch_size:(p.iteration+1) * p.batch_size]
-    if p.negative_prompts is None or len(p.negative_prompts) == 0:
+    if not p.negative_prompts:
         p.negative_prompts = p.all_negative_prompts[p.iteration * p.batch_size:(p.iteration+1) * p.batch_size]
 
     sd_models_compile.openvino_recompile_model(p, hires=False, refiner=False) # recompile if a parameter changes

modules/sd_models.py

@@ -964,7 +964,7 @@ def get_diffusers_task(pipe: diffusers.DiffusionPipeline) -> DiffusersTaskType:
         return DiffusersTaskType.TEXT_2_IMAGE
 
 
-def switch_pipe(cls: diffusers.DiffusionPipeline, pipeline: diffusers.DiffusionPipeline = None, force = False, args: dict = None):
+def switch_pipe(cls: type[diffusers.DiffusionPipeline] | str, pipeline: diffusers.DiffusionPipeline | None = None, force = False, args: dict | None = None):
     """
     args:
     - cls: can be pipeline class or a string from custom pipelines
@@ -978,13 +978,22 @@ def switch_pipe(cls: diffusers.DiffusionPipeline, pipeline: diffusers.DiffusionP
             args = {}
         if isinstance(cls, str):
             shared.log.debug(f'Pipeline switch: custom={cls}')
-            cls = diffusers.utils.get_class_from_dynamic_module(cls, module_file='pipeline.py')
+            cls_object = diffusers.utils.get_class_from_dynamic_module(cls, module_file='pipeline.py')
+            if not cls_object:
+                log.error(f"Pipeline switch: Failed to get class for '{cls}'")
+                if shared.sd_model is not None:
+                    return shared.sd_model
+                raise RuntimeError("Pipeline switch: No existing pipeline to fall back to")
+        else:
+            cls_object = cls
         if pipeline is None:
+            if shared.sd_model is None:
+                raise RuntimeError("Pipeline switch: No existing pipeline to use as default")
             pipeline = shared.sd_model
         new_pipe = None
-        signature = get_signature(cls)
+        signature = get_signature(cls_object)
         possible = signature.keys()
-        if not force and isinstance(pipeline, cls) and args == {}:
+        if not force and isinstance(pipeline, cls_object) and args == {}:
             return pipeline
         pipe_dict = {}
         components_used = []
@@ -1007,10 +1016,10 @@ def switch_pipe(cls: diffusers.DiffusionPipeline, pipeline: diffusers.DiffusionP
                     shared.log.warning(f'Pipeling switch: missing component={item} type={signature[item].annotation}')
                     pipe_dict[item] = None # try but not likely to work
                     components_missing.append(item)
-            new_pipe = cls(**pipe_dict)
+            new_pipe = cls_object(**pipe_dict)
             switch_mode = 'auto'
         elif 'tokenizer_2' in possible and hasattr(pipeline, 'tokenizer_2'):
-            new_pipe = cls(
+            new_pipe = cls_object(
                 vae=pipeline.vae,
                 text_encoder=pipeline.text_encoder,
                 text_encoder_2=pipeline.text_encoder_2,
@@ -1023,7 +1032,7 @@ def switch_pipe(cls: diffusers.DiffusionPipeline, pipeline: diffusers.DiffusionP
             move_model(new_pipe, pipeline.device)
             switch_mode = 'sdxl'
         elif 'tokenizer' in possible and hasattr(pipeline, 'tokenizer'):
-            new_pipe = cls(
+            new_pipe = cls_object(
                 vae=pipeline.vae,
                 text_encoder=pipeline.text_encoder,
                 tokenizer=pipeline.tokenizer,
@@ -1057,9 +1066,9 @@ def switch_pipe(cls: diffusers.DiffusionPipeline, pipeline: diffusers.DiffusionP
                 shared.log.debug(f'Pipeline switch: from={pipeline.__class__.__name__} to={new_pipe.__class__.__name__} mode={switch_mode}')
             return new_pipe
         else:
-            shared.log.error(f'Pipeline switch error: from={pipeline.__class__.__name__} to={cls.__name__} empty pipeline')
+            shared.log.error(f'Pipeline switch error: from={pipeline.__class__.__name__} to={cls_object.__name__} empty pipeline')
     except Exception as e:
-        shared.log.error(f'Pipeline switch error: from={pipeline.__class__.__name__} to={cls.__name__} {e}')
+        shared.log.error(f'Pipeline switch error: from={pipeline.__class__.__name__} to={cls if isinstance(cls, str) else cls.__name__} {e}')
         errors.display(e, 'Pipeline switch')
     return pipeline
 

modules/ui_common.py

@@ -427,7 +427,10 @@ def update_token_counter(text):
         shared.log.debug('Tokenizer busy')
         return f"<span class='gr-box gr-text-input'>{token_count}/{max_length}</span>"
     from modules import extra_networks
-    prompt, _ = extra_networks.parse_prompt(text)
+    if isinstance(text, list):
+        prompt, _ = extra_networks.parse_prompts(text)
+    else:
+        prompt, _ = extra_networks.parse_prompt(text)
     if shared.sd_loaded and hasattr(shared.sd_model, 'tokenizer') and shared.sd_model.tokenizer is not None:
         tokenizer = shared.sd_model.tokenizer
         # For multi-modal processors (e.g., PixtralProcessor), use the underlying text tokenizer