dynamic pipelines

CHANGELOG.md

@@ -8,7 +8,6 @@ BLOCKERS:
 OPTIONAL:
 - pending `diffusers==0.26.0`
 - wuerstchen v3 [pr](https://github.com/huggingface/diffusers/pull/6487)
-- tiledvae [pr](https://github.com/huggingface/diffusers/pull/1441)
 - style aligned [pr](https://github.com/huggingface/diffusers/pull/6489)
 - mixture tiling [pr](https://github.com/huggingface/diffusers/tree/main/examples/community#stable-diffusion-mixture-tiling)
 - instaflow [pr](https://github.com/huggingface/diffusers/pull/6057)[repo](https://github.com/gnobitab/RectifiedFlow)
@@ -271,6 +270,10 @@ As of this release, default backend is set to **diffusers** as its more feature
   - extensions `sd-webui-controlnet` is locked to commit `ecd33eb` due to breaking changes  
   - extension `stable-diffusion-webui-images-browser` is locked to commit `27fe4a7` due to breaking changes  
   - updated core requirements  
+  - fully dynamic pipelines  
+    pipeline switch is now done on-the-fly and does not require manual initialization of individual components  
+    this allows for quick implementation of new pipelines  
+    see `modules/sd_models.py:switch_pipe` for details  
   - major internal ui module refactoring  
     this may cause compatibility issues if an extension is doing a direct import from `ui.py`  
     in which case, report it so we can add a compatibility layer  

modules/api/api.py

@@ -10,6 +10,8 @@ from modules.processing import StableDiffusionProcessingTxt2Img, StableDiffusion
 
 
 errors.install()
+decode_base64_to_image = helpers.decode_base64_to_image
+encode_pil_to_base64 = helpers.encode_pil_to_base64
 
 
 class Api:

modules/control/run.py

@@ -13,8 +13,7 @@ from modules.control.units import xs # VisLearn ControlNet-XS
 from modules.control.units import lite # Kohya ControlLLLite
 from modules.control.units import t2iadapter # TencentARC T2I-Adapter
 from modules.control.units import reference # ControlNet-Reference
-from scripts import ipadapter # pylint: disable=no-name-in-module
-from modules import devices, shared, errors, processing, images, sd_models, scripts, masking # pylint: disable=ungrouped-imports
+from modules import devices, shared, errors, processing, images, sd_models, scripts, masking, ipadapter # pylint: disable=ungrouped-imports
 
 
 debug = shared.log.trace if os.environ.get('SD_CONTROL_DEBUG', None) is not None else lambda *args, **kwargs: None

modules/ipadapter.py

@@ -0,0 +1,136 @@
+"""
+Lightweight IP-Adapter applied to existing pipeline in Diffusers
+- Downloads image_encoder or first usage (2.5GB)
+- Introduced via: https://github.com/huggingface/diffusers/pull/5713
+- IP adapters: https://huggingface.co/h94/IP-Adapter
+TODO ipadapter items:
+- SD/SDXL autodetect
+"""
+
+import time
+from modules import processing, shared, devices
+
+
+image_encoder = None
+image_encoder_type = None
+image_encoder_name = None
+loaded = None
+checkpoint = None
+base_repo = "h94/IP-Adapter"
+ADAPTERS = {
+    'None': 'none',
+    'Base': 'ip-adapter_sd15.safetensors',
+    'Base ViT-G': 'ip-adapter_sd15_vit-G.safetensors',
+    'Light': 'ip-adapter_sd15_light.safetensors',
+    'Plus': 'ip-adapter-plus_sd15.safetensors',
+    'Plus Face': 'ip-adapter-plus-face_sd15.safetensors',
+    'Full Face': 'ip-adapter-full-face_sd15.safetensors',
+    'Base SXDL': 'ip-adapter_sdxl.safetensors',
+    'Base ViT-H SXDL': 'ip-adapter_sdxl_vit-h.safetensors',
+    'Plus ViT-H SXDL': 'ip-adapter-plus_sdxl_vit-h.safetensors',
+    'Plus Face ViT-H SXDL': 'ip-adapter-plus-face_sdxl_vit-h.safetensors',
+}
+
+
+def apply(pipe, p: processing.StableDiffusionProcessing, adapter_name='None', scale=1.0, image=None): # pylint: disable=arguments-differ
+    # overrides
+    if hasattr(p, 'ip_adapter_name'):
+        adapter = ADAPTERS.get(p.ip_adapter_name, None)
+        adapter_name = p.ip_adapter_name
+    else:
+        adapter = ADAPTERS.get(adapter_name, None)
+    if hasattr(p, 'ip_adapter_scale'):
+        scale = p.ip_adapter_scale
+    if hasattr(p, 'ip_adapter_image'):
+        image = p.ip_adapter_image
+    if adapter is None:
+        return False
+    # init code
+    global loaded, checkpoint, image_encoder, image_encoder_type, image_encoder_name # pylint: disable=global-statement
+    if pipe is None:
+        return False
+    if shared.backend != shared.Backend.DIFFUSERS:
+        shared.log.warning('IP adapter: not in diffusers mode')
+        return False
+    if image is None and adapter != 'none':
+        shared.log.error('IP adapter: no image provided')
+        adapter = 'none' # unload adapter if previously loaded as it will cause runtime errors
+    if adapter == 'none':
+        if hasattr(pipe, 'set_ip_adapter_scale'):
+            pipe.set_ip_adapter_scale(0)
+        if loaded is not None:
+            loaded = None
+            try:
+                if pipe.unet.config.encoder_hid_dim_type == 'ip_image_proj':
+                    shared.log.debug('IP adapter: unload attention processor')
+                    pipe.unet.config.encoder_hid_dim_type = None
+            except Exception:
+                pass
+        return False
+    if not hasattr(pipe, 'load_ip_adapter'):
+        import diffusers
+        diffusers.StableDiffusionPipeline.load_ip_adapter()
+        shared.log.error(f'IP adapter: pipeline not supported: {pipe.__class__.__name__}')
+        return False
+
+    # which clip to use
+    if 'ViT' not in adapter_name:
+        clip_repo = base_repo
+        subfolder = 'models/image_encoder' if shared.sd_model_type == 'sd' else 'sdxl_models/image_encoder' # defaults per model
+    elif 'ViT-H' in adapter_name:
+        clip_repo = base_repo
+        subfolder = 'models/image_encoder' # this is vit-h
+    elif 'ViT-G' in adapter_name:
+        clip_repo = base_repo
+        subfolder = 'sdxl_models/image_encoder' # this is vit-g
+    else:
+        shared.log.error(f'IP adapter: unknown model type: {adapter_name}')
+        return False
+
+    # load image encoder used by ip adapter
+    if getattr(pipe, 'image_encoder', None) is None or image_encoder_name != clip_repo + '/' + subfolder or image_encoder is None:
+        if image_encoder_type != shared.sd_model_type or checkpoint != shared.opts.sd_model_checkpoint or image_encoder_name != clip_repo + '/' + subfolder:
+            if shared.sd_model_type != 'sd' and shared.sd_model_type != 'sdxl':
+                shared.log.error(f'IP adapter: unsupported model type: {shared.sd_model_type}')
+                return False
+            try:
+                from transformers import CLIPVisionModelWithProjection
+                shared.log.debug(f'IP adapter load: image encoder="{clip_repo}/{subfolder}"')
+                image_encoder = CLIPVisionModelWithProjection.from_pretrained(clip_repo, subfolder=subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.diffusers_dir, use_safetensors=True).to(devices.device)
+                image_encoder_type = shared.sd_model_type
+                image_encoder_name = clip_repo + '/' + subfolder
+            except Exception as e:
+                shared.log.error(f'IP adapter: failed to load image encoder: {e}')
+                return
+    if getattr(pipe, 'feature_extractor', None) is None:
+        from transformers import CLIPImageProcessor
+        shared.log.debug('IP adapter load: feature extractor')
+        pipe.feature_extractor = CLIPImageProcessor()
+
+    # main code
+    # subfolder = 'models' if 'sd15' in adapter else 'sdxl_models'
+    if adapter != loaded or getattr(pipe.unet.config, 'encoder_hid_dim_type', None) is None or checkpoint != shared.opts.sd_model_checkpoint or pipe.image_encoder is None:
+        t0 = time.time()
+        if loaded is not None:
+            shared.log.debug('IP adapter: reset attention processor')
+            loaded = None
+        else:
+            shared.log.debug('IP adapter: load attention processor')
+        pipe.image_encoder = image_encoder
+        subfolder = 'models' if shared.sd_model_type == 'sd' else 'sdxl_models'
+        pipe.load_ip_adapter(base_repo, subfolder=subfolder, weight_name=adapter)
+        t1 = time.time()
+        shared.log.info(f'IP adapter load: adapter="{adapter}" scale={scale} image={image} time={t1-t0:.2f}')
+        loaded = adapter
+        checkpoint = shared.opts.sd_model_checkpoint
+    else:
+        shared.log.debug(f'IP adapter cache: adapter="{adapter}" scale={scale} image={image}')
+    pipe.set_ip_adapter_scale(scale)
+
+    if isinstance(image, str):
+        from modules.api.api import decode_base64_to_image
+        image = decode_base64_to_image(image).convert("RGB")
+
+    p.task_args['ip_adapter_image'] = p.batch_size * [image]
+    p.extra_generation_params["IP Adapter"] = f'{adapter}:{scale}'
+    return True

modules/processing.py

@@ -16,31 +16,15 @@ from skimage import exposure
 from einops import repeat, rearrange
 from blendmodes.blend import blendLayers, BlendType
 from installer import git_commit
-from modules import shared, devices, errors
-import modules.memstats
-import modules.lowvram
-import modules.masking
-import modules.paths
-import modules.scripts
-import modules.script_callbacks
-import modules.prompt_parser
-import modules.extra_networks
-import modules.face_restoration
-import modules.images as images
-import modules.styles
-import modules.sd_hijack_freeu
-import modules.sd_samplers
-import modules.sd_samplers_common
-import modules.sd_models
-import modules.sd_vae
-import modules.sd_vae_approx
-import modules.taesd.sd_vae_taesd
-import modules.generation_parameters_copypaste
+from modules import shared, devices, errors, images, scripts, memstats, lowvram, masking, prompt_parser, script_callbacks, extra_networks, face_restoration, sd_hijack_freeu, sd_samplers, sd_samplers_common, sd_models, sd_vae, generation_parameters_copypaste
+from modules.taesd import sd_vae_taesd
 from modules.sd_hijack_hypertile import context_hypertile_vae, context_hypertile_unet, hypertile_set
 
 
 if shared.backend == shared.Backend.ORIGINAL:
-    import modules.sd_hijack
+    from modules import sd_hijack
+else:
+    sd_hijack = None
 
 opt_C = 4
 opt_f = 8
@@ -127,11 +111,11 @@ def txt2img_image_conditioning(sd_model, x, width, height):
 
 def get_sampler_name(sampler_index: int, img: bool = False) -> str:
     sampler_index = sampler_index or 0
-    if len(modules.sd_samplers.samplers) > sampler_index:
-        sampler_name = modules.sd_samplers.samplers[sampler_index].name
+    if len(sd_samplers.samplers) > sampler_index:
+        sampler_name = sd_samplers.samplers[sampler_index].name
     else:
         sampler_name = "UniPC"
-        shared.log.warning(f'Sampler not found: index={sampler_index} available={[s.name for s in modules.sd_samplers.samplers]} fallback={sampler_name}')
+        shared.log.warning(f'Sampler not found: index={sampler_index} available={[s.name for s in sd_samplers.samplers]} fallback={sampler_name}')
     if img and sampler_name == "PLMS":
         sampler_name = "UniPC"
         shared.log.warning(f'Sampler not compatible: name=PLMS fallback={sampler_name}')
@@ -189,7 +173,7 @@ class StableDiffusionProcessing:
         self.disable_extra_networks = False
         self.token_merging_ratio = 0
         self.token_merging_ratio_hr = 0
-        # self.scripts = modules.scripts.ScriptRunner() # set via property
+        # self.scripts = scripts.ScriptRunner() # set via property
         # self.script_args = script_args or [] # set via property
         self.per_script_args = {}
         self.all_prompts = None
@@ -237,7 +221,7 @@ class StableDiffusionProcessing:
         self.all_hr_negative_prompts = []
         self.comments = {}
         self.is_api = False
-        self.scripts_value: modules.scripts.ScriptRunner = field(default=None, init=False)
+        self.scripts_value: scripts.ScriptRunner = field(default=None, init=False)
         self.script_args_value: list = field(default=None, init=False)
         self.scripts_setup_complete: bool = field(default=False, init=False)
         # hdr
@@ -561,7 +545,7 @@ def decode_first_stage(model, x, full_quality=True):
             else:
                 x_sample = torch.zeros((len(x), 3, x.shape[2] * 8, x.shape[3] * 8), dtype=devices.dtype_vae, device=devices.device)
                 for i in range(len(x_sample)):
-                    x_sample[i] = modules.taesd.sd_vae_taesd.decode(x[i])
+                    x_sample[i] = sd_vae_taesd.decode(x[i])
         except Exception as e:
             x_sample = x
             shared.log.error(f'Decode VAE: {e}')
@@ -616,7 +600,7 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No
         "Parser": shared.opts.prompt_attention,
         "Model": None if (not shared.opts.add_model_name_to_info) or (not shared.sd_model.sd_checkpoint_info.model_name) else shared.sd_model.sd_checkpoint_info.model_name.replace(',', '').replace(':', ''),
         "Model hash": getattr(p, 'sd_model_hash', None if (not shared.opts.add_model_hash_to_info) or (not shared.sd_model.sd_model_hash) else shared.sd_model.sd_model_hash),
-        "VAE": (None if not shared.opts.add_model_name_to_info or modules.sd_vae.loaded_vae_file is None else os.path.splitext(os.path.basename(modules.sd_vae.loaded_vae_file))[0]) if p.full_quality else 'TAESD',
+        "VAE": (None if not shared.opts.add_model_name_to_info or sd_vae.loaded_vae_file is None else os.path.splitext(os.path.basename(sd_vae.loaded_vae_file))[0]) if p.full_quality else 'TAESD',
         "Seed resize from": None if p.seed_resize_from_w == 0 or p.seed_resize_from_h == 0 else f"{p.seed_resize_from_w}x{p.seed_resize_from_h}",
         "Clip skip": p.clip_skip if p.clip_skip > 1 else None,
         "Prompt2": p.refiner_prompt if len(p.refiner_prompt) > 0 else None,
@@ -673,10 +657,10 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No
     if 'color' in p.ops:
         args["Color correction"] = True
     # embeddings
-    if hasattr(modules.sd_hijack.model_hijack, 'embedding_db') and len(modules.sd_hijack.model_hijack.embedding_db.embeddings_used) > 0: # this is for original hijaacked models only, diffusers are handled separately
-        args["Embeddings"] = ', '.join(modules.sd_hijack.model_hijack.embedding_db.embeddings_used)
+    if sd_hijack is not None and hasattr(sd_hijack.model_hijack, 'embedding_db') and len(sd_hijack.model_hijack.embedding_db.embeddings_used) > 0: # this is for original hijaacked models only, diffusers are handled separately
+        args["Embeddings"] = ', '.join(sd_hijack.model_hijack.embedding_db.embeddings_used)
     # samplers
-    args["Sampler ENSD"] = shared.opts.eta_noise_seed_delta if shared.opts.eta_noise_seed_delta != 0 and modules.sd_samplers_common.is_sampler_using_eta_noise_seed_delta(p) else None
+    args["Sampler ENSD"] = shared.opts.eta_noise_seed_delta if shared.opts.eta_noise_seed_delta != 0 and sd_samplers_common.is_sampler_using_eta_noise_seed_delta(p) else None
     args["Sampler ENSM"] = p.initial_noise_multiplier if getattr(p, 'initial_noise_multiplier', 1.0) != 1.0 else None
     args['Sampler order'] = shared.opts.schedulers_solver_order if shared.opts.schedulers_solver_order != shared.opts.data_labels.get('schedulers_solver_order').default else None
     if shared.backend == shared.Backend.DIFFUSERS:
@@ -705,7 +689,7 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No
     args['ToMe hires'] = token_merging_ratio_hr if token_merging_ratio_hr != 0 else None
 
     args.update(p.extra_generation_params)
-    params_text = ", ".join([k if k == v else f'{k}: {modules.generation_parameters_copypaste.quote(v)}' for k, v in args.items() if v is not None])
+    params_text = ", ".join([k if k == v else f'{k}: {generation_parameters_copypaste.quote(v)}' for k, v in args.items() if v is not None])
     negative_prompt_text = f"\nNegative prompt: {all_negative_prompts[index]}" if all_negative_prompts[index] else ""
     infotext = f"{all_prompts[index]}{negative_prompt_text}\n{params_text}".strip()
     return infotext
@@ -715,7 +699,7 @@ def process_images(p: StableDiffusionProcessing) -> Processed:
     debug(f'Process images: {vars(p)}')
     if not hasattr(p.sd_model, 'sd_checkpoint_info'):
         return None
-    if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner):
+    if p.scripts is not None and isinstance(p.scripts, scripts.ScriptRunner):
         p.scripts.before_process(p)
     stored_opts = {}
     for k, v in p.override_settings.copy().items():
@@ -729,14 +713,14 @@ def process_images(p: StableDiffusionProcessing) -> Processed:
     res = None
     try:
         # if no checkpoint override or the override checkpoint can't be found, remove override entry and load opts checkpoint
-        if p.override_settings.get('sd_model_checkpoint', None) is not None and modules.sd_models.checkpoint_aliases.get(p.override_settings.get('sd_model_checkpoint')) is None:
+        if p.override_settings.get('sd_model_checkpoint', None) is not None and sd_models.checkpoint_aliases.get(p.override_settings.get('sd_model_checkpoint')) is None:
             shared.log.warning(f"Override not found: checkpoint={p.override_settings.get('sd_model_checkpoint', None)}")
             p.override_settings.pop('sd_model_checkpoint', None)
-            modules.sd_models.reload_model_weights()
-        if p.override_settings.get('sd_model_refiner', None) is not None and modules.sd_models.checkpoint_aliases.get(p.override_settings.get('sd_model_refiner')) is None:
+            sd_models.reload_model_weights()
+        if p.override_settings.get('sd_model_refiner', None) is not None and sd_models.checkpoint_aliases.get(p.override_settings.get('sd_model_refiner')) is None:
             shared.log.warning(f"Override not found: refiner={p.override_settings.get('sd_model_refiner', None)}")
             p.override_settings.pop('sd_model_refiner', None)
-            modules.sd_models.reload_model_weights()
+            sd_models.reload_model_weights()
         if p.override_settings.get('sd_vae', None) is not None:
             if p.override_settings.get('sd_vae', None) == 'TAESD':
                 p.full_quality = False
@@ -748,16 +732,16 @@ def process_images(p: StableDiffusionProcessing) -> Processed:
         for k, v in p.override_settings.items():
             setattr(shared.opts, k, v)
             if k == 'sd_model_checkpoint':
-                modules.sd_models.reload_model_weights()
+                sd_models.reload_model_weights()
             if k == 'sd_vae':
-                modules.sd_vae.reload_vae_weights()
+                sd_vae.reload_vae_weights()
 
         shared.prompt_styles.apply_styles_to_extra(p)
         if not shared.opts.cuda_compile:
-            modules.sd_models.apply_token_merging(p.sd_model, p.get_token_merging_ratio())
-            modules.sd_hijack_freeu.apply_freeu(p, shared.backend == shared.Backend.ORIGINAL)
+            sd_models.apply_token_merging(p.sd_model, p.get_token_merging_ratio())
+            sd_hijack_freeu.apply_freeu(p, shared.backend == shared.Backend.ORIGINAL)
 
-        modules.script_callbacks.before_process_callback(p)
+        script_callbacks.before_process_callback(p)
 
         if shared.cmd_opts.profile:
             import cProfile
@@ -782,17 +766,17 @@ def process_images(p: StableDiffusionProcessing) -> Processed:
 
     finally:
         if not shared.opts.cuda_compile:
-            modules.sd_models.apply_token_merging(p.sd_model, 0)
-        modules.script_callbacks.after_process_callback(p)
+            sd_models.apply_token_merging(p.sd_model, 0)
+        script_callbacks.after_process_callback(p)
         if p.override_settings_restore_afterwards: # restore opts to original state
             for k, v in stored_opts.items():
                 setattr(shared.opts, k, v)
                 if k == 'sd_model_checkpoint':
-                    modules.sd_models.reload_model_weights()
+                    sd_models.reload_model_weights()
                 if k == 'sd_model_refiner':
-                    modules.sd_models.reload_model_weights()
+                    sd_models.reload_model_weights()
                 if k == 'sd_vae':
-                    modules.sd_vae.reload_vae_weights()
+                    sd_vae.reload_vae_weights()
     return res
 
 
@@ -850,6 +834,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
         assert p.prompt is not None
 
     if shared.backend == shared.Backend.ORIGINAL:
+        import modules.sd_hijack # pylint: disable=redefined-outer-name
         modules.sd_hijack.model_hijack.apply_circular(p.tiling)
         modules.sd_hijack.model_hijack.clear_comments()
     comments = {}
@@ -861,11 +846,11 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
     process_init(p)
     if os.path.exists(shared.opts.embeddings_dir) and not p.do_not_reload_embeddings and shared.backend == shared.Backend.ORIGINAL:
         modules.sd_hijack.model_hijack.embedding_db.load_textual_inversion_embeddings(force_reload=False)
-    if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner):
+    if p.scripts is not None and isinstance(p.scripts, scripts.ScriptRunner):
         p.scripts.process(p)
 
     if shared.backend == shared.Backend.DIFFUSERS:
-        from scripts import ipadapter # pylint: disable=no-name-in-module
+        from modules import ipadapter
         ipadapter.apply(shared.sd_model, p)
 
     def get_conds_with_caching(function, required_prompts, steps, cache):
@@ -900,23 +885,23 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
             p.negative_prompts = p.all_negative_prompts[n * p.batch_size:(n + 1) * p.batch_size]
             p.seeds = p.all_seeds[n * p.batch_size:(n + 1) * p.batch_size]
             p.subseeds = p.all_subseeds[n * p.batch_size:(n + 1) * p.batch_size]
-            if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner):
+            if p.scripts is not None and isinstance(p.scripts, scripts.ScriptRunner):
                 p.scripts.before_process_batch(p, batch_number=n, prompts=p.prompts, seeds=p.seeds, subseeds=p.subseeds)
             if len(p.prompts) == 0:
                 break
-            p.prompts, extra_network_data = modules.extra_networks.parse_prompts(p.prompts)
+            p.prompts, extra_network_data = extra_networks.parse_prompts(p.prompts)
             if not p.disable_extra_networks:
                 with devices.autocast():
-                    modules.extra_networks.activate(p, extra_network_data)
-            if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner):
+                    extra_networks.activate(p, extra_network_data)
+            if p.scripts is not None and isinstance(p.scripts, scripts.ScriptRunner):
                 p.scripts.process_batch(p, batch_number=n, prompts=p.prompts, seeds=p.seeds, subseeds=p.subseeds)
             step_multiplier = 1
-            sampler_config = modules.sd_samplers.find_sampler_config(p.sampler_name)
+            sampler_config = sd_samplers.find_sampler_config(p.sampler_name)
             step_multiplier = 2 if sampler_config and sampler_config.options.get("second_order", False) else 1
 
             if shared.backend == shared.Backend.ORIGINAL:
-                uc = get_conds_with_caching(modules.prompt_parser.get_learned_conditioning, p.negative_prompts, p.steps * step_multiplier, cached_uc)
-                c = get_conds_with_caching(modules.prompt_parser.get_multicond_learned_conditioning, p.prompts, p.steps * step_multiplier, cached_c)
+                uc = get_conds_with_caching(prompt_parser.get_learned_conditioning, p.negative_prompts, p.steps * step_multiplier, cached_uc)
+                c = get_conds_with_caching(prompt_parser.get_multicond_learned_conditioning, p.prompts, p.steps * step_multiplier, cached_c)
                 if len(modules.sd_hijack.model_hijack.comments) > 0:
                     for comment in modules.sd_hijack.model_hijack.comments:
                         comments[comment] = 1
@@ -931,7 +916,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
                         shared.log.warning('Tensor with all NaNs was produced in VAE')
                         devices.dtype_vae = torch.bfloat16
                         vae_file, vae_source = modules.sd_vae.resolve_vae(p.sd_model.sd_model_checkpoint)
-                        modules.sd_vae.load_vae(p.sd_model, vae_file, vae_source)
+                        sd_vae.load_vae(p.sd_model, vae_file, vae_source)
                         x_samples_ddim = [decode_first_stage(p.sd_model, samples_ddim[i:i+1].to(dtype=devices.dtype_vae), p.full_quality)[0].cpu() for i in range(samples_ddim.size(0))]
                         for x in x_samples_ddim:
                             devices.test_for_nans(x, "vae")
@@ -951,14 +936,14 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
                 x_samples_ddim = []
 
             if shared.cmd_opts.lowvram or shared.cmd_opts.medvram and shared.backend == shared.Backend.ORIGINAL:
-                modules.lowvram.send_everything_to_cpu()
+                lowvram.send_everything_to_cpu()
                 devices.torch_gc()
-            if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner):
+            if p.scripts is not None and isinstance(p.scripts, scripts.ScriptRunner):
                 p.scripts.postprocess_batch(p, x_samples_ddim, batch_number=n)
-            if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner):
+            if p.scripts is not None and isinstance(p.scripts, scripts.ScriptRunner):
                 p.prompts = p.all_prompts[n * p.batch_size:(n + 1) * p.batch_size]
                 p.negative_prompts = p.all_negative_prompts[n * p.batch_size:(n + 1) * p.batch_size]
-                batch_params = modules.scripts.PostprocessBatchListArgs(list(x_samples_ddim))
+                batch_params = scripts.PostprocessBatchListArgs(list(x_samples_ddim))
                 p.scripts.postprocess_batch_list(p, batch_params, batch_number=n)
                 x_samples_ddim = batch_params.images
 
@@ -981,10 +966,10 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
                         p.restore_faces = orig
                         images.save_image(Image.fromarray(x_sample), path=p.outpath_samples, basename="", seed=p.seeds[i], prompt=p.prompts[i], extension=shared.opts.samples_format, info=info, p=p, suffix="-before-face-restore")
                     p.ops.append('face')
-                    x_sample = modules.face_restoration.restore_faces(x_sample)
+                    x_sample = face_restoration.restore_faces(x_sample)
                     image = Image.fromarray(x_sample)
-                if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner):
-                    pp = modules.scripts.PostprocessImageArgs(image)
+                if p.scripts is not None and isinstance(p.scripts, scripts.ScriptRunner):
+                    pp = scripts.PostprocessImageArgs(image)
                     p.scripts.postprocess_image(p, pp)
                     image = pp.image
                 if p.color_corrections is not None and i < len(p.color_corrections):
@@ -1019,7 +1004,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
             devices.torch_gc()
 
         t1 = time.time()
-        shared.log.info(f'Processed: images={len(output_images)} time={t1 - t0:.2f} its={(p.steps * len(output_images)) / (t1 - t0):.2f} memory={modules.memstats.memory_stats()}')
+        shared.log.info(f'Processed: images={len(output_images)} time={t1 - t0:.2f} its={(p.steps * len(output_images)) / (t1 - t0):.2f} memory={memstats.memory_stats()}')
 
         p.color_corrections = None
         index_of_first_image = 0
@@ -1036,7 +1021,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
                     images.save_image(grid, p.outpath_grids, "", p.all_seeds[0], p.all_prompts[0], shared.opts.grid_format, info=infotext(-1), p=p, grid=True, suffix="-grid") # main save grid
 
     if not p.disable_extra_networks:
-        modules.extra_networks.deactivate(p, extra_network_data)
+        extra_networks.deactivate(p, extra_network_data)
 
     res = Processed(
         p,
@@ -1048,7 +1033,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
         index_of_first_image=index_of_first_image,
         infotexts=infotexts,
     )
-    if p.scripts is not None and isinstance(p.scripts, modules.scripts.ScriptRunner) and not (shared.state.interrupted or shared.state.skipped):
+    if p.scripts is not None and isinstance(p.scripts, scripts.ScriptRunner) and not (shared.state.interrupted or shared.state.skipped):
         p.scripts.postprocess(p, res)
     return res
 
@@ -1108,7 +1093,7 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
 
     def init(self, all_prompts, all_seeds, all_subseeds):
         if shared.backend == shared.Backend.DIFFUSERS:
-            shared.sd_model = modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.TEXT_2_IMAGE)
+            shared.sd_model = sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
         self.width = self.width or 512
         self.height = self.height or 512
 
@@ -1157,7 +1142,7 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
 
         self.ops.append('txt2img')
         hypertile_set(self)
-        self.sampler = modules.sd_samplers.create_sampler(self.sampler_name, self.sd_model)
+        self.sampler = sd_samplers.create_sampler(self.sampler_name, self.sd_model)
         if hasattr(self.sampler, "initialize"):
             self.sampler.initialize(self)
         x = create_random_tensors([4, self.height // 8, self.width // 8], seeds=seeds, subseeds=subseeds, subseed_strength=self.subseed_strength, seed_resize_from_h=self.seed_resize_from_h, seed_resize_from_w=self.seed_resize_from_w, p=self)
@@ -1219,15 +1204,15 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
                 if self.denoising_strength > 0:
                     self.ops.append('hires')
                     devices.torch_gc() # GC now before running the next img2img to prevent running out of memory
-                    self.sampler = modules.sd_samplers.create_sampler(self.hr_sampler_name or self.sampler_name, self.sd_model)
+                    self.sampler = sd_samplers.create_sampler(self.hr_sampler_name or self.sampler_name, self.sd_model)
                     if hasattr(self.sampler, "initialize"):
                         self.sampler.initialize(self)
                     samples = samples[:, :, self.truncate_y//2:samples.shape[2]-(self.truncate_y+1)//2, self.truncate_x//2:samples.shape[3]-(self.truncate_x+1)//2]
                     noise = create_random_tensors(samples.shape[1:], seeds=seeds, subseeds=subseeds, subseed_strength=subseed_strength, p=self)
-                    modules.sd_models.apply_token_merging(self.sd_model, self.get_token_merging_ratio(for_hr=True))
+                    sd_models.apply_token_merging(self.sd_model, self.get_token_merging_ratio(for_hr=True))
                     hypertile_set(self, hr=True)
                     samples = self.sampler.sample_img2img(self, samples, noise, conditioning, unconditional_conditioning, steps=self.hr_second_pass_steps or self.steps, image_conditioning=image_conditioning)
-                    modules.sd_models.apply_token_merging(self.sd_model, self.get_token_merging_ratio())
+                    sd_models.apply_token_merging(self.sd_model, self.get_token_merging_ratio())
                 else:
                     self.ops.append('upscale')
             x = None
@@ -1275,13 +1260,13 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
 
     def init(self, all_prompts, all_seeds, all_subseeds):
         if shared.backend == shared.Backend.DIFFUSERS and self.image_mask is not None and not self.is_control:
-            shared.sd_model = modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.INPAINTING)
+            shared.sd_model = sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.INPAINTING)
         elif shared.backend == shared.Backend.DIFFUSERS and self.image_mask is None and not self.is_control:
-            shared.sd_model = modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.IMAGE_2_IMAGE)
+            shared.sd_model = sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE)
 
         if self.sampler_name == "PLMS":
             self.sampler_name = 'UniPC'
-        self.sampler = modules.sd_samplers.create_sampler(self.sampler_name, self.sd_model)
+        self.sampler = sd_samplers.create_sampler(self.sampler_name, self.sd_model)
         if hasattr(self.sampler, "initialize"):
             self.sampler.initialize(self)
 
@@ -1306,7 +1291,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
                     self.image_mask = Image.fromarray(np_mask)
             else:
                 if hasattr(self, 'init_images'):
-                    self.image_mask = modules.masking.run_mask(
+                    self.image_mask = masking.run_mask(
                         input_image=self.init_images,
                         input_mask=self.image_mask,
                         return_type='Grayscale',
@@ -1318,8 +1303,8 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
             if self.inpaint_full_res: # mask only inpaint
                 self.mask_for_overlay = self.image_mask
                 mask = self.image_mask.convert('L')
-                crop_region = modules.masking.get_crop_region(np.array(mask), self.inpaint_full_res_padding)
-                crop_region = modules.masking.expand_crop_region(crop_region, self.width, self.height, mask.width, mask.height)
+                crop_region = masking.get_crop_region(np.array(mask), self.inpaint_full_res_padding)
+                crop_region = masking.expand_crop_region(crop_region, self.width, self.height, mask.width, mask.height)
                 x1, y1, x2, y2 = crop_region
                 crop_mask = mask.crop(crop_region)
                 self.image_mask = images.resize_image(2, crop_mask, self.width, self.height)
@@ -1373,7 +1358,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
                 if image.width != self.width or image.height != self.height:
                     image = images.resize_image(3, image, self.width, self.height, self.resize_name)
             if self.image_mask is not None and self.inpainting_fill != 1:
-                image = modules.masking.fill(image, latent_mask)
+                image = masking.fill(image, latent_mask)
             if add_color_corrections:
                 self.color_corrections.append(setup_color_correction(image))
             processed.append(image)

modules/processing_diffusers.py

@@ -397,7 +397,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
             update_sampler(shared.sd_model)
             supported = ['DDIMScheduler', 'PNDMScheduler', 'DDPMScheduler', 'DEISMultistepScheduler', 'UniPCMultistepScheduler', 'DPMSolverMultistepScheduler', 'DPMSolverSinlgestepScheduler']
             if sd_model.scheduler.__class__.__name__ in supported:
-                sd_model = sd_models.switch_diffuser_pipe(sd_model, diffusers.StableDiffusionSAGPipeline)
+                sd_model = sd_models.switch_pipe(sd_model, diffusers.StableDiffusionSAGPipeline)
                 p.extra_generation_params["SAG scale"] = p.sag_scale
                 p.task_args['sag_scale'] = p.sag_scale
             else:
@@ -640,6 +640,9 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
     # final decode since there is no refiner
     if not is_refiner_enabled():
         if output is not None:
+            if isinstance(output, dict):
+                from types import SimpleNamespace
+                output = SimpleNamespace(**output)
             if not hasattr(output, 'images') and hasattr(output, 'frames'):
                 shared.log.debug(f'Generated: frames={len(output.frames[0])}')
                 output.images = output.frames[0]

modules/sd_models.py

@@ -4,6 +4,7 @@ import sys
 import json
 import time
 import copy
+import inspect
 import logging
 import contextlib
 import collections
@@ -722,7 +723,6 @@ def set_diffuser_options(sd_model, vae = None, op: str = 'model'):
 
 
 def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=None, op='model'): # pylint: disable=unused-argument
-    import torch # pylint: disable=reimported,redefined-outer-name
     if shared.cmd_opts.profile:
         import cProfile
         pr = cProfile.Profile()
@@ -973,12 +973,40 @@ def get_diffusers_task(pipe: diffusers.DiffusionPipeline) -> DiffusersTaskType:
         return DiffusersTaskType.TEXT_2_IMAGE
 
 
-def switch_diffuser_pipe(pipeline, cls):
+def switch_pipe(cls: diffusers.DiffusionPipeline, pipeline: diffusers.DiffusionPipeline = None, args = {}): # noqa:B006
+    """
+    args:
+    - cls: can be pipeline class or a string from custom pipelines
+      for example: diffusers.StableDiffusionPipeline or 'mixture_tiling'
+    - pipeline: source model to be used, if not provided currently loaded model is used
+    - args: any additional components to load into the pipeline
+      for example: { 'vae': None }
+    """
     try:
+        if isinstance(cls, str):
+            shared.log.debug(f'Pipeline switch: custom={cls}')
+            cls = diffusers.utils.get_class_from_dynamic_module(cls, module_file='pipeline.py')
+        if pipeline is None:
+            pipeline = shared.sd_model
         new_pipe = None
+        signature = inspect.signature(cls.__init__, follow_wrapped=True, eval_str=True)
+        possible = signature.parameters.keys()
         if isinstance(pipeline, cls):
             return pipeline
-        elif isinstance(pipeline, diffusers.StableDiffusionXLPipeline):
+        pipe_dict = {}
+        components_used = []
+        components_skipped = []
+        switch_mode = 'none'
+        if hasattr(pipeline, '_internal_dict'):
+            for item in pipeline._internal_dict.keys(): # pylint: disable=protected-access
+                if item in possible:
+                    pipe_dict[item] = getattr(pipeline, item, None)
+                    components_used.append(item)
+                else:
+                    components_skipped.append(item)
+            new_pipe = cls(**pipe_dict)
+            switch_mode = 'auto'
+        elif 'tokenizer_2' in possible and hasattr(pipeline, 'tokenizer_2'):
             new_pipe = cls(
                 vae=pipeline.vae,
                 text_encoder=pipeline.text_encoder,
@@ -989,7 +1017,8 @@ def switch_diffuser_pipe(pipeline, cls):
                 scheduler=pipeline.scheduler,
                 feature_extractor=getattr(pipeline, 'feature_extractor', None),
             ).to(pipeline.device)
-        elif isinstance(pipeline, diffusers.StableDiffusionPipeline):
+            switch_mode = 'sdxl'
+        elif 'tokenizer' in possible and hasattr(pipeline, 'tokenizer'):
             new_pipe = cls(
                 vae=pipeline.vae,
                 text_encoder=pipeline.text_encoder,
@@ -1000,17 +1029,30 @@ def switch_diffuser_pipe(pipeline, cls):
                 requires_safety_checker=False,
                 safety_checker=None,
             ).to(pipeline.device)
+            switch_mode = 'sd'
         else:
             shared.log.error(f'Pipeline switch error: {pipeline.__class__.__name__} unrecognized')
             return pipeline
+        if new_pipe is not None:
+            for k, v in args.items():
+                if k in possible:
+                    setattr(new_pipe, k, v)
+                    components_used.append(k)
+                else:
+                    shared.log.warning(f'Pipeline switch skipping unknown: component={k}')
+                    components_skipped.append(k)
         if new_pipe is not None:
             copy_diffuser_options(new_pipe, pipeline)
-            shared.log.debug(f'Pipeline switch: from={pipeline.__class__.__name__} to={new_pipe.__class__.__name__}')
+            if switch_mode == 'auto':
+                shared.log.debug(f'Pipeline switch: from={pipeline.__class__.__name__} to={new_pipe.__class__.__name__} components={components_used} skipped={components_skipped}')
+            else:
+                shared.log.debug(f'Pipeline switch: from={pipeline.__class__.__name__} to={new_pipe.__class__.__name__} mode={switch_mode}')
             return new_pipe
         else:
             shared.log.error(f'Pipeline switch error: from={pipeline.__class__.__name__} to={cls.__name__} empty pipeline')
     except Exception as e:
         shared.log.error(f'Pipeline switch error: from={pipeline.__class__.__name__} to={cls.__name__} {e}')
+        errors.display(e, 'Pipeline switch')
     return pipeline
 
 

modules/ui_control.py

@@ -10,7 +10,7 @@ from modules.control.units import xs # vislearn ControlNet-XS
 from modules.control.units import lite # vislearn ControlNet-XS
 from modules.control.units import t2iadapter # TencentARC T2I-Adapter
 from modules.control.units import reference # reference pipeline
-from modules import errors, shared, progress, sd_samplers, ui_components, ui_symbols, ui_common, ui_sections, generation_parameters_copypaste, call_queue, scripts, masking # pylint: disable=ungrouped-imports
+from modules import errors, shared, progress, sd_samplers, ui_components, ui_symbols, ui_common, ui_sections, generation_parameters_copypaste, call_queue, scripts, masking, ipadapter # pylint: disable=ungrouped-imports
 
 
 gr_height = None
@@ -437,8 +437,7 @@ def create_ui(_blocks: gr.Blocks=None):
                     with gr.Row():
                         with gr.Column():
                             gr.HTML('<a href="https://github.com/tencent-ailab/IP-Adapter">IP-Adapter</a>')
-                            from scripts.ipadapter import ADAPTERS # pylint: disable=no-name-in-module
-                            ip_adapter = gr.Dropdown(label='Adapter', choices=ADAPTERS, value='none')
+                            ip_adapter_name = gr.Dropdown(label='Adapter', choices=ipadapter.ADAPTERS, value='none')
                             ip_scale = gr.Slider(label='Scale', minimum=0.0, maximum=1.0, step=0.01, value=0.5)
                         with gr.Column():
                             ip_image = gr.Image(label="Input", show_label=False, type="pil", source="upload", interactive=True, tool="editor", height=256, width=256)
@@ -695,7 +694,7 @@ def create_ui(_blocks: gr.Blocks=None):
                     resize_mode_after, resize_name_after, width_after, height_after, scale_by_after, selected_scale_tab_after,
                     denoising_strength, batch_count, batch_size,
                     video_skip_frames, video_type, video_duration, video_loop, video_pad, video_interpolate,
-                    ip_adapter, ip_scale, ip_image,
+                    ip_adapter_name, ip_scale, ip_image,
                 ]
                 output_fields = [
                     preview_process,

scripts/ipadapter.py

@@ -1,140 +1,5 @@
-"""
-Lightweight IP-Adapter applied to existing pipeline in Diffusers
-- Downloads image_encoder or first usage (2.5GB)
-- Introduced via: https://github.com/huggingface/diffusers/pull/5713
-- IP adapters: https://huggingface.co/h94/IP-Adapter
-TODO ipadapter items:
-- SD/SDXL autodetect
-"""
-
-import time
 import gradio as gr
-from modules import scripts, processing, shared, devices
-
-
-image_encoder = None
-image_encoder_type = None
-image_encoder_name = None
-loaded = None
-checkpoint = None
-base_repo = "h94/IP-Adapter"
-ADAPTERS = {
-    'None': 'none',
-    'Base': 'ip-adapter_sd15.safetensors',
-    'Base ViT-G': 'ip-adapter_sd15_vit-G.safetensors',
-    'Light': 'ip-adapter_sd15_light.safetensors',
-    'Plus': 'ip-adapter-plus_sd15.safetensors',
-    'Plus Face': 'ip-adapter-plus-face_sd15.safetensors',
-    'Full Face': 'ip-adapter-full-face_sd15.safetensors',
-    'Base SXDL': 'ip-adapter_sdxl.safetensors',
-    'Base ViT-H SXDL': 'ip-adapter_sdxl_vit-h.safetensors',
-    'Plus ViT-H SXDL': 'ip-adapter-plus_sdxl_vit-h.safetensors',
-    'Plus Face ViT-H SXDL': 'ip-adapter-plus-face_sdxl_vit-h.safetensors',
-}
-
-
-def apply(pipe, p: processing.StableDiffusionProcessing, adapter_name='None', scale=1.0, image=None): # pylint: disable=arguments-differ
-    # overrides
-    if hasattr(p, 'ip_adapter_name'):
-        adapter = ADAPTERS.get(p.ip_adapter_name, None)
-        adapter_name = p.ip_adapter_name
-    else:
-        adapter = ADAPTERS.get(adapter_name, None)
-    if hasattr(p, 'ip_adapter_scale'):
-        scale = p.ip_adapter_scale
-    if hasattr(p, 'ip_adapter_image'):
-        image = p.ip_adapter_image
-    if adapter is None:
-        return False
-    # init code
-    global loaded, checkpoint, image_encoder, image_encoder_type, image_encoder_name # pylint: disable=global-statement
-    if pipe is None:
-        return False
-    if shared.backend != shared.Backend.DIFFUSERS:
-        shared.log.warning('IP adapter: not in diffusers mode')
-        return False
-    if image is None and adapter != 'none':
-        shared.log.error('IP adapter: no image provided')
-        adapter = 'none' # unload adapter if previously loaded as it will cause runtime errors
-    if adapter == 'none':
-        if hasattr(pipe, 'set_ip_adapter_scale'):
-            pipe.set_ip_adapter_scale(0)
-        if loaded is not None:
-            loaded = None
-            try:
-                if pipe.unet.config.encoder_hid_dim_type == 'ip_image_proj':
-                    shared.log.debug('IP adapter: unload attention processor')
-                    pipe.unet.config.encoder_hid_dim_type = None
-            except Exception:
-                pass
-        return False
-    if not hasattr(pipe, 'load_ip_adapter'):
-        import diffusers
-        diffusers.StableDiffusionPipeline.load_ip_adapter()
-        shared.log.error(f'IP adapter: pipeline not supported: {pipe.__class__.__name__}')
-        return False
-
-    # which clip to use
-    if 'ViT' not in adapter_name:
-        clip_repo = base_repo
-        subfolder = 'models/image_encoder' if shared.sd_model_type == 'sd' else 'sdxl_models/image_encoder' # defaults per model
-    elif 'ViT-H' in adapter_name:
-        clip_repo = base_repo
-        subfolder = 'models/image_encoder' # this is vit-h
-    elif 'ViT-G' in adapter_name:
-        clip_repo = base_repo
-        subfolder = 'sdxl_models/image_encoder' # this is vit-g
-    else:
-        shared.log.error(f'IP adapter: unknown model type: {adapter_name}')
-        return False
-
-    # load image encoder used by ip adapter
-    if getattr(pipe, 'image_encoder', None) is None or image_encoder_name != clip_repo + '/' + subfolder or image_encoder is None:
-        if image_encoder_type != shared.sd_model_type or checkpoint != shared.opts.sd_model_checkpoint or image_encoder_name != clip_repo + '/' + subfolder:
-            if shared.sd_model_type != 'sd' and shared.sd_model_type != 'sdxl':
-                shared.log.error(f'IP adapter: unsupported model type: {shared.sd_model_type}')
-                return False
-            try:
-                from transformers import CLIPVisionModelWithProjection
-                shared.log.debug(f'IP adapter load: image encoder="{clip_repo}/{subfolder}"')
-                image_encoder = CLIPVisionModelWithProjection.from_pretrained(clip_repo, subfolder=subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.diffusers_dir, use_safetensors=True).to(devices.device)
-                image_encoder_type = shared.sd_model_type
-                image_encoder_name = clip_repo + '/' + subfolder
-            except Exception as e:
-                shared.log.error(f'IP adapter: failed to load image encoder: {e}')
-                return
-    if getattr(pipe, 'feature_extractor', None) is None:
-        from transformers import CLIPImageProcessor
-        shared.log.debug('IP adapter load: feature extractor')
-        pipe.feature_extractor = CLIPImageProcessor()
-
-    # main code
-    # subfolder = 'models' if 'sd15' in adapter else 'sdxl_models'
-    if adapter != loaded or getattr(pipe.unet.config, 'encoder_hid_dim_type', None) is None or checkpoint != shared.opts.sd_model_checkpoint or pipe.image_encoder is None:
-        t0 = time.time()
-        if loaded is not None:
-            shared.log.debug('IP adapter: reset attention processor')
-            loaded = None
-        else:
-            shared.log.debug('IP adapter: load attention processor')
-        pipe.image_encoder = image_encoder
-        subfolder = 'models' if shared.sd_model_type == 'sd' else 'sdxl_models'
-        pipe.load_ip_adapter(base_repo, subfolder=subfolder, weight_name=adapter)
-        t1 = time.time()
-        shared.log.info(f'IP adapter load: adapter="{adapter}" scale={scale} image={image} time={t1-t0:.2f}')
-        loaded = adapter
-        checkpoint = shared.opts.sd_model_checkpoint
-    else:
-        shared.log.debug(f'IP adapter cache: adapter="{adapter}" scale={scale} image={image}')
-    pipe.set_ip_adapter_scale(scale)
-
-    if isinstance(image, str):
-        from modules.api.api import decode_base64_to_image
-        image = decode_base64_to_image(image).convert("RGB")
-
-    p.task_args['ip_adapter_image'] = p.batch_size * [image]
-    p.extra_generation_params["IP Adapter"] = f'{adapter}:{scale}'
-    return True
+from modules import scripts, processing, shared, ipadapter
 
 
 class Script(scripts.Script):
@@ -147,7 +12,7 @@ class Script(scripts.Script):
     def ui(self, _is_img2img):
         with gr.Accordion('IP Adapter', open=False, elem_id='ipadapter'):
             with gr.Row():
-                adapter = gr.Dropdown(label='Adapter', choices=list(ADAPTERS), value='none')
+                adapter = gr.Dropdown(label='Adapter', choices=list(ipadapter.ADAPTERS), value='none')
                 scale = gr.Slider(label='Scale', minimum=0.0, maximum=1.0, step=0.01, value=0.5)
             with gr.Row():
                 image = gr.Image(image_mode='RGB', label='Image', source='upload', type='pil', width=512)