diffusers better model autodetect

2026-09-19 01:04:32 +02:00 · 2023-09-22 10:28:10 -04:00
parent 4518cfb3b1
commit 4e7ad0e15a
7 changed files with 70 additions and 63 deletions
@@ -2,17 +2,24 @@

 ## Update for 2023-09-20

- Added **change log** to UI, see *System -> Changelog*  
+- Added **change log** to UI  
+  see *System -> Changelog*  
 - **Extra networks**:  
  - faster search, ability to show/hide/sort networks  
  - refactored subfolder handling  
- **Upscalers**: complete refactor...  
+    *note*: this will trigger model hash recaclulation on first model use  
+- **Upscalers**:
  - more high quality upscalers available by default  
  - unified init/download/execute/progress code  
  - easier installation  
  - available in **xyz grid**  
  - allow upscale-only as part of **txt2img** and **img2img** workflows  
    simply set *denoising strength* to 0 so hires does not get triggered  
+- **Samplers**:
+  - default list for new installs is now all samplers, list can be modified in settings  
+  - simplified samplers configuration in settings
+- **Diffusers**
+  - better pipeline auto-detect when loading from safetensors  

 ## Update for 2023-09-13

@@ -1045,7 +1045,7 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
                else:
                    image_conditioning = self.txt2img_image_conditioning(samples.to(dtype=devices.dtype_vae))
                if self.latent_sampler == "PLMS":
-                   self.latent_sampler = 'UniPC'
+                    self.latent_sampler = 'UniPC'
            if self.hr_force or latent_scale_mode is not None:
                if self.denoising_strength > 0:
                    self.ops.append('hires')
@@ -353,7 +353,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
        **task_specific_kwargs
    )
    p.extra_generation_params['CFG rescale'] = p.diffusers_guidance_rescale
-    p.extra_generation_params["Eta"] = shared.opts.scheduler_eta if shared.opts.scheduler_eta is not None and shared.opts.scheduler_eta > 0 else None
+    p.extra_generation_params["Eta"] = shared.opts.scheduler_eta if shared.opts.scheduler_eta is not None and shared.opts.scheduler_eta > 0 and shared.opts.scheduler_eta < 1 else None
    try:
        output = shared.sd_model(**base_args) # pylint: disable=not-callable
    except AssertionError as e:
@@ -19,7 +19,7 @@ import tomesd
 from transformers import logging as transformers_logging
 import ldm.modules.midas as midas
 from ldm.util import instantiate_from_config
-from modules import paths, shared, modelloader, devices, script_callbacks, sd_vae, sd_disable_initialization, errors, hashes, sd_models_config
+from modules import paths, shared, shared_items, modelloader, devices, script_callbacks, sd_vae, sd_disable_initialization, errors, hashes, sd_models_config
 from modules.sd_hijack_inpainting import do_inpainting_hijack
 from modules.timer import Timer
 from modules.memstats import memory_stats
@@ -42,6 +42,7 @@ sd_metadata = None
 sd_metadata_pending = 0
 sd_metadata_timer = 0

+
 class CheckpointInfo:
    def __init__(self, filename):
        self.name = None
@@ -70,7 +71,7 @@ class CheckpointInfo:
                self.sha256 = None
                self.type = 'unknown'
            else:
-                self.name = repo[0]['name']
+                self.name = os.path.join(os.path.basename(shared.opts.diffusers_dir), repo[0]['name'])
                self.filename = repo[0]['path']
                self.sha256 = repo[0]['hash']
                self.type = 'diffusers'
@@ -598,58 +599,49 @@ def detect_pipeline(f: str, op: str = 'model'):
    guess = shared.opts.diffusers_pipeline
    if guess == 'Autodetect':
        try:
-            size = round(os.path.getsize(f) / 1024 / 1024 / 1024, 2)
-            if size < 1:
-                shared.log.warning(f'Model size smaller than expected: {f} size={size} GB')
-            elif size < 5.5: # maximum size of sd1.5 fp32 unpruned is 5.3GB
-                guess = 'Stable Diffusion'
-            elif size < 6: # sdxl refiner is 5.7gb
+            size = round(os.path.getsize(f) / 1024 / 1024)
+            if size < 128:
+                shared.log.warning(f'Model size smaller than expected: {f} size={size} MB')
+            elif size >= 331 and size <= 339: # 335
+                shared.log.warning(f'Model detected as VAE model, but attempting to load as model: {op}={f} size={size} MB')
+                guess = 'VAE'
+            elif size >= 5351 and size <= 5359: # 5353
+                guess = 'Stable Diffusion' # SD v2
+            elif size >= 5791 and size <= 5799: # 5795
                if shared.backend == shared.Backend.ORIGINAL:
-                    shared.log.warning(f'Model detected as SD-XL refiner model, but attempting to load using backend=original: {f} size={size} GB')
+                    shared.log.warning(f'Model detected as SD-XL refiner model, but attempting to load using backend=original: {op}={f} size={size} MB')
                if op == 'model':
-                    shared.log.warning(f'Model detected as SD-XL refiner model, but attempting to load a base model: {f} size={size} GB')
+                    shared.log.warning(f'Model detected as SD-XL refiner model, but attempting to load a base model: {op}={f} size={size} MB')
                guess = 'Stable Diffusion XL'
-            elif size < 7:
+            elif size >= 6611 and size <= 6619: # 6617
                if shared.backend == shared.Backend.ORIGINAL:
-                    shared.log.warning(f'Model detected as SD-XL base model, but attempting to load using backend=original: {f} size={size} GB')
+                    shared.log.warning(f'Model detected as SD-XL base model, but attempting to load using backend=original: {op}={f} size={size} MB')
                guess = 'Stable Diffusion XL'
+            elif size >= 3361 and size <= 3369: # 3368
+                if shared.backend == shared.Backend.ORIGINAL:
+                    shared.log.warning(f'Model detected as SD upscale model, but attempting to load using backend=original: {op}={f} size={size} MB')
+                guess = 'Stable Diffusion Upscale'
+            elif size >= 4891 and size <= 4899: # 4897
+                if shared.backend == shared.Backend.ORIGINAL:
+                    shared.log.warning(f'Model detected as SD XL inpaint model, but attempting to load using backend=original: {op}={f} size={size} MB')
+                guess = 'Stable Diffusion XL Inpaint'
+            elif size >= 9791 and size <= 9799: # 9794
+                if shared.backend == shared.Backend.ORIGINAL:
+                    shared.log.warning(f'Model detected as SD XL instruct pix2pix model, but attempting to load using backend=original: {op}={f} size={size} MB')
+                guess = 'Stable Diffusion XL Instruct'
            else:
-                guess = 'Unknown'
-                shared.log.error(f'Model autodetect failed, set diffuser pipeline manually: {f}')
-                return None, None
-            shared.log.debug(f'Model autodetect: {op}="{f}" pipeline="{guess}" size={size} GB')
+                guess = 'Stable Diffusion'
+            pipeline = shared_items.get_pipelines().get(guess, None)
+            shared.log.info(f'Autodetect: {op}="{guess}" class={pipeline.__name__} file="{f}" size={size}MB')
        except Exception as e:
            shared.log.error(f'Error detecting diffusers pipeline: model={f} {e}')
            return None, None
-    if guess == shared.pipelines[1]:
+    if pipeline is None:
+        shared.log.warning(f'Autodetect: pipeline not recognized: {guess}: {op}={f} size={size}')
        pipeline = diffusers.StableDiffusionPipeline
-    elif guess == shared.pipelines[2]:
-        pipeline = diffusers.StableDiffusionXLPipeline
-    elif guess == shared.pipelines[3]:
-        pipeline = diffusers.KandinskyPipeline
-    elif guess == shared.pipelines[4]:
-        pipeline = diffusers.KandinskyV22Pipeline
-    elif guess == shared.pipelines[5]:
-        pipeline = diffusers.IFPipeline
-    elif guess == shared.pipelines[6]:
-        pipeline = diffusers.ShapEPipeline
-    elif guess == shared.pipelines[7]:
-        pipeline = diffusers.StableDiffusionImg2ImgPipeline
-    elif guess == shared.pipelines[8]:
-        pipeline = diffusers.StableDiffusionXLImg2ImgPipeline
-    elif guess == shared.pipelines[9]:
-        pipeline = diffusers.KandinskyImg2ImgPipeline
-    elif guess == shared.pipelines[10]:
-        pipeline = diffusers.KandinskyV22Img2ImgPipeline
-    elif guess == shared.pipelines[11]:
-        pipeline = diffusers.IFImg2ImgPipeline
-    elif guess == shared.pipelines[12]:
-        pipeline = diffusers.ShapEImg2ImgPipeline
-    else:
-        shared.log.error(f'Diffusers unknown pipeline: {guess}')
-        pipeline = None, None
    return pipeline, guess

+
 def compile_diffusers(sd_model):
    try:
        if shared.opts.ipex_optimize:
@@ -758,20 +750,14 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
            if vae is not None:
                diffusers_load_config["vae"] = vae

-        # shared.log.info(f'Loading diffuser {op}: {checkpoint_info.filename}')
-        if not os.path.isfile(checkpoint_info.path):
+        if os.path.isdir(checkpoint_info.path):
            try:
-                # os.environ.setdefault('HUGGINGFACE_HUB_CACHE', shared.opts.diffusers_dir) # evalulated only on initial diffusers load
-                # diffusers_load_config["cache_dir "] = shared.opts.diffusers_dir # ignored for connected pipelines such as kandinsky-prior
-                # diffusers.utils.constants.DIFFUSERS_CACHE = shared.opts.diffusers_dir
-                # shared.log.debug(f'Diffusers load {op} config: {diffusers_load_config}')
-                # sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, **diffusers_load_config)
                sd_model = diffusers.AutoPipelineForText2Image.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config)
                sd_model.model_type = sd_model.__class__.__name__
            except Exception as e:
                shared.log.error(f'Failed loading {op}: {checkpoint_info.path} {e}')
                return
-        else:
+        elif os.path.isfile(checkpoint_info.path) and checkpoint_info.path.lower().endswith('.safetensors'):
            diffusers_load_config["local_files_only"] = True
            diffusers_load_config["extract_ema"] = shared.opts.diffusers_extract_ema
            pipeline, model_type = detect_pipeline(checkpoint_info.path, op)
@@ -805,8 +791,11 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
                    diffusers_load_config.pop('local_files_only', None)
                    shared.log.debug(f'Setting {op}: pipeline={sd_model.__class__.__name__} config={diffusers_load_config}') # pylint: disable=protected-access
            except Exception as e:
-                shared.log.error(f'Diffusers failed loading model using pipeline: {checkpoint_info.path} {shared.opts.diffusers_pipeline} {e}')
+                shared.log.error(f'Diffusers failed loading: {op}={checkpoint_info.path} pipeline={shared.opts.diffusers_pipeline}/{sd_model.__class__.__name__} {e}')
                return
+        else:
+            shared.log.error(f'Diffusers cannot load: {op}={checkpoint_info.path}')
+            return

        if "StableDiffusion" in sd_model.__class__.__name__:
            pass # scheduler is created on first use
@@ -43,11 +43,6 @@ hypernetworks = {}
 loaded_hypernetworks = []
 gradio_theme = gr.themes.Base()
 settings_components = None
-pipelines = [
-    'Autodetect',
-    'Stable Diffusion', 'Stable Diffusion XL', 'Kandinsky V1', 'Kandinsky V2', 'DeepFloyd IF', 'Shap-E',
-    'Stable Diffusion Img2Img', 'Stable Diffusion XL Img2Img', 'Kandinsky V1 Img2Img', 'Kandinsky V2 Img2Img', 'DeepFloyd IF Img2Img', 'Shap-E Img2Img'
-]
 latent_upscale_default_mode = "None"
 latent_upscale_modes = {
    "Latent": {"mode": "bilinear", "antialias": False},
@@ -431,7 +426,7 @@ options_templates.update(options_section(('cuda', "Compute Settings"), {
 }))

 options_templates.update(options_section(('diffusers', "Diffusers Settings"), {
-    "diffusers_pipeline": OptionInfo(pipelines[0], 'Diffusers pipeline', gr.Dropdown, lambda: {"choices": pipelines}),
+    "diffusers_pipeline": OptionInfo('Autodetect', 'Diffusers pipeline', gr.Dropdown, lambda: {"choices": list(shared_items.get_pipelines()) }),
    "diffusers_move_base": OptionInfo(True, "Move base model to CPU when using refiner"),
    "diffusers_move_unet": OptionInfo(True, "Move base model to CPU when using VAE"),
    "diffusers_move_refiner": OptionInfo(True, "Move refiner model to CPU when not in use"),
@@ -23,3 +23,18 @@ def list_crossattention():
        "Sub-quadratic",
        "Split attention"
    ]
+
+def get_pipelines():
+    import diffusers
+    return {
+        'Autodetect': None,
+        'Stable Diffusion': diffusers.StableDiffusionPipeline,
+        'Stable Diffusion Img2Img': diffusers.StableDiffusionImg2ImgPipeline,
+        'Stable Diffusion Instruct': diffusers.StableDiffusionInstructPix2PixPipeline,
+        'Stable Diffusion Upscale': diffusers.StableDiffusionUpscalePipeline,
+        'Stable Diffusion XL': diffusers.StableDiffusionXLPipeline,
+        'Stable Diffusion XL Img2Img': diffusers.StableDiffusionXLImg2ImgPipeline,
+        'Stable Diffusion XL Inpaint': diffusers.StableDiffusionXLInpaintPipeline,
+        'Stable Diffusion XL Instruct': diffusers.StableDiffusionXLInstructPix2PixPipeline,
+        # 'Kandinsky V1', 'Kandinsky V2', 'DeepFloyd IF', 'Shap-E', 'Kandinsky V1 Img2Img', 'Kandinsky V2 Img2Img', 'DeepFloyd IF Img2Img', 'Shap-E Img2Img',
+    }
@@ -991,9 +991,10 @@ def create_ui(startup_timer = None):
                modules.shared.opts.sd_backend = "diffusers"
        try:
            opts.save(modules.shared.config_filename)
-            log.info(f'Settings changed: {len(changed)} {changed}')
+            if len(changed) > 0:
+                log.info(f'Settings: changed={len(changed)} {changed}')
        except RuntimeError:
-            log.error(f'Settings change failed: {len(changed)} {changed}')
+            log.error(f'Settings failed: change={len(changed)} {changed}')
            return opts.dumpjson(), f'{len(changed)} Settings changed without save: {", ".join(changed)}'
        return opts.dumpjson(), f'{len(changed)} Settings changed{": " if len(changed) > 0 else ""}{", ".join(changed)}'