update ip-adapter, schedulers and xyz-grid

CHANGELOG.md

@@ -1,17 +1,21 @@
 # Change Log for SD.Next
 
-## Update for 2023-12-06
+## Update for 2023-12-07
 
 *Note*: based on `diffusers==0.25.0.dev0`
 
 - **Diffusers**
   - **AnimateDiff** can now be used with *second pass* - enhance, upscale and hires your videos!  
-  - **IP Adapter** add support for `ip-adapter-plus_sd15` and `ip-adapter-plus-face_sd15`  
+  - **IP Adapter** add support for `ip-adapter-plus_sd15`, `ip-adapter-plus-face_sd15` and `ip-adapter-full-face_sd15`  
+    additionally, ip-adapter can now be used in xyz-grid  
   - **HDR controls** are now batch-aware for enhancement of multiple images or video frames  
   - [Playground v1](https://huggingface.co/playgroundai/playground-v1), [Playground v2 256](https://huggingface.co/playgroundai/playground-v2-256px-base), [Playground v2 512](https://huggingface.co/playgroundai/playground-v2-512px-base), [Playground v2 1024](https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic) model support  
     - simply select from *networks -> reference* and use as usual  
   - [ModelScope T2V](https://huggingface.co/damo-vilab/text-to-video-ms-1.7b) model support  
     - simply select from *networks -> reference* and use from *txt2img* tab
+  - **Schedulers**
+    - add timesteps range, changing it will make scheduler to be over-complete or under-complete  
+    - add rescale betas with zero SNR option (applicable to Euler and DDIM, allows for higher dynamic range)  
 - **General**
   - **LoRA** add support for block weights, thanks @AI-Casanova  
     example `<lora:SDXL_LCM_LoRA:1.0:in=0:mid=1:out=0>`  

README.md

@@ -211,6 +211,9 @@ General goals:
 
 ### **Docs**
 
+If you're unsure how to use a feature, best place to start is [Wiki](https://github.com/vladmandic/automatic/wiki) and if its not there,  
+check [ChangeLog](CHANGELOG.md) for when feature was first introduced as it will always have a short note on how to use it  
+
 - [Wiki](https://github.com/vladmandic/automatic/wiki)
 - [ReadMe](README.md)
 - [ToDo](TODO.md)  

modules/sd_samplers_diffusers.py

@@ -26,7 +26,7 @@ except Exception as e:
 config = {
     # beta_start, beta_end are typically per-scheduler, but we don't want them as they should be taken from the model itself as those are values model was trained on
     # prediction_type is ideally set in model as well, but it maybe needed that we do auto-detect of model type in the future
-    'All': { 'num_train_timesteps': 1000, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'linear', 'prediction_type': 'epsilon' },
+    'All': { 'num_train_timesteps': 500, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'linear', 'prediction_type': 'epsilon' },
     'DDIM': { 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'clip_sample_range': 1.0, 'sample_max_value': 1.0, 'timestep_spacing': 'linspace', 'rescale_betas_zero_snr': False },
     'DDPM': { 'variance_type': "fixed_small", 'clip_sample': True, 'thresholding': False, 'clip_sample_range': 1.0, 'sample_max_value': 1.0, 'timestep_spacing': 'linspace'},
     'DEIS': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "deis", 'solver_type': "logrho", 'lower_order_final': True },
@@ -34,14 +34,14 @@ config = {
     'DPM++ 2M': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False },
     'DPM SDE': { 'use_karras_sigmas': False },
     'Euler a': { },
-    'Euler': { 'interpolation_type': "linear", 'use_karras_sigmas': False },
+    'Euler': { 'interpolation_type': "linear", 'use_karras_sigmas': False, 'rescale_betas_zero_snr': False },
     'Heun': { 'use_karras_sigmas': False },
     'KDPM2': { 'steps_offset': 0 },
     'KDPM2 a': { 'steps_offset': 0 },
     'LMSD': { 'use_karras_sigmas': False, 'timestep_spacing': 'linspace', 'steps_offset': 0 },
     'PNDM': { 'skip_prk_steps': False, 'set_alpha_to_one': False, 'steps_offset': 0 },
     'UniPC': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'predict_x0': 'bh2', 'lower_order_final': True },
-    'LCM': { 'num_train_timesteps': 1000, 'beta_start': 0.00085, 'beta_end': 0.012, 'beta_schedule': "scaled_linear", 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False },
+    'LCM': { 'beta_start': 0.00085, 'beta_end': 0.012, 'beta_schedule': "scaled_linear", 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False },
 }
 
 samplers_data_diffusers = [
@@ -108,6 +108,10 @@ class DiffusionSampler:
             self.config['beta_start'] = shared.opts.schedulers_beta_start
         if 'beta_end' in self.config and shared.opts.schedulers_beta_end > 0:
             self.config['beta_end'] = shared.opts.schedulers_beta_end
+        if 'rescale_betas_zero_snr' in self.config:
+            self.config['rescale_betas_zero_snr'] = shared.opts.schedulers_rescale_betas
+        if 'num_train_timesteps' in self.config:
+            self.config['num_train_timesteps'] = shared.opts.schedulers_timesteps_range
         if name == 'DPM++ 2M':
             self.config['algorithm_type'] = shared.opts.schedulers_dpm_solver
         if name == 'DEIS':

modules/shared.py

@@ -513,6 +513,8 @@ options_templates.update(options_section(('sampler-params', "Sampler Settings"),
     "schedulers_beta_schedule": OptionInfo("default", "Beta schedule", gr.Radio, {"choices": ['default', 'linear', 'scaled_linear', 'squaredcos_cap_v2']}),
     'schedulers_beta_start': OptionInfo(0, "Beta start", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.00001}),
     'schedulers_beta_end': OptionInfo(0, "Beta end", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.00001}),
+    'schedulers_timesteps_range': OptionInfo(1000, "Timesteps range", gr.Slider, {"minimum": 250, "maximum": 4000, "step": 1}),
+    "schedulers_rescale_betas": OptionInfo(False, "Rescale betas with zero terminal SNR", gr.Checkbox),
 
     # managed from ui.py for backend original k-diffusion
     "schedulers_sep_kdiffusers": OptionInfo("<h2>K-Diffusion specific config</h2>", "", gr.HTML),

modules/ui.py

@@ -448,7 +448,7 @@ def create_ui(startup_timer = None):
                                 enable_hr = gr.Checkbox(label='Enable second pass', value=False, elem_id="txt2img_enable_hr")
                             with FormRow(elem_id="sampler_selection_txt2img_alt_row1"):
                                 latent_index = gr.Dropdown(label='Secondary sampler', elem_id="txt2img_sampling_alt", choices=[x.name for x in modules.sd_samplers.samplers], value='Default', type="index")
-                                denoising_strength = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Denoising strength', value=0.5, elem_id="txt2img_denoising_strength")
+                                denoising_strength = gr.Slider(minimum=0.0, maximum=0.99, step=0.01, label='Denoising strength', value=0.5, elem_id="txt2img_denoising_strength")
                             with FormRow(elem_id="txt2img_hires_finalres", variant="compact"):
                                 hr_final_resolution = FormHTML(value="", elem_id="txtimg_hr_finalres", label="Upscaled resolution", interactive=False)
                             with FormRow(elem_id="txt2img_hires_fix_row1", variant="compact"):
@@ -721,7 +721,7 @@ def create_ui(startup_timer = None):
 
                     with gr.Accordion(open=False, label="Denoise", elem_classes=["small-accordion"], elem_id="img2img_denoise_group"):
                         with FormRow():
-                            denoising_strength = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Denoising strength', value=0.75, elem_id="img2img_denoising_strength")
+                            denoising_strength = gr.Slider(minimum=0.0, maximum=0.99, step=0.01, label='Denoising strength', value=0.75, elem_id="img2img_denoising_strength")
                             refiner_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Denoise start', value=0.0, elem_id="img2img_refiner_start")
 
                     with gr.Accordion(open=False, label="Advanced", elem_classes=["small-accordion"], elem_id="img2img_advanced_group"):

scripts/ipadapter.py

@@ -17,13 +17,13 @@ image_encoder = None
 loaded = None
 ADAPTERS = [
     'none',
-    'models/ip-adapter_sd15',
-    'models/ip-adapter_sd15_light',
-    'models/ip-adapter-plus_sd15',
-    'models/ip-adapter-plus-face_sd15',
-    # 'models/ip-adapter-full-face_sd15', # KeyError: 'proj.weight'
-    'sdxl_models/ip-adapter_sdxl',
+    'ip-adapter_sd15',
+    'ip-adapter_sd15_light',
+    'ip-adapter-plus_sd15',
+    'ip-adapter-plus-face_sd15',
+    'ip-adapter-full-face_sd15',
     # 'models/ip-adapter_sd15_vit-G', # RuntimeError: mat1 and mat2 shapes cannot be multiplied (2x1024 and 1280x3072)
+    'ip-adapter_sdxl',
     # 'sdxl_models/ip-adapter_sdxl_vit-h',
     # 'sdxl_models/ip-adapter-plus_sdxl_vit-h',
     # 'sdxl_models/ip-adapter-plus-face_sdxl_vit-h',
@@ -47,9 +47,14 @@ class Script(scripts.Script):
         return [adapter, scale, image]
 
     def process(self, p: processing.StableDiffusionProcessing, adapter, scale, image): # pylint: disable=arguments-differ
-        import torch
         from transformers import CLIPVisionModelWithProjection
-
+        # overrides
+        if hasattr(p, 'ip_adapter_name'):
+            adapter = p.ip_adapter_name
+        if hasattr(p, 'ip_adapter_scale'):
+            scale = p.ip_adapter_scale
+        if hasattr(p, 'ip_adapter_image'):
+            image = p.ip_adapter_image
         # init code
         global loaded, image_encoder # pylint: disable=global-statement
         if shared.sd_model is None:
@@ -88,8 +93,8 @@ class Script(scripts.Script):
                     return
 
         # main code
-        subfolder, model = adapter.split('/')
-        if model != loaded or getattr(shared.sd_model.unet.config, 'encoder_hid_dim_type', None) is None:
+        subfolder = 'models' if 'sd15' in adapter else 'sdxl_models'
+        if adapter != loaded or getattr(shared.sd_model.unet.config, 'encoder_hid_dim_type', None) is None:
             t0 = time.time()
             if loaded is not None:
                 shared.log.debug('IP adapter: reset attention processor')
@@ -98,12 +103,12 @@ class Script(scripts.Script):
             else:
                 shared.log.debug('IP adapter: load attention processor')
             shared.sd_model.image_encoder = image_encoder
-            shared.sd_model.load_ip_adapter("h94/IP-Adapter", subfolder=subfolder, weight_name=f'{model}.safetensors')
+            shared.sd_model.load_ip_adapter("h94/IP-Adapter", subfolder=subfolder, weight_name=f'{adapter}.safetensors')
             t1 = time.time()
-            shared.log.info(f'IP adapter load: adapter="{model}" scale={scale} image={image} time={t1-t0:.2f}')
-            loaded = model
+            shared.log.info(f'IP adapter load: adapter="{adapter}" scale={scale} image={image} time={t1-t0:.2f}')
+            loaded = adapter
         else:
-            shared.log.debug(f'IP adapter cache: adapter="{model}" scale={scale} image={image}')
+            shared.log.debug(f'IP adapter cache: adapter="{adapter}" scale={scale} image={image}')
         shared.sd_model.set_ip_adapter_scale(scale)
         p.task_args['ip_adapter_image'] = p.batch_size * [image]
         p.extra_generation_params["IP Adapter"] = f'{adapter}:{scale}'

scripts/xyz_grid.py

@@ -269,6 +269,8 @@ axis_options = [
     AxisOption("[FreeU] 2nd stage backbone factor", float, apply_setting('freeu_b2')),
     AxisOption("[FreeU] 1st stage skip factor", float, apply_setting('freeu_s1')),
     AxisOption("[FreeU] 2nd stage skip factor", float, apply_setting('freeu_s2')),
+    AxisOption("[IP adapter] Name", str, apply_field('ip_adapter_name'), cost=1.0),
+    AxisOption("[IP adapter] Scale", float, apply_field('ip_adapter_scale')),
 ]