From bd64bac2a39da712eefa7b3cd73165d09001628a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 7 Dec 2023 12:38:59 -0500 Subject: [PATCH] update ip-adapter, schedulers and xyz-grid --- CHANGELOG.md | 8 +++++-- README.md | 3 +++ extensions-builtin/sd-webui-controlnet | 2 +- modules/k-diffusion | 2 +- modules/sd_samplers_diffusers.py | 10 +++++--- modules/shared.py | 2 ++ modules/ui.py | 4 ++-- scripts/ipadapter.py | 33 +++++++++++++++----------- scripts/xyz_grid.py | 2 ++ 9 files changed, 43 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8db1f12af..72d8cc5b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,17 +1,21 @@ # Change Log for SD.Next -## Update for 2023-12-06 +## Update for 2023-12-07 *Note*: based on `diffusers==0.25.0.dev0` - **Diffusers** - **AnimateDiff** can now be used with *second pass* - enhance, upscale and hires your videos! - - **IP Adapter** add support for `ip-adapter-plus_sd15` and `ip-adapter-plus-face_sd15` + - **IP Adapter** add support for `ip-adapter-plus_sd15`, `ip-adapter-plus-face_sd15` and `ip-adapter-full-face_sd15` + additionally, ip-adapter can now be used in xyz-grid - **HDR controls** are now batch-aware for enhancement of multiple images or video frames - [Playground v1](https://huggingface.co/playgroundai/playground-v1), [Playground v2 256](https://huggingface.co/playgroundai/playground-v2-256px-base), [Playground v2 512](https://huggingface.co/playgroundai/playground-v2-512px-base), [Playground v2 1024](https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic) model support - simply select from *networks -> reference* and use as usual - [ModelScope T2V](https://huggingface.co/damo-vilab/text-to-video-ms-1.7b) model support - simply select from *networks -> reference* and use from *txt2img* tab + - **Schedulers** + - add timesteps range, changing it will make scheduler to be over-complete or under-complete + - add rescale betas with zero SNR option (applicable to Euler and DDIM, allows for higher dynamic range) - **General** - **LoRA** add support for block weights, thanks @AI-Casanova example `` diff --git a/README.md b/README.md index 4c9e62ce3..79a647d11 100644 --- a/README.md +++ b/README.md @@ -211,6 +211,9 @@ General goals: ### **Docs** +If you're unsure how to use a feature, best place to start is [Wiki](https://github.com/vladmandic/automatic/wiki) and if its not there, +check [ChangeLog](CHANGELOG.md) for when feature was first introduced as it will always have a short note on how to use it + - [Wiki](https://github.com/vladmandic/automatic/wiki) - [ReadMe](README.md) - [ToDo](TODO.md) diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index 10bd9b25f..96dbc601a 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit 10bd9b25f62deab9acb256301bbf3363c42645e7 +Subproject commit 96dbc601a6c880571d3a2a1314052d0922114604 diff --git a/modules/k-diffusion b/modules/k-diffusion index 045515774..cc49cf618 160000 --- a/modules/k-diffusion +++ b/modules/k-diffusion @@ -1 +1 @@ -Subproject commit 045515774882014cc14c1ba2668ab5bad9cbf7c0 +Subproject commit cc49cf6182284e577e896943f8e29c7c9d1a7f2c diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index 9cd3ee806..dc0ccc03c 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -26,7 +26,7 @@ except Exception as e: config = { # beta_start, beta_end are typically per-scheduler, but we don't want them as they should be taken from the model itself as those are values model was trained on # prediction_type is ideally set in model as well, but it maybe needed that we do auto-detect of model type in the future - 'All': { 'num_train_timesteps': 1000, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'linear', 'prediction_type': 'epsilon' }, + 'All': { 'num_train_timesteps': 500, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'linear', 'prediction_type': 'epsilon' }, 'DDIM': { 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'clip_sample_range': 1.0, 'sample_max_value': 1.0, 'timestep_spacing': 'linspace', 'rescale_betas_zero_snr': False }, 'DDPM': { 'variance_type': "fixed_small", 'clip_sample': True, 'thresholding': False, 'clip_sample_range': 1.0, 'sample_max_value': 1.0, 'timestep_spacing': 'linspace'}, 'DEIS': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "deis", 'solver_type': "logrho", 'lower_order_final': True }, @@ -34,14 +34,14 @@ config = { 'DPM++ 2M': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False }, 'DPM SDE': { 'use_karras_sigmas': False }, 'Euler a': { }, - 'Euler': { 'interpolation_type': "linear", 'use_karras_sigmas': False }, + 'Euler': { 'interpolation_type': "linear", 'use_karras_sigmas': False, 'rescale_betas_zero_snr': False }, 'Heun': { 'use_karras_sigmas': False }, 'KDPM2': { 'steps_offset': 0 }, 'KDPM2 a': { 'steps_offset': 0 }, 'LMSD': { 'use_karras_sigmas': False, 'timestep_spacing': 'linspace', 'steps_offset': 0 }, 'PNDM': { 'skip_prk_steps': False, 'set_alpha_to_one': False, 'steps_offset': 0 }, 'UniPC': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'predict_x0': 'bh2', 'lower_order_final': True }, - 'LCM': { 'num_train_timesteps': 1000, 'beta_start': 0.00085, 'beta_end': 0.012, 'beta_schedule': "scaled_linear", 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False }, + 'LCM': { 'beta_start': 0.00085, 'beta_end': 0.012, 'beta_schedule': "scaled_linear", 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False }, } samplers_data_diffusers = [ @@ -108,6 +108,10 @@ class DiffusionSampler: self.config['beta_start'] = shared.opts.schedulers_beta_start if 'beta_end' in self.config and shared.opts.schedulers_beta_end > 0: self.config['beta_end'] = shared.opts.schedulers_beta_end + if 'rescale_betas_zero_snr' in self.config: + self.config['rescale_betas_zero_snr'] = shared.opts.schedulers_rescale_betas + if 'num_train_timesteps' in self.config: + self.config['num_train_timesteps'] = shared.opts.schedulers_timesteps_range if name == 'DPM++ 2M': self.config['algorithm_type'] = shared.opts.schedulers_dpm_solver if name == 'DEIS': diff --git a/modules/shared.py b/modules/shared.py index 204ae7714..6569ab443 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -513,6 +513,8 @@ options_templates.update(options_section(('sampler-params', "Sampler Settings"), "schedulers_beta_schedule": OptionInfo("default", "Beta schedule", gr.Radio, {"choices": ['default', 'linear', 'scaled_linear', 'squaredcos_cap_v2']}), 'schedulers_beta_start': OptionInfo(0, "Beta start", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.00001}), 'schedulers_beta_end': OptionInfo(0, "Beta end", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.00001}), + 'schedulers_timesteps_range': OptionInfo(1000, "Timesteps range", gr.Slider, {"minimum": 250, "maximum": 4000, "step": 1}), + "schedulers_rescale_betas": OptionInfo(False, "Rescale betas with zero terminal SNR", gr.Checkbox), # managed from ui.py for backend original k-diffusion "schedulers_sep_kdiffusers": OptionInfo("

K-Diffusion specific config

", "", gr.HTML), diff --git a/modules/ui.py b/modules/ui.py index 6b076469b..39aaaf9b1 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -448,7 +448,7 @@ def create_ui(startup_timer = None): enable_hr = gr.Checkbox(label='Enable second pass', value=False, elem_id="txt2img_enable_hr") with FormRow(elem_id="sampler_selection_txt2img_alt_row1"): latent_index = gr.Dropdown(label='Secondary sampler', elem_id="txt2img_sampling_alt", choices=[x.name for x in modules.sd_samplers.samplers], value='Default', type="index") - denoising_strength = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Denoising strength', value=0.5, elem_id="txt2img_denoising_strength") + denoising_strength = gr.Slider(minimum=0.0, maximum=0.99, step=0.01, label='Denoising strength', value=0.5, elem_id="txt2img_denoising_strength") with FormRow(elem_id="txt2img_hires_finalres", variant="compact"): hr_final_resolution = FormHTML(value="", elem_id="txtimg_hr_finalres", label="Upscaled resolution", interactive=False) with FormRow(elem_id="txt2img_hires_fix_row1", variant="compact"): @@ -721,7 +721,7 @@ def create_ui(startup_timer = None): with gr.Accordion(open=False, label="Denoise", elem_classes=["small-accordion"], elem_id="img2img_denoise_group"): with FormRow(): - denoising_strength = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Denoising strength', value=0.75, elem_id="img2img_denoising_strength") + denoising_strength = gr.Slider(minimum=0.0, maximum=0.99, step=0.01, label='Denoising strength', value=0.75, elem_id="img2img_denoising_strength") refiner_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Denoise start', value=0.0, elem_id="img2img_refiner_start") with gr.Accordion(open=False, label="Advanced", elem_classes=["small-accordion"], elem_id="img2img_advanced_group"): diff --git a/scripts/ipadapter.py b/scripts/ipadapter.py index f581b4868..c822942ad 100644 --- a/scripts/ipadapter.py +++ b/scripts/ipadapter.py @@ -17,13 +17,13 @@ image_encoder = None loaded = None ADAPTERS = [ 'none', - 'models/ip-adapter_sd15', - 'models/ip-adapter_sd15_light', - 'models/ip-adapter-plus_sd15', - 'models/ip-adapter-plus-face_sd15', - # 'models/ip-adapter-full-face_sd15', # KeyError: 'proj.weight' - 'sdxl_models/ip-adapter_sdxl', + 'ip-adapter_sd15', + 'ip-adapter_sd15_light', + 'ip-adapter-plus_sd15', + 'ip-adapter-plus-face_sd15', + 'ip-adapter-full-face_sd15', # 'models/ip-adapter_sd15_vit-G', # RuntimeError: mat1 and mat2 shapes cannot be multiplied (2x1024 and 1280x3072) + 'ip-adapter_sdxl', # 'sdxl_models/ip-adapter_sdxl_vit-h', # 'sdxl_models/ip-adapter-plus_sdxl_vit-h', # 'sdxl_models/ip-adapter-plus-face_sdxl_vit-h', @@ -47,9 +47,14 @@ class Script(scripts.Script): return [adapter, scale, image] def process(self, p: processing.StableDiffusionProcessing, adapter, scale, image): # pylint: disable=arguments-differ - import torch from transformers import CLIPVisionModelWithProjection - + # overrides + if hasattr(p, 'ip_adapter_name'): + adapter = p.ip_adapter_name + if hasattr(p, 'ip_adapter_scale'): + scale = p.ip_adapter_scale + if hasattr(p, 'ip_adapter_image'): + image = p.ip_adapter_image # init code global loaded, image_encoder # pylint: disable=global-statement if shared.sd_model is None: @@ -88,8 +93,8 @@ class Script(scripts.Script): return # main code - subfolder, model = adapter.split('/') - if model != loaded or getattr(shared.sd_model.unet.config, 'encoder_hid_dim_type', None) is None: + subfolder = 'models' if 'sd15' in adapter else 'sdxl_models' + if adapter != loaded or getattr(shared.sd_model.unet.config, 'encoder_hid_dim_type', None) is None: t0 = time.time() if loaded is not None: shared.log.debug('IP adapter: reset attention processor') @@ -98,12 +103,12 @@ class Script(scripts.Script): else: shared.log.debug('IP adapter: load attention processor') shared.sd_model.image_encoder = image_encoder - shared.sd_model.load_ip_adapter("h94/IP-Adapter", subfolder=subfolder, weight_name=f'{model}.safetensors') + shared.sd_model.load_ip_adapter("h94/IP-Adapter", subfolder=subfolder, weight_name=f'{adapter}.safetensors') t1 = time.time() - shared.log.info(f'IP adapter load: adapter="{model}" scale={scale} image={image} time={t1-t0:.2f}') - loaded = model + shared.log.info(f'IP adapter load: adapter="{adapter}" scale={scale} image={image} time={t1-t0:.2f}') + loaded = adapter else: - shared.log.debug(f'IP adapter cache: adapter="{model}" scale={scale} image={image}') + shared.log.debug(f'IP adapter cache: adapter="{adapter}" scale={scale} image={image}') shared.sd_model.set_ip_adapter_scale(scale) p.task_args['ip_adapter_image'] = p.batch_size * [image] p.extra_generation_params["IP Adapter"] = f'{adapter}:{scale}' diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py index 15d11e555..f6fc5d553 100644 --- a/scripts/xyz_grid.py +++ b/scripts/xyz_grid.py @@ -269,6 +269,8 @@ axis_options = [ AxisOption("[FreeU] 2nd stage backbone factor", float, apply_setting('freeu_b2')), AxisOption("[FreeU] 1st stage skip factor", float, apply_setting('freeu_s1')), AxisOption("[FreeU] 2nd stage skip factor", float, apply_setting('freeu_s2')), + AxisOption("[IP adapter] Name", str, apply_field('ip_adapter_name'), cost=1.0), + AxisOption("[IP adapter] Scale", float, apply_field('ip_adapter_scale')), ]