Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-19 09:14:35 +02:00
Code Issues Packages Projects Releases Wiki Activity

add sa solver and prototype instaflow

Browse Source
This commit is contained in:
Vladimir Mandic
2024-01-31 09:51:34 -05:00
parent b1ccadf793
commit 09d1f1e788
12 changed files with 108 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+6 -3
View File
@@ -8,13 +8,11 @@ BLOCKERS:
OPTIONAL:
- pending `diffusers==0.26.0`
- wuerstchen v3 [pr](https://github.com/huggingface/diffusers/pull/6487)
- style aligned [pr](https://github.com/huggingface/diffusers/pull/6489)
- instaflow [pr](https://github.com/huggingface/diffusers/pull/6057)[repo](https://github.com/gnobitab/RectifiedFlow)
- control api
- masking api
- preprocess api
## Update for 2023-01-30
## Update for 2023-01-31
Another big release, highlights being:
- A lot more functionality in the **Control** module:
@@ -133,6 +131,10 @@ As of this release, default backend is set to **diffusers** as its more feature
- requires input image
- last word in prompt and negative prompt will be used as source and target subjects
- sampler must be set to default before loading the model
- [InstaFlow](https://github.com/gnobitab/InstaFlow)
- another take on super-fast image generation in a single step
- set sampler:default steps:1
- load from networks -> models -> reference
- **Improvements**
- **ui**
- check version and **update** SD.Next via UI
@@ -183,6 +185,7 @@ As of this release, default backend is set to **diffusers** as its more feature
for example, you can now deploy a zip of the sdnext folder
- **latent upscale**: updated latent upscalers (some are new)
*nearest, nearest-exact, area, bilinear, bicubic, bilinear-antialias, bicubic-antialias*
- **scheduler**: added `SA Solver`
- **model load to gpu**
new option in settings->diffusers allowing models to be loaded directly to GPU while keeping RAM free
this option is not compatible with any kind of model offloading as model is expected to stay in GPU
extensions-builtin/sd-webui-controlnet
+1 -1
Submodule extensions-builtin/sd-webui-controlnet updated: c5432dd4f6...0ee0281178
html/reference.json
+5
View File
@@ -168,5 +168,10 @@
"path": "salesforce/blipdiffusion",
"desc": "BLIP-Diffusion, a new subject-driven image generation model that supports multimodal control which consumes inputs of subject images and text prompts. Unlike other subject-driven generation models, BLIP-Diffusion introduces a new multimodal encoder which is pre-trained to provide subject representation.",
"preview": "salesforce--blipdiffusion.jpg"
},
"InstaFlow 0.9B": {
"path": "XCLiu/instaflow_0_9B_from_sd_1_5",
"desc": "InstaFlow is an ultra-fast, one-step image generator that achieves image quality close to Stable Diffusion. This efficiency is made possible through a recent Rectified Flow technique, which trains probability flows with straight trajectories, hence inherently requiring only a single step for fast inference.",
"preview": "XCLiu--instaflow_0_9B_from_sd_1_5.jpg"
}
}
models/Reference/XCLiu--instaflow_0_9B_from_sd_1_5.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 63 KiB

modules/modeldata.py
+3 -1
View File
@@ -84,8 +84,10 @@ class Shared(sys.modules[__name__].__class__):
model_type = 'sd'
elif "LatentConsistencyModel" in self.sd_model.__class__.__name__:
model_type = 'sd' # lcm is compatible with sd
elif "InstaFlowPipeline" in self.sd_model.__class__.__name__:
model_type = 'sd' # instaflow is compatible with sd
elif "AnimateDiffPipeline" in self.sd_model.__class__.__name__:
model_type = 'sd' # ad is compatible with sd
model_type = 'sd' # sd is compatible with sd
elif "Kandinsky" in self.sd_model.__class__.__name__:
model_type = 'kandinsky'
else:
modules/processing_diffusers.py
+3 -3
View File
@@ -442,7 +442,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
else:
steps = p.steps
debug_steps(f'Steps: type=base input={p.steps} output={steps} task={sd_models.get_diffusers_task(shared.sd_model)} refiner={use_refiner_start} denoise={p.denoising_strength} model={shared.sd_model_type}')
return max(2, int(steps))
return max(1, int(steps))
def calculate_hires_steps():
if p.hr_second_pass_steps > 0:
@@ -452,7 +452,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
else:
steps = 0
debug_steps(f'Steps: type=hires input={p.hr_second_pass_steps} output={steps} denoise={p.denoising_strength} model={shared.sd_model_type}')
return max(2, int(steps))
return max(1, int(steps))
def calculate_refiner_steps():
if "StableDiffusionXL" in shared.sd_refiner.__class__.__name__:
@@ -467,7 +467,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
#steps = p.refiner_steps # SD 1.5 with denoise strenght
steps = (p.refiner_steps * 1.25) + 1
debug_steps(f'Steps: type=refiner input={p.refiner_steps} output={steps} start={p.refiner_start} denoise={p.denoising_strength}')
return max(2, int(steps))
return max(1, int(steps))
shared.sd_model = update_pipeline(shared.sd_model, p)
base_args = set_pipeline_args(
modules/sd_models.py
+67 -66
View File
@@ -536,52 +536,48 @@ def change_backend():
def detect_pipeline(f: str, op: str = 'model', warning=True):
if not f.endswith('.safetensors'):
return None, None
guess = shared.opts.diffusers_pipeline
warn = shared.log.warning if warning else lambda *args, **kwargs: None
size = 0
if guess == 'Autodetect':
try:
guess = 'Stable Diffusion XL' if 'XL' in f.upper() else 'Stable Diffusion'
# guess by size
size = round(os.path.getsize(f) / 1024 / 1024)
if size < 128:
warn(f'Model size smaller than expected: {f} size={size} MB')
elif (size >= 316 and size <= 324) or (size >= 156 and size <= 164): # 320 or 160
warn(f'Model detected as VAE model, but attempting to load as model: {op}={f} size={size} MB')
guess = 'VAE'
elif size >= 5351 and size <= 5359: # 5353
guess = 'Stable Diffusion' # SD v2
elif size >= 5791 and size <= 5799: # 5795
if shared.backend == shared.Backend.ORIGINAL:
warn(f'Model detected as SD-XL refiner model, but attempting to load using backend=original: {op}={f} size={size} MB')
if op == 'model':
warn(f'Model detected as SD-XL refiner model, but attempting to load a base model: {op}={f} size={size} MB')
guess = 'Stable Diffusion XL'
elif (size >= 6611 and size <= 7220): # 6617, HassakuXL is 6776, monkrenRealisticINT_v10 is 7217
if shared.backend == shared.Backend.ORIGINAL:
warn(f'Model detected as SD-XL base model, but attempting to load using backend=original: {op}={f} size={size} MB')
guess = 'Stable Diffusion XL'
elif size >= 3361 and size <= 3369: # 3368
if shared.backend == shared.Backend.ORIGINAL:
warn(f'Model detected as SD upscale model, but attempting to load using backend=original: {op}={f} size={size} MB')
guess = 'Stable Diffusion Upscale'
elif size >= 4891 and size <= 4899: # 4897
if shared.backend == shared.Backend.ORIGINAL:
warn(f'Model detected as SD XL inpaint model, but attempting to load using backend=original: {op}={f} size={size} MB')
guess = 'Stable Diffusion XL Inpaint'
elif size >= 9791 and size <= 9799: # 9794
if shared.backend == shared.Backend.ORIGINAL:
warn(f'Model detected as SD XL instruct pix2pix model, but attempting to load using backend=original: {op}={f} size={size} MB')
guess = 'Stable Diffusion XL Instruct'
elif size > 3138 and size < 3142: #3140
if shared.backend == shared.Backend.ORIGINAL:
warn(f'Model detected as Segmind Vega model, but attempting to load using backend=original: {op}={f} size={size} MB')
guess = 'Stable Diffusion XL'
else:
if 'XL' in f.upper():
if os.path.isfile(f) and f.endswith('.safetensors'):
size = round(os.path.getsize(f) / 1024 / 1024)
if size < 128:
warn(f'Model size smaller than expected: {f} size={size} MB')
elif (size >= 316 and size <= 324) or (size >= 156 and size <= 164): # 320 or 160
warn(f'Model detected as VAE model, but attempting to load as model: {op}={f} size={size} MB')
guess = 'VAE'
elif size >= 5351 and size <= 5359: # 5353
guess = 'Stable Diffusion' # SD v2
elif size >= 5791 and size <= 5799: # 5795
if shared.backend == shared.Backend.ORIGINAL:
warn(f'Model detected as SD-XL refiner model, but attempting to load using backend=original: {op}={f} size={size} MB')
if op == 'model':
warn(f'Model detected as SD-XL refiner model, but attempting to load a base model: {op}={f} size={size} MB')
guess = 'Stable Diffusion XL'
elif (size >= 6611 and size <= 7220): # 6617, HassakuXL is 6776, monkrenRealisticINT_v10 is 7217
if shared.backend == shared.Backend.ORIGINAL:
warn(f'Model detected as SD-XL base model, but attempting to load using backend=original: {op}={f} size={size} MB')
guess = 'Stable Diffusion XL'
elif size >= 3361 and size <= 3369: # 3368
if shared.backend == shared.Backend.ORIGINAL:
warn(f'Model detected as SD upscale model, but attempting to load using backend=original: {op}={f} size={size} MB')
guess = 'Stable Diffusion Upscale'
elif size >= 4891 and size <= 4899: # 4897
if shared.backend == shared.Backend.ORIGINAL:
warn(f'Model detected as SD XL inpaint model, but attempting to load using backend=original: {op}={f} size={size} MB')
guess = 'Stable Diffusion XL Inpaint'
elif size >= 9791 and size <= 9799: # 9794
if shared.backend == shared.Backend.ORIGINAL:
warn(f'Model detected as SD XL instruct pix2pix model, but attempting to load using backend=original: {op}={f} size={size} MB')
guess = 'Stable Diffusion XL Instruct'
elif size > 3138 and size < 3142: #3140
if shared.backend == shared.Backend.ORIGINAL:
warn(f'Model detected as Segmind Vega model, but attempting to load using backend=original: {op}={f} size={size} MB')
guess = 'Stable Diffusion XL'
else:
guess = 'Stable Diffusion'
# guess by name
"""
if 'LCM_' in f.upper() or 'LCM-' in f.upper() or '_LCM' in f.upper() or '-LCM' in f.upper():
@@ -589,6 +585,10 @@ def detect_pipeline(f: str, op: str = 'model', warning=True):
warn(f'Model detected as LCM model, but attempting to load using backend=original: {op}={f} size={size} MB')
guess = 'Latent Consistency Model'
"""
if 'instaflow' in f:
if shared.backend == shared.Backend.ORIGINAL:
warn(f'Model detected as InstaFlow model, but attempting to load using backend=original: {op}={f} size={size} MB')
guess = 'InstaFlow'
if 'PixArt' in f:
if shared.backend == shared.Backend.ORIGINAL:
warn(f'Model detected as PixArt Alpha model, but attempting to load using backend=original: {op}={f} size={size} MB')
@@ -788,37 +788,38 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
diffusers_load_config["vae"] = vae
shared.log.debug(f'Diffusers loading: path="{checkpoint_info.path}"')
pipeline, model_type = detect_pipeline(checkpoint_info.path, op)
if os.path.isdir(checkpoint_info.path):
err1 = None
err2 = None
err3 = None
try: # try autopipeline first, best choice but not all pipelines are available
sd_model = diffusers.AutoPipelineForText2Image.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config)
sd_model.model_type = sd_model.__class__.__name__
except Exception as e:
err1 = e
# shared.log.error(f'AutoPipeline: {e}')
try: # try diffusion pipeline next second-best choice, works for most non-linked pipelines
if err1 is not None:
sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config)
if model_type in ['InstaFlow']: # forced pipeline
sd_model = pipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config)
else:
err1, err2, err3 = None, None, None
try: # 1 - autopipeline, best choice but not all pipelines are available
sd_model = diffusers.AutoPipelineForText2Image.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config)
sd_model.model_type = sd_model.__class__.__name__
except Exception as e:
err2 = e
# shared.log.error(f'DiffusionPipeline: {e}')
try: # try basic pipeline next just in case
if err2 is not None:
sd_model = diffusers.StableDiffusionPipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config)
sd_model.model_type = sd_model.__class__.__name__
except Exception as e:
err3 = e # ignore last error
shared.log.error(f'StableDiffusionPipeline: {e}')
if err3 is not None:
shared.log.error(f'Failed loading {op}: {checkpoint_info.path} auto={err1} diffusion={err2}')
return
except Exception as e:
err1 = e
# shared.log.error(f'AutoPipeline: {e}')
try: # 2 - diffusion pipeline, works for most non-linked pipelines
if err1 is not None:
sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config)
sd_model.model_type = sd_model.__class__.__name__
except Exception as e:
err2 = e
# shared.log.error(f'DiffusionPipeline: {e}')
try: # 3 - try basic pipeline just in case
if err2 is not None:
sd_model = diffusers.StableDiffusionPipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config)
sd_model.model_type = sd_model.__class__.__name__
except Exception as e:
err3 = e # ignore last error
shared.log.error(f'StableDiffusionPipeline: {e}')
if err3 is not None:
shared.log.error(f'Failed loading {op}: {checkpoint_info.path} auto={err1} diffusion={err2}')
return
elif os.path.isfile(checkpoint_info.path) and checkpoint_info.path.lower().endswith('.safetensors'):
# diffusers_load_config["local_files_only"] = True
diffusers_load_config["extract_ema"] = shared.opts.diffusers_extract_ema
pipeline, model_type = detect_pipeline(checkpoint_info.path, op)
if pipeline is None:
shared.log.error(f'Diffusers {op} pipeline not initialized: {shared.opts.diffusers_pipeline}')
return
modules/sd_samplers_diffusers.py
+17 -9
View File
@@ -11,6 +11,7 @@ try:
from diffusers import (
DDIMScheduler,
DDPMScheduler,
UniPCMultistepScheduler,
DEISMultistepScheduler,
DPMSolverMultistepScheduler,
DPMSolverSinglestepScheduler,
@@ -19,10 +20,9 @@ try:
EulerDiscreteScheduler,
HeunDiscreteScheduler,
KDPM2DiscreteScheduler,
PNDMScheduler,
UniPCMultistepScheduler,
LMSDiscreteScheduler,
KDPM2AncestralDiscreteScheduler,
LMSDiscreteScheduler,
PNDMScheduler,
LCMScheduler,
)
except Exception as e:
@@ -32,22 +32,22 @@ except Exception as e:
config = {
# beta_start, beta_end are typically per-scheduler, but we don't want them as they should be taken from the model itself as those are values model was trained on
# prediction_type is ideally set in model as well, but it maybe needed that we do auto-detect of model type in the future
'All': { 'num_train_timesteps': 500, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'linear', 'prediction_type': 'epsilon' },
'All': { 'num_train_timesteps': 1000, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'linear', 'prediction_type': 'epsilon' },
'DDIM': { 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'clip_sample_range': 1.0, 'sample_max_value': 1.0, 'timestep_spacing': 'linspace', 'rescale_betas_zero_snr': False },
'DDPM': { 'variance_type': "fixed_small", 'clip_sample': True, 'thresholding': False, 'clip_sample_range': 1.0, 'sample_max_value': 1.0, 'timestep_spacing': 'linspace'},
'UniPC': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'predict_x0': 'bh2', 'lower_order_final': True },
'DEIS': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "deis", 'solver_type': "logrho", 'lower_order_final': True },
'DPM++ 1S': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False },
'DPM++ 2M': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False },
'DPM 1S': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'final_sigmas_type': 'zero' },
'DPM 2M': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'final_sigmas_type': 'zero' },
'DPM SDE': { 'use_karras_sigmas': False },
'Euler a': { 'rescale_betas_zero_snr': False },
'Euler': { 'interpolation_type': "linear", 'use_karras_sigmas': False, 'rescale_betas_zero_snr': False },
'Heun': { 'use_karras_sigmas': False },
'DDPM': { 'variance_type': "fixed_small", 'clip_sample': True, 'thresholding': False, 'clip_sample_range': 1.0, 'sample_max_value': 1.0, 'timestep_spacing': 'linspace', 'rescale_betas_zero_snr': False },
'KDPM2': { 'steps_offset': 0 },
'KDPM2 a': { 'steps_offset': 0 },
'LMSD': { 'use_karras_sigmas': False, 'timestep_spacing': 'linspace', 'steps_offset': 0 },
'PNDM': { 'skip_prk_steps': False, 'set_alpha_to_one': False, 'steps_offset': 0 },
'UniPC': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'predict_x0': 'bh2', 'lower_order_final': True },
'LCM': { 'beta_start': 0.00085, 'beta_end': 0.012, 'beta_schedule': "scaled_linear", 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False },
'LCM': { 'beta_start': 0.00085, 'beta_end': 0.012, 'beta_schedule': "scaled_linear", 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False, 'thresholding': False },
}
samplers_data_diffusers = [
@@ -69,6 +69,14 @@ samplers_data_diffusers = [
sd_samplers_common.SamplerData('LCM', lambda model: DiffusionSampler('LCM', LCMScheduler, model), [], {}),
]
try:
from diffusers import SASolverScheduler
config['SA Solver'] = {'predictor_order': 2, 'corrector_order': 2, 'thresholding': False, 'lower_order_final': True, 'use_karras_sigmas': False, 'timestep_spacing': 'linspace'}
samplers_data_diffusers.append(sd_samplers_common.SamplerData('SA Solver', lambda model: DiffusionSampler('SA Solver', SASolverScheduler, model), [], {}))
except Exception as e:
shared.log.debug(f'Sampler: {e}')
class DiffusionSampler:
def __init__(self, name, constructor, model, **kwargs):
if name == 'Default':
modules/shared.py
+1 -1
View File
@@ -581,7 +581,7 @@ options_templates.update(options_section(('sampler-params', "Sampler Settings"),
# managed from ui.py for backend diffusers
"schedulers_sep_diffusers": OptionInfo("<h2>Diffusers specific config</h2>", "", gr.HTML),
"schedulers_dpm_solver": OptionInfo("sde-dpmsolver++", "DPM solver algorithm", gr.Radio, {"choices": ['dpmsolver', 'dpmsolver++', 'sde-dpmsolver', 'sde-dpmsolver++']}),
"schedulers_dpm_solver": OptionInfo("sde-dpmsolver++", "DPM solver algorithm", gr.Radio, {"choices": ['dpmsolver++', 'sde-dpmsolver++']}),
"schedulers_beta_schedule": OptionInfo("default", "Beta schedule", gr.Radio, {"choices": ['default', 'linear', 'scaled_linear', 'squaredcos_cap_v2']}),
'schedulers_beta_start': OptionInfo(0, "Beta start", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.00001}),
'schedulers_beta_end': OptionInfo(0, "Beta end", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.00001}),
modules/shared_items.py
+1
View File
@@ -47,6 +47,7 @@ def get_pipelines():
'Kandinsky 3': getattr(diffusers, 'Kandinsky3Pipeline', None),
'DeepFloyd IF': getattr(diffusers, 'IFPipeline', None),
'Custom Diffusers Pipeline': getattr(diffusers, 'DiffusionPipeline', None),
'InstaFlow': diffusers.utils.get_class_from_dynamic_module('instaflow_one_step', module_file='pipeline.py')
# Segmind SSD-1B, Segmind Tiny
}
for k, v in pipelines.items():
modules/ui_extra_networks.py
+2 -2
View File
@@ -145,7 +145,7 @@ class ExtraNetworksPage:
def link_preview(self, filename):
quoted_filename = urllib.parse.quote(filename.replace('\\', '/'))
mtime = os.path.getmtime(filename)
mtime = os.path.getmtime(filename) if os.path.exists(filename) else 0
preview = f"./sd_extra_networks/thumb?filename={quoted_filename}&mtime={mtime}"
return preview
@@ -582,7 +582,7 @@ def create_ui(container, button_parent, tabname, skip_indexing = False):
import concurrent
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
for page in get_pages():
executor.submit(page.create_items, page)
executor.submit(page.create_items, ui.tabname)
for page in get_pages():
page.create_page(ui.tabname, skip_indexing)
with gr.Tab(page.title, id=page.title.lower().replace(" ", "_"), elem_classes="extra-networks-tab") as tab:
modules/ui_extra_networks_checkpoints.py
+2
View File
@@ -64,6 +64,8 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage):
return record
def list_items(self):
import sys
shared.log.debug(f'List items: function={sys._getframe(1).f_code.co_name}') # pylint: disable=protected-access
# items = [self.create_item(cp) for cp in list(sd_models.checkpoints_list)] + list(self.list_reference())
items = []
with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor: