diff --git a/CHANGELOG.md b/CHANGELOG.md index 35d3fcbe9..3a07f154c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,19 @@ # Change Log for SD.Next -## Update for 2024-12-15 +## Update for 2024-12-16 + +- Sana: both 1.6B and 0.6B +- ControlNet: better Union results, support for ProMax and Tile +- FreeScale: run optimized iterative generation of images at different scales +- Samplers: UniPC, DEIS, SA, DPM-Multistep: add FlowMatch sigma method and prediction type ### New models and integrations +- [NVLabs Sana](https://huggingface.co/Efficient-Large-Model/Sana_1600M_1024px) + **Sana** can synthesize high-resolution images with strong text-image alignment by using **Gemma2** as text-encoder + support for both 1.6B and 0.6B models + to use, select from *networks -> models -> reference* and models will be auto-downloaded on first use + *reference values*: sampler: default, width/height: 1024, guidance scale: 4.5, attention guidance: 3.0, adaptive scaling: 0.0 - [Flux Tools](https://blackforestlabs.ai/flux-1-tools/) **Redux** is actually a tool, **Fill** is inpaint/outpaint optimized version of *Flux-dev* **Canny** & **Depth** are optimized versions of *Flux-dev* for their respective tasks: they are *not* ControlNets that work on top of a model @@ -98,6 +108,7 @@ - **IPEX**: update to IPEX 2.5.10+xpu - **OpenVINO**: update to 2024.5.0 - **Sampler** improvements + - UniPC, DEIS, SA, DPM-Multistep: allow FlowMatch method - Euler FlowMatch: add sigma methods (*karras/exponential/betas*) - Euler FlowMatch: allow using timestep presets to set sigmas - DPM FlowMatch: update all and add sigma methods diff --git a/html/reference.json b/html/reference.json index 4a549586f..8a0965697 100644 --- a/html/reference.json +++ b/html/reference.json @@ -180,6 +180,19 @@ "extras": "sampler: Default, cfg_scale: 3.5" }, + "NVLabs Sana 1.6B": { + "path": "Efficient-Large-Model/Sana_1600M_1024px_diffusers", + "desc": "Sana is a text-to-image framework that can efficiently generate images up to 4096 × 4096 resolution. Sana can synthesize high-resolution, high-quality images with strong text-image alignment at a remarkably fast speed, deployable on laptop GPU.", + "preview": "Efficient-Large-Model--Sana_1600M_1024px_diffusers.jpg", + "skip": true + }, + "NVLabs Sana 0.6B": { + "path": "Efficient-Large-Model/Sana_600M_1024px_diffusers", + "desc": "Sana is a text-to-image framework that can efficiently generate images up to 4096 × 4096 resolution. Sana can synthesize high-resolution, high-quality images with strong text-image alignment at a remarkably fast speed, deployable on laptop GPU.", + "preview": "Efficient-Large-Model--Sana_1600M_1024px_diffusers.jpg", + "skip": true + }, + "VectorSpaceLab OmniGen v1": { "path": "Shitao/OmniGen-v1", "desc": "OmniGen is a unified image generation model that can generate a wide range of images from multi-modal prompts. It is designed to be simple, flexible and easy to use.", diff --git a/installer.py b/installer.py index 18a8ad1f1..a12b09d4d 100644 --- a/installer.py +++ b/installer.py @@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): def check_diffusers(): if args.skip_all or args.skip_requirements: return - sha = '63243406ba5510c10d5cac931882918ceba926f9' # diffusers commit hash + sha = '5fb3a985173efaae7ff381b9040c386751d643da' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/models/Reference/Efficient-Large-Model--Sana_1600M_1024px_diffusers.jpg b/models/Reference/Efficient-Large-Model--Sana_1600M_1024px_diffusers.jpg new file mode 100644 index 000000000..654f85403 Binary files /dev/null and b/models/Reference/Efficient-Large-Model--Sana_1600M_1024px_diffusers.jpg differ diff --git a/modules/model_flux.py b/modules/model_flux.py index ce2c55f70..8d6a02ef6 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -34,7 +34,8 @@ def load_flux_quanto(checkpoint_info): with torch.device("meta"): transformer = diffusers.FluxTransformer2DModel.from_config(os.path.join(repo_path, "transformer", "config.json")).to(dtype=dtype) quanto.requantize(transformer, state_dict, quantization_map, device=torch.device("cpu")) - transformer.eval() + if shared.opts.diffusers_eval: + transformer.eval() if transformer.dtype != devices.dtype: try: transformer = transformer.to(dtype=devices.dtype) @@ -61,7 +62,8 @@ def load_flux_quanto(checkpoint_info): with torch.device("meta"): text_encoder_2 = transformers.T5EncoderModel(t5_config).to(dtype=dtype) quanto.requantize(text_encoder_2, state_dict, quantization_map, device=torch.device("cpu")) - text_encoder_2.eval() + if shared.opts.diffusers_eval: + text_encoder_2.eval() if text_encoder_2.dtype != devices.dtype: try: text_encoder_2 = text_encoder_2.to(dtype=devices.dtype) diff --git a/modules/model_omnigen.py b/modules/model_omnigen.py index 64c99ddd7..a08ad4ed5 100644 --- a/modules/model_omnigen.py +++ b/modules/model_omnigen.py @@ -17,7 +17,8 @@ def load_omnigen(checkpoint_info, diffusers_load_config={}): # pylint: disable=u pipe.separate_cfg_infer = True pipe.use_kv_cache = False pipe.model.to(device=devices.device, dtype=devices.dtype) - pipe.model.eval() + if shared.opts.diffusers_eval: + pipe.model.eval() pipe.vae.to(devices.device, dtype=devices.dtype) devices.torch_gc() diff --git a/modules/model_sana.py b/modules/model_sana.py new file mode 100644 index 000000000..06e6fe981 --- /dev/null +++ b/modules/model_sana.py @@ -0,0 +1,25 @@ +import diffusers + + +def load_sana(checkpoint_info, diffusers_load_config={}): + from modules import shared, sd_models, devices, modelloader, model_quant + modelloader.hf_login() + + repo_id = checkpoint_info if isinstance(checkpoint_info, str) else checkpoint_info.path + repo_id = sd_models.path_to_repo(repo_id) + + diffusers_load_config['variant'] = 'fp16' + diffusers_load_config['torch_dtype'] = devices.dtype + diffusers_load_config = model_quant.create_bnb_config(diffusers_load_config) + pipe = diffusers.SanaPAGPipeline.from_pretrained( + repo_id, + # pag_applied_layers=["transformer_blocks.8"], + cache_dir = shared.opts.diffusers_dir, + **diffusers_load_config, + ).to(devices.dtype) + if shared.opts.diffusers_eval: + pipe.text_encoder.eval() + pipe.transformer.eval() + + devices.torch_gc() + return pipe diff --git a/modules/model_te.py b/modules/model_te.py index 606cdb86d..8227ba3e8 100644 --- a/modules/model_te.py +++ b/modules/model_te.py @@ -52,7 +52,8 @@ def load_t5(name=None, cache_dir=None): if torch.is_floating_point(param) and not is_param_float8_e4m3fn: param = param.to(devices.dtype) set_module_tensor_to_device(t5, param_name, device=0, value=param) - t5.eval() + if shared.opts.diffusers_eval: + t5.eval() if t5.dtype != devices.dtype: try: t5 = t5.to(dtype=devices.dtype) diff --git a/modules/modeldata.py b/modules/modeldata.py index 604ff4623..4b7ec1776 100644 --- a/modules/modeldata.py +++ b/modules/modeldata.py @@ -3,6 +3,45 @@ import threading from modules import shared, errors +def get_model_type(pipe): + name = pipe.__class__.__name__ + if not shared.native: + model_type = 'ldm' + elif "StableDiffusion3" in name: + model_type = 'sd3' + elif "StableDiffusionXL" in name: + model_type = 'sdxl' + elif "StableDiffusion" in name: + model_type = 'sd' + elif "LatentConsistencyModel" in name: + model_type = 'sd' # lcm is compatible with sd + elif "InstaFlowPipeline" in name: + model_type = 'sd' # instaflow is compatible with sd + elif "AnimateDiffPipeline" in name: + model_type = 'sd' # animatediff is compatible with sd + elif "Kandinsky" in name: + model_type = 'kandinsky' + elif "HunyuanDiT" in name: + model_type = 'hunyuandit' + elif "Cascade" in name: + model_type = 'sc' + elif "AuraFlow" in name: + model_type = 'auraflow' + elif "Flux" in name: + model_type = 'f1' + elif "Lumina" in name: + model_type = 'lumina' + elif "OmniGen" in name: + model_type = 'omnigen' + elif "CogVideo" in name: + model_type = 'cogvideox' + elif "Sana" in name: + model_type = 'sana' + else: + model_type = name + return model_type + + class ModelData: def __init__(self): self.sd_model = None @@ -82,36 +121,7 @@ class Shared(sys.modules[__name__].__class__): if modules.sd_models.model_data.sd_model is None: model_type = 'none' return model_type - if not shared.native: - model_type = 'ldm' - elif "StableDiffusion3" in self.sd_model.__class__.__name__: - model_type = 'sd3' - elif "StableDiffusionXL" in self.sd_model.__class__.__name__: - model_type = 'sdxl' - elif "StableDiffusion" in self.sd_model.__class__.__name__: - model_type = 'sd' - elif "LatentConsistencyModel" in self.sd_model.__class__.__name__: - model_type = 'sd' # lcm is compatible with sd - elif "InstaFlowPipeline" in self.sd_model.__class__.__name__: - model_type = 'sd' # instaflow is compatible with sd - elif "AnimateDiffPipeline" in self.sd_model.__class__.__name__: - model_type = 'sd' # animatediff is compatible with sd - elif "Kandinsky" in self.sd_model.__class__.__name__: - model_type = 'kandinsky' - elif "HunyuanDiT" in self.sd_model.__class__.__name__: - model_type = 'hunyuandit' - elif "Cascade" in self.sd_model.__class__.__name__: - model_type = 'sc' - elif "AuraFlow" in self.sd_model.__class__.__name__: - model_type = 'auraflow' - elif "Flux" in self.sd_model.__class__.__name__: - model_type = 'f1' - elif "OmniGen" in self.sd_model.__class__.__name__: - model_type = 'omnigen' - elif "CogVideo" in self.sd_model.__class__.__name__: - model_type = 'cogvideox' - else: - model_type = self.sd_model.__class__.__name__ + model_type = get_model_type(self.sd_model) except Exception: model_type = 'unknown' return model_type @@ -123,18 +133,7 @@ class Shared(sys.modules[__name__].__class__): if modules.sd_models.model_data.sd_refiner is None: model_type = 'none' return model_type - if not shared.native: - model_type = 'ldm' - elif "StableDiffusion3" in self.sd_refiner.__class__.__name__: - model_type = 'sd3' - elif "StableDiffusionXL" in self.sd_refiner.__class__.__name__: - model_type = 'sdxl' - elif "StableDiffusion" in self.sd_refiner.__class__.__name__: - model_type = 'sd' - elif "Kandinsky" in self.sd_refiner.__class__.__name__: - model_type = 'kandinsky' - else: - model_type = self.sd_refiner.__class__.__name__ + model_type = get_model_type(self.sd_refiner) except Exception: model_type = 'unknown' return model_type diff --git a/modules/pag/__init__.py b/modules/pag/__init__.py index 29cdee8ca..8fe54c198 100644 --- a/modules/pag/__init__.py +++ b/modules/pag/__init__.py @@ -20,7 +20,9 @@ def apply(p: processing.StableDiffusionProcessing): # pylint: disable=arguments- if sd_models.get_diffusers_task(shared.sd_model) != sd_models.DiffusersTaskType.TEXT_2_IMAGE: shared.log.warning(f'PAG: pipeline={c} not implemented') return None - if detect.is_sd15(c): + if 'PAG' in shared.sd_model.__class__.__name__: + pass + elif detect.is_sd15(c): orig_pipeline = shared.sd_model shared.sd_model = sd_models.switch_pipe(StableDiffusionPAGPipeline, shared.sd_model) elif detect.is_sdxl(c): @@ -32,13 +34,14 @@ def apply(p: processing.StableDiffusionProcessing): # pylint: disable=arguments- p.task_args['pag_scale'] = p.pag_scale p.task_args['pag_adaptive_scaling'] = p.pag_adaptive + p.task_args['pag_adaptive_scale'] = p.pag_adaptive pag_applied_layers = shared.opts.pag_apply_layers pag_applied_layers_index = pag_applied_layers.split() if len(pag_applied_layers) > 0 else [] pag_applied_layers_index = [p.strip() for p in pag_applied_layers_index] p.task_args['pag_applied_layers_index'] = pag_applied_layers_index if len(pag_applied_layers_index) > 0 else ['m0'] # Available layers: d[0-5], m[0], u[0-8] p.extra_generation_params["PAG scale"] = p.pag_scale p.extra_generation_params["PAG adaptive"] = p.pag_adaptive - shared.log.debug(f'{c}: args={p.task_args}') + # shared.log.debug(f'{c}: args={p.task_args}') def unapply(): diff --git a/modules/processing_vae.py b/modules/processing_vae.py index 77a89c512..772a48adc 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -114,10 +114,11 @@ def full_vae_decode(latents, model): else: # manual upcast and we restore it later model.vae.orig_dtype = model.vae.dtype model.vae = model.vae.to(dtype=torch.float32) - latents = latents.to(torch.float32) latents = latents.to(devices.device) if getattr(model.vae, "post_quant_conv", None) is not None: latents = latents.to(next(iter(model.vae.post_quant_conv.parameters())).dtype) + else: + latents = latents.to(model.vae.dtype) # normalize latents latents_mean = model.vae.config.get("latents_mean", None) diff --git a/modules/schedulers/scheduler_dpm_flowmatch.py b/modules/schedulers/scheduler_dpm_flowmatch.py index 1afe54498..69452aca9 100644 --- a/modules/schedulers/scheduler_dpm_flowmatch.py +++ b/modules/schedulers/scheduler_dpm_flowmatch.py @@ -22,7 +22,8 @@ class BatchedBrownianTree: t0, t1, self.sign = self.sort(t0, t1) w0 = kwargs.get("w0", torch.zeros_like(x)) if seed is None: - seed = torch.randint(0, 2**63 - 1, []).item() + seed = [torch.randint(0, 2**63 - 1, []).item()] + seed = [s.initial_seed() if isinstance(s, torch.Generator) else s for s in seed] self.batched = True try: assert len(seed) == x.shape[0] diff --git a/modules/sd_detect.py b/modules/sd_detect.py index 071a83d7e..1931b9077 100644 --- a/modules/sd_detect.py +++ b/modules/sd_detect.py @@ -71,6 +71,8 @@ def detect_pipeline(f: str, op: str = 'model', warning=True, quiet=False): guess = 'Stable Cascade' if 'pixart-sigma' in f.lower(): guess = 'PixArt-Sigma' + if 'sana' in f.lower(): + guess = 'Sana' if 'lumina-next' in f.lower(): guess = 'Lumina-Next' if 'kolors' in f.lower(): diff --git a/modules/sd_models.py b/modules/sd_models.py index 5d42e314b..661559ae9 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -705,6 +705,9 @@ def load_diffuser_force(model_type, checkpoint_info, diffusers_load_config, op=' elif model_type in ['PixArt-Sigma']: # forced pipeline from modules.model_pixart import load_pixart sd_model = load_pixart(checkpoint_info, diffusers_load_config) + elif model_type in ['Sana']: # forced pipeline + from modules.model_sana import load_sana + sd_model = load_sana(checkpoint_info, diffusers_load_config) elif model_type in ['Lumina-Next']: # forced pipeline from modules.model_lumina import load_lumina sd_model = load_lumina(checkpoint_info, diffusers_load_config) diff --git a/modules/sd_samplers.py b/modules/sd_samplers.py index d8416e5d9..1b1be2d1a 100644 --- a/modules/sd_samplers.py +++ b/modules/sd_samplers.py @@ -3,6 +3,7 @@ import copy from modules import shared from modules.sd_samplers_common import samples_to_image_grid, sample_to_image # pylint: disable=unused-import + debug = shared.log.trace if os.environ.get('SD_SAMPLER_DEBUG', None) is not None else lambda *args, **kwargs: None debug('Trace: SAMPLER') all_samplers = [] @@ -75,15 +76,15 @@ def create_sampler(name, model): shared.log.debug(f'Sampler: sampler="{name}" config={config.options}') return sampler elif shared.native: - FlowModels = ['Flux', 'StableDiffusion3', 'Lumina', 'AuraFlow'] + FlowModels = ['Flux', 'StableDiffusion3', 'Lumina', 'AuraFlow', 'Sana'] if 'KDiffusion' in model.__class__.__name__: return None - if any(x in model.__class__.__name__ for x in FlowModels) and 'FlowMatch' not in name: - shared.log.warning(f'Sampler: default={current} target="{name}" class={model.__class__.__name__} linear scheduler unsupported') - return None if not any(x in model.__class__.__name__ for x in FlowModels) and 'FlowMatch' in name: shared.log.warning(f'Sampler: default={current} target="{name}" class={model.__class__.__name__} flow-match scheduler unsupported') return None + # if any(x in model.__class__.__name__ for x in FlowModels) and 'FlowMatch' not in name: + # shared.log.warning(f'Sampler: default={current} target="{name}" class={model.__class__.__name__} linear scheduler unsupported') + # return None sampler = config.constructor(model) if sampler is None: sampler = config.constructor(model) diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index 723f7b181..5297d0bcd 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -9,6 +9,7 @@ from modules import shared, devices, processing, images, sd_vae_approx, sd_vae_t SamplerData = namedtuple('SamplerData', ['name', 'constructor', 'aliases', 'options']) approximation_indexes = { "Simple": 0, "Approximate": 1, "TAESD": 2, "Full VAE": 3 } +flow_models = ['f1', 'sd3', 'lumina', 'auraflow', 'sana'] warned = False queue_lock = threading.Lock() diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index 7c23d4342..c95ae0858 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -4,13 +4,12 @@ import copy import inspect import diffusers from modules import shared, errors -from modules import sd_samplers_common +from modules.sd_samplers_common import SamplerData, flow_models debug = shared.log.trace if os.environ.get('SD_SAMPLER_DEBUG', None) is not None else lambda *args, **kwargs: None debug('Trace: SAMPLER') - try: from diffusers import ( CMStochasticIterativeScheduler, @@ -63,7 +62,7 @@ config = { # prediction_type is ideally set in model as well, but it maybe needed that we do auto-detect of model type in the future 'All': { 'num_train_timesteps': 1000, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'linear', 'prediction_type': 'epsilon' }, - 'UniPC': { 'predict_x0': True, 'sample_max_value': 1.0, 'solver_order': 2, 'solver_type': 'bh2', 'thresholding': False, 'use_beta_sigmas': False, 'use_exponential_sigmas': False, 'use_karras_sigmas': False, 'lower_order_final': True, 'timestep_spacing': 'linspace', 'final_sigmas_type': 'zero', 'rescale_betas_zero_snr': False }, + 'UniPC': { 'predict_x0': True, 'sample_max_value': 1.0, 'solver_order': 2, 'solver_type': 'bh2', 'thresholding': False, 'use_beta_sigmas': False, 'use_exponential_sigmas': False, 'use_flow_sigmas': False, 'use_karras_sigmas': False, 'lower_order_final': True, 'timestep_spacing': 'linspace', 'final_sigmas_type': 'zero', 'rescale_betas_zero_snr': False }, 'DDIM': { 'clip_sample': False, 'set_alpha_to_one': True, 'steps_offset': 0, 'clip_sample_range': 1.0, 'sample_max_value': 1.0, 'timestep_spacing': 'leading', 'rescale_betas_zero_snr': False, 'thresholding': False }, 'Euler': { 'steps_offset': 0, 'interpolation_type': "linear", 'rescale_betas_zero_snr': False, 'final_sigmas_type': 'zero', 'timestep_spacing': 'linspace', 'use_beta_sigmas': False, 'use_exponential_sigmas': False, 'use_karras_sigmas': False }, @@ -72,11 +71,11 @@ config = { 'Euler EDM': { 'sigma_schedule': "karras" }, 'Euler FlowMatch': { 'timestep_spacing': "linspace", 'shift': 1, 'use_dynamic_shifting': False, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False }, - 'DPM++': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False, 'final_sigmas_type': 'sigma_min' }, - 'DPM++ 1S': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False, 'use_lu_lambdas': False, 'final_sigmas_type': 'zero', 'timestep_spacing': 'linspace', 'solver_order': 1 }, - 'DPM++ 2M': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False, 'use_lu_lambdas': False, 'final_sigmas_type': 'zero', 'timestep_spacing': 'linspace', 'solver_order': 2 }, - 'DPM++ 3M': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False, 'use_lu_lambdas': False, 'final_sigmas_type': 'zero', 'timestep_spacing': 'linspace', 'solver_order': 3 }, - 'DPM++ 2M SDE': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "sde-dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False, 'use_lu_lambdas': False, 'final_sigmas_type': 'zero', 'timestep_spacing': 'linspace', 'solver_order': 2 }, + 'DPM++': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_flow_sigmas': False, 'use_beta_sigmas': False, 'final_sigmas_type': 'sigma_min' }, + 'DPM++ 1S': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_flow_sigmas': False, 'use_beta_sigmas': False, 'use_lu_lambdas': False, 'final_sigmas_type': 'zero', 'timestep_spacing': 'linspace', 'solver_order': 1 }, + 'DPM++ 2M': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_flow_sigmas': False, 'use_beta_sigmas': False, 'use_lu_lambdas': False, 'final_sigmas_type': 'zero', 'timestep_spacing': 'linspace', 'solver_order': 2 }, + 'DPM++ 3M': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_flow_sigmas': False, 'use_beta_sigmas': False, 'use_lu_lambdas': False, 'final_sigmas_type': 'zero', 'timestep_spacing': 'linspace', 'solver_order': 3 }, + 'DPM++ 2M SDE': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "sde-dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_flow_sigmas': False, 'use_beta_sigmas': False, 'use_lu_lambdas': False, 'final_sigmas_type': 'zero', 'timestep_spacing': 'linspace', 'solver_order': 2 }, 'DPM++ 2M EDM': { 'solver_order': 2, 'solver_type': 'midpoint', 'final_sigmas_type': 'zero', 'algorithm_type': 'dpmsolver++' }, 'DPM++ Cosine': { 'solver_order': 2, 'sigma_schedule': "exponential", 'prediction_type': "v-prediction" }, 'DPM SDE': { 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False, 'noise_sampler_seed': None, 'timestep_spacing': 'linspace', 'steps_offset': 0, }, @@ -92,8 +91,8 @@ config = { 'Heun': { 'use_beta_sigmas': False, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'timestep_spacing': 'linspace' }, 'Heun FlowMatch': { 'timestep_spacing': "linspace", 'shift': 1 }, - 'DEIS': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "deis", 'solver_type': "logrho", 'lower_order_final': True, 'timestep_spacing': 'linspace', 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False }, - 'SA Solver': {'predictor_order': 2, 'corrector_order': 2, 'thresholding': False, 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False, 'timestep_spacing': 'linspace'}, + 'DEIS': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "deis", 'solver_type': "logrho", 'lower_order_final': True, 'timestep_spacing': 'linspace', 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_flow_sigmas': False, 'use_beta_sigmas': False }, + 'SA Solver': {'predictor_order': 2, 'corrector_order': 2, 'thresholding': False, 'lower_order_final': True, 'use_karras_sigmas': False, 'use_flow_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False, 'timestep_spacing': 'linspace'}, 'DC Solver': { 'beta_start': 0.0001, 'beta_end': 0.02, 'solver_order': 2, 'prediction_type': "epsilon", 'thresholding': False, 'solver_type': 'bh2', 'lower_order_final': True, 'dc_order': 2, 'disable_corrector': [0] }, 'VDM Solver': { 'clip_sample_range': 2.0, }, 'LCM': { 'beta_start': 0.00085, 'beta_end': 0.012, 'beta_schedule': "scaled_linear", 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False, 'thresholding': False, 'timestep_spacing': 'linspace' }, @@ -110,54 +109,54 @@ config = { } samplers_data_diffusers = [ - sd_samplers_common.SamplerData('Default', None, [], {}), + SamplerData('Default', None, [], {}), - sd_samplers_common.SamplerData('UniPC', lambda model: DiffusionSampler('UniPC', UniPCMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DDIM', lambda model: DiffusionSampler('DDIM', DDIMScheduler, model), [], {}), - sd_samplers_common.SamplerData('Euler', lambda model: DiffusionSampler('Euler', EulerDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('Euler a', lambda model: DiffusionSampler('Euler a', EulerAncestralDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('Euler SGM', lambda model: DiffusionSampler('Euler SGM', EulerDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('Euler EDM', lambda model: DiffusionSampler('Euler EDM', EDMEulerScheduler, model), [], {}), - sd_samplers_common.SamplerData('Euler FlowMatch', lambda model: DiffusionSampler('Euler FlowMatch', FlowMatchEulerDiscreteScheduler, model), [], {}), + SamplerData('UniPC', lambda model: DiffusionSampler('UniPC', UniPCMultistepScheduler, model), [], {}), + SamplerData('DDIM', lambda model: DiffusionSampler('DDIM', DDIMScheduler, model), [], {}), + SamplerData('Euler', lambda model: DiffusionSampler('Euler', EulerDiscreteScheduler, model), [], {}), + SamplerData('Euler a', lambda model: DiffusionSampler('Euler a', EulerAncestralDiscreteScheduler, model), [], {}), + SamplerData('Euler SGM', lambda model: DiffusionSampler('Euler SGM', EulerDiscreteScheduler, model), [], {}), + SamplerData('Euler EDM', lambda model: DiffusionSampler('Euler EDM', EDMEulerScheduler, model), [], {}), + SamplerData('Euler FlowMatch', lambda model: DiffusionSampler('Euler FlowMatch', FlowMatchEulerDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM++', lambda model: DiffusionSampler('DPM++', DPMSolverSinglestepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM++ 1S', lambda model: DiffusionSampler('DPM++ 1S', DPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM++ 2M', lambda model: DiffusionSampler('DPM++ 2M', DPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM++ 3M', lambda model: DiffusionSampler('DPM++ 3M', DPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM++ 2M SDE', lambda model: DiffusionSampler('DPM++ 2M SDE', DPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM++ 2M EDM', lambda model: DiffusionSampler('DPM++ 2M EDM', EDMDPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM++ Cosine', lambda model: DiffusionSampler('DPM++ 2M EDM', CosineDPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM SDE', lambda model: DiffusionSampler('DPM SDE', DPMSolverSDEScheduler, model), [], {}), + SamplerData('DPM++', lambda model: DiffusionSampler('DPM++', DPMSolverSinglestepScheduler, model), [], {}), + SamplerData('DPM++ 1S', lambda model: DiffusionSampler('DPM++ 1S', DPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM++ 2M', lambda model: DiffusionSampler('DPM++ 2M', DPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM++ 3M', lambda model: DiffusionSampler('DPM++ 3M', DPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM++ 2M SDE', lambda model: DiffusionSampler('DPM++ 2M SDE', DPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM++ 2M EDM', lambda model: DiffusionSampler('DPM++ 2M EDM', EDMDPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM++ Cosine', lambda model: DiffusionSampler('DPM++ 2M EDM', CosineDPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM SDE', lambda model: DiffusionSampler('DPM SDE', DPMSolverSDEScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM2 FlowMatch', lambda model: DiffusionSampler('DPM2 FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM2a FlowMatch', lambda model: DiffusionSampler('DPM2a FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM2++ 2M FlowMatch', lambda model: DiffusionSampler('DPM2++ 2M FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM2++ 2S FlowMatch', lambda model: DiffusionSampler('DPM2++ 2S FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM2++ SDE FlowMatch', lambda model: DiffusionSampler('DPM2++ SDE FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM2++ 2M SDE FlowMatch', lambda model: DiffusionSampler('DPM2++ 2M SDE FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM2++ 3M SDE FlowMatch', lambda model: DiffusionSampler('DPM2++ 3M SDE FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM2 FlowMatch', lambda model: DiffusionSampler('DPM2 FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM2a FlowMatch', lambda model: DiffusionSampler('DPM2a FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM2++ 2M FlowMatch', lambda model: DiffusionSampler('DPM2++ 2M FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM2++ 2S FlowMatch', lambda model: DiffusionSampler('DPM2++ 2S FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM2++ SDE FlowMatch', lambda model: DiffusionSampler('DPM2++ SDE FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM2++ 2M SDE FlowMatch', lambda model: DiffusionSampler('DPM2++ 2M SDE FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM2++ 3M SDE FlowMatch', lambda model: DiffusionSampler('DPM2++ 3M SDE FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('Heun', lambda model: DiffusionSampler('Heun', HeunDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('Heun FlowMatch', lambda model: DiffusionSampler('Heun FlowMatch', FlowMatchHeunDiscreteScheduler, model), [], {}), + SamplerData('Heun', lambda model: DiffusionSampler('Heun', HeunDiscreteScheduler, model), [], {}), + SamplerData('Heun FlowMatch', lambda model: DiffusionSampler('Heun FlowMatch', FlowMatchHeunDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('DEIS', lambda model: DiffusionSampler('DEIS', DEISMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('SA Solver', lambda model: DiffusionSampler('SA Solver', SASolverScheduler, model), [], {}), - sd_samplers_common.SamplerData('DC Solver', lambda model: DiffusionSampler('DC Solver', DCSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('VDM Solver', lambda model: DiffusionSampler('VDM Solver', VDMScheduler, model), [], {}), - sd_samplers_common.SamplerData('BDIA DDIM', lambda model: DiffusionSampler('BDIA DDIM g=0', BDIA_DDIMScheduler, model), [], {}), + SamplerData('DEIS', lambda model: DiffusionSampler('DEIS', DEISMultistepScheduler, model), [], {}), + SamplerData('SA Solver', lambda model: DiffusionSampler('SA Solver', SASolverScheduler, model), [], {}), + SamplerData('DC Solver', lambda model: DiffusionSampler('DC Solver', DCSolverMultistepScheduler, model), [], {}), + SamplerData('VDM Solver', lambda model: DiffusionSampler('VDM Solver', VDMScheduler, model), [], {}), + SamplerData('BDIA DDIM', lambda model: DiffusionSampler('BDIA DDIM g=0', BDIA_DDIMScheduler, model), [], {}), - sd_samplers_common.SamplerData('PNDM', lambda model: DiffusionSampler('PNDM', PNDMScheduler, model), [], {}), - sd_samplers_common.SamplerData('IPNDM', lambda model: DiffusionSampler('IPNDM', IPNDMScheduler, model), [], {}), - sd_samplers_common.SamplerData('DDPM', lambda model: DiffusionSampler('DDPM', DDPMScheduler, model), [], {}), - sd_samplers_common.SamplerData('LMSD', lambda model: DiffusionSampler('LMSD', LMSDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('KDPM2', lambda model: DiffusionSampler('KDPM2', KDPM2DiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('KDPM2 a', lambda model: DiffusionSampler('KDPM2 a', KDPM2AncestralDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('CMSI', lambda model: DiffusionSampler('CMSI', CMStochasticIterativeScheduler, model), [], {}), + SamplerData('PNDM', lambda model: DiffusionSampler('PNDM', PNDMScheduler, model), [], {}), + SamplerData('IPNDM', lambda model: DiffusionSampler('IPNDM', IPNDMScheduler, model), [], {}), + SamplerData('DDPM', lambda model: DiffusionSampler('DDPM', DDPMScheduler, model), [], {}), + SamplerData('LMSD', lambda model: DiffusionSampler('LMSD', LMSDiscreteScheduler, model), [], {}), + SamplerData('KDPM2', lambda model: DiffusionSampler('KDPM2', KDPM2DiscreteScheduler, model), [], {}), + SamplerData('KDPM2 a', lambda model: DiffusionSampler('KDPM2 a', KDPM2AncestralDiscreteScheduler, model), [], {}), + SamplerData('CMSI', lambda model: DiffusionSampler('CMSI', CMStochasticIterativeScheduler, model), [], {}), - sd_samplers_common.SamplerData('LCM', lambda model: DiffusionSampler('LCM', LCMScheduler, model), [], {}), - sd_samplers_common.SamplerData('TCD', lambda model: DiffusionSampler('TCD', TCDScheduler, model), [], {}), + SamplerData('LCM', lambda model: DiffusionSampler('LCM', LCMScheduler, model), [], {}), + SamplerData('TCD', lambda model: DiffusionSampler('TCD', TCDScheduler, model), [], {}), - sd_samplers_common.SamplerData('Same as primary', None, [], {}), + SamplerData('Same as primary', None, [], {}), ] @@ -178,14 +177,14 @@ class DiffusionSampler: orig_config = model.default_scheduler.scheduler_config else: orig_config = model.default_scheduler.config - for key, value in config.get(name, {}).items(): # apply diffusers per-scheduler defaults - self.config[key] = value debug(f'Sampler: diffusers="{self.config}"') debug(f'Sampler: original="{orig_config}"') for key, value in orig_config.items(): # apply model defaults if key in self.config: self.config[key] = value debug(f'Sampler: default="{self.config}"') + for key, value in config.get(name, {}).items(): # apply diffusers per-scheduler defaults + self.config[key] = value for key, value in kwargs.items(): # apply user args, if any if key in self.config: self.config[key] = value @@ -205,10 +204,14 @@ class DiffusionSampler: if len(timesteps) == 0: if 'sigma_schedule' in self.config: self.config['sigma_schedule'] = shared.opts.schedulers_sigma if shared.opts.schedulers_sigma != 'default' else None - if shared.opts.schedulers_sigma == 'betas' and 'use_beta_sigmas' in self.config: + if shared.opts.schedulers_sigma == 'default' and shared.sd_model_type in flow_models and 'use_flow_sigmas' in self.config: + self.config['use_flow_sigmas'] = True + elif shared.opts.schedulers_sigma == 'betas' and 'use_beta_sigmas' in self.config: self.config['use_beta_sigmas'] = True elif shared.opts.schedulers_sigma == 'karras' and 'use_karras_sigmas' in self.config: self.config['use_karras_sigmas'] = True + elif shared.opts.schedulers_sigma == 'flowmatch' and 'use_flow_sigmas' in self.config: + self.config['use_flow_sigmas'] = True elif shared.opts.schedulers_sigma == 'exponential' and 'use_exponential_sigmas' in self.config: self.config['use_exponential_sigmas'] = True elif shared.opts.schedulers_sigma == 'lambdas' and 'use_lu_lambdas' in self.config: diff --git a/modules/shared_items.py b/modules/shared_items.py index 4d9b325c8..9abb64718 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -86,6 +86,14 @@ def get_pipelines(): 'Kolors': getattr(diffusers, 'KolorsPipeline', None), 'AuraFlow': getattr(diffusers, 'AuraFlowPipeline', None), 'CogView': getattr(diffusers, 'CogView3PlusPipeline', None), + 'Stable Cascade': getattr(diffusers, 'StableCascadeCombinedPipeline', None), + 'PixArt-Sigma': getattr(diffusers, 'PixArtSigmaPipeline', None), + 'HunyuanDiT': getattr(diffusers, 'HunyuanDiTPipeline', None), + 'Stable Diffusion 3': getattr(diffusers, 'StableDiffusion3Pipeline', None), + 'Stable Diffusion 3 Img2Img': getattr(diffusers, 'StableDiffusion3Img2ImgPipeline', None), + 'Lumina-Next': getattr(diffusers, 'LuminaText2ImgPipeline', None), + 'FLUX': getattr(diffusers, 'FluxPipeline', None), + 'Sana': getattr(diffusers, 'SanaPAGPipeline', None), } if hasattr(diffusers, 'OnnxStableDiffusionPipeline'): onnx_pipelines = { @@ -103,19 +111,10 @@ def get_pipelines(): pipelines.update(onnx_pipelines) # items that may rely on diffusers dev version - if hasattr(diffusers, 'StableCascadeCombinedPipeline'): - pipelines['Stable Cascade'] = getattr(diffusers, 'StableCascadeCombinedPipeline', None) - if hasattr(diffusers, 'PixArtSigmaPipeline'): - pipelines['PixArt-Sigma'] = getattr(diffusers, 'PixArtSigmaPipeline', None) - if hasattr(diffusers, 'HunyuanDiTPipeline'): - pipelines['HunyuanDiT'] = getattr(diffusers, 'HunyuanDiTPipeline', None) - if hasattr(diffusers, 'StableDiffusion3Pipeline'): - pipelines['Stable Diffusion 3'] = getattr(diffusers, 'StableDiffusion3Pipeline', None) - pipelines['Stable Diffusion 3 Img2Img'] = getattr(diffusers, 'StableDiffusion3Img2ImgPipeline', None) - if hasattr(diffusers, 'LuminaText2ImgPipeline'): - pipelines['Lumina-Next'] = getattr(diffusers, 'LuminaText2ImgPipeline', None) + """ if hasattr(diffusers, 'FluxPipeline'): pipelines['FLUX'] = getattr(diffusers, 'FluxPipeline', None) + """ for k, v in pipelines.items(): if k != 'Autodetect' and v is None: diff --git a/modules/ui_sections.py b/modules/ui_sections.py index f15edb4bd..fcf53cf70 100644 --- a/modules/ui_sections.py +++ b/modules/ui_sections.py @@ -276,11 +276,11 @@ def create_sampler_options(tabname): else: # shared.native with gr.Row(elem_classes=['flex-break']): - sampler_sigma = gr.Dropdown(label='Sigma method', elem_id=f"{tabname}_sampler_sigma", choices=['default', 'karras', 'betas', 'exponential', 'lambdas'], value=shared.opts.schedulers_sigma, type='value') + sampler_sigma = gr.Dropdown(label='Sigma method', elem_id=f"{tabname}_sampler_sigma", choices=['default', 'karras', 'betas', 'exponential', 'lambdas', 'flowmatch'], value=shared.opts.schedulers_sigma, type='value') sampler_spacing = gr.Dropdown(label='Timestep spacing', elem_id=f"{tabname}_sampler_spacing", choices=['default', 'linspace', 'leading', 'trailing'], value=shared.opts.schedulers_timestep_spacing, type='value') with gr.Row(elem_classes=['flex-break']): sampler_beta = gr.Dropdown(label='Beta schedule', elem_id=f"{tabname}_sampler_beta", choices=['default', 'linear', 'scaled', 'cosine'], value=shared.opts.schedulers_beta_schedule, type='value') - sampler_prediction = gr.Dropdown(label='Prediction method', elem_id=f"{tabname}_sampler_prediction", choices=['default', 'epsilon', 'sample', 'v_prediction'], value=shared.opts.schedulers_prediction_type, type='value') + sampler_prediction = gr.Dropdown(label='Prediction method', elem_id=f"{tabname}_sampler_prediction", choices=['default', 'epsilon', 'sample', 'v_prediction', 'flow_prediction'], value=shared.opts.schedulers_prediction_type, type='value') with gr.Row(elem_classes=['flex-break']): sampler_presets = gr.Dropdown(label='Timesteps presets', elem_id=f"{tabname}_sampler_presets", choices=['None', 'AYS SD15', 'AYS SDXL'], value='None', type='value') sampler_timesteps = gr.Textbox(label='Timesteps override', elem_id=f"{tabname}_sampler_timesteps", value=shared.opts.schedulers_timesteps)