diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index 0b4c7dfe1..c05529ee9 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -105,6 +105,24 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {} shared.state.current_latent = pipe._unpack_latents(kwargs['latents'], height, width, pipe.vae_scale_factor) # pylint: disable=protected-access else: shared.state.current_latent = kwargs['latents'] + + if hasattr(pipe, "scheduler") and hasattr(pipe.scheduler, "sigmas"): + noise_pred = None + if kwargs.get("noise_pred", None) is not None: + noise_pred = kwargs.get("noise_pred") + elif kwargs.get("predicted_image_embedding", None) is not None: + noise_pred = kwargs.get("predicted_image_embedding") + if noise_pred is not None: + sigma = pipe.scheduler.sigmas[step] + sigma_next = pipe.scheduler.sigmas[step + 1] + original_sample = shared.state.current_latent - (noise_pred * (sigma_next-sigma)) + if "flow" in pipe.scheduler.__class__.__name__.lower(): + shared.state.current_latent = original_sample - (noise_pred * sigma) + elif hasattr(pipe.scheduler, "config") and hasattr(pipe.scheduler.config, "prediction_type"): + if pipe.scheduler.config.prediction_type == "epsilon": + shared.state.current_latent = original_sample - (noise_pred * sigma) + elif pipe.scheduler.config.prediction_type == "v_prediction": + shared.state.current_latent = noise_pred * (-sigma / (sigma**2 + 1) ** 0.5) + (original_sample / (sigma**2 + 1)) except Exception as e: shared.log.error(f'Callback: {e}') if shared.cmd_opts.profile and shared.profiler is not None: diff --git a/modules/sd_models.py b/modules/sd_models.py index 087e8ecfc..fa6398d3e 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -221,6 +221,7 @@ def copy_diffuser_options(new_pipe, orig_pipe): new_pipe.is_sdxl = getattr(orig_pipe, 'is_sdxl', False) # a1111 compatibility item new_pipe.is_sd2 = getattr(orig_pipe, 'is_sd2', False) new_pipe.is_sd1 = getattr(orig_pipe, 'is_sd1', True) + add_noise_pred_to_diffusers_callback(new_pipe) if new_pipe.has_accelerate: set_accelerate(new_pipe) @@ -975,6 +976,8 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No if model_type not in ['Stable Cascade']: # need a special-case sd_unet.load_unet(sd_model) + add_noise_pred_to_diffusers_callback(sd_model) + timer.record("load") if op == 'refiner': @@ -1276,9 +1279,11 @@ def set_diffuser_pipe(pipe, new_pipe_type): new_pipe.is_sd1 = getattr(pipe, 'is_sd1', True) if hasattr(new_pipe, 'watermark'): new_pipe.watermark = NoWatermark() + add_noise_pred_to_diffusers_callback(new_pipe) if hasattr(new_pipe, 'pipe'): # also handle nested pipelines new_pipe.pipe = set_diffuser_pipe(new_pipe.pipe, new_pipe_type) + add_noise_pred_to_diffusers_callback(new_pipe.pipe) fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access shared.log.debug(f"Pipeline class change: original={cls} target={new_pipe.__class__.__name__} device={pipe.device} fn={fn}") # pylint: disable=protected-access @@ -1333,6 +1338,16 @@ def set_diffusers_attention(pipe): pipe.current_attn_name = shared.opts.cross_attention_optimization +def add_noise_pred_to_diffusers_callback(pipe): + if pipe.__class__.__name__.startswith("StableDiffusion"): + pipe._callback_tensor_inputs.append("noise_pred") + elif pipe.__class__.__name__.startswith("StableCascade"): + pipe.prior_pipe._callback_tensor_inputs.append("predicted_image_embedding") + elif hasattr(pipe, "scheduler") and "flow" in pipe.scheduler.__class__.__name__.lower(): + pipe._callback_tensor_inputs.append("noise_pred") + return pipe + + def get_native(pipe: diffusers.DiffusionPipeline): if hasattr(pipe, "vae") and hasattr(pipe.vae.config, "sample_size"): # Stable Diffusion