From e368472cbb7124cc58ae8d42633cbc559c9931b4 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 13 May 2026 09:46:39 +0200 Subject: [PATCH] add scale_noise and improve set_timesteps to multiple schedulers Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 8 ++-- installer.py | 2 +- javascript/progressBar.js | 16 ++++---- .../schedulers/perflow/scheduler_perflow.py | 21 ++++++++++ modules/schedulers/scheduler_bdia.py | 20 ++++++++++ modules/schedulers/scheduler_dc.py | 18 +++++++++ modules/schedulers/scheduler_dpm_flowmatch.py | 4 ++ modules/schedulers/scheduler_ersde.py | 6 ++- modules/schedulers/scheduler_flashflow.py | 6 ++- modules/schedulers/scheduler_tcd.py | 20 ++++++++++ modules/schedulers/scheduler_ufogen.py | 20 ++++++++++ .../schedulers/scheduler_unipc_flowmatch.py | 39 +++++++++++++++++++ modules/schedulers/scheduler_vdm.py | 18 +++++++++ modules/sd_models_compile.py | 1 + pipelines/anima/anima_lora.py | 2 +- requirements.txt | 2 +- 16 files changed, 187 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13ea2cc7b..0e3ab6ded 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Change Log for SD.Next -## Update for 2026-05-12 +## Update for 2026-05-13 -### Highlights for 2026-05-12 +### Highlights for 2026-05-13 *What's New?* - Image editing models now can work with multiple image inputs! @@ -15,7 +15,7 @@ For full details, see [ChangeLog](https://github.com/vladmandic/automatic/blob/m [ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) | [Sponsor](https://github.com/sponsors/vladmandic) -### Details for 2026-05-12 +### Details for 2026-05-13 - **Models** - [HiDream-O1-Image](https://huggingface.co/HiDream-ai/HiDream-O1-Image) pixel-level unified transformer model support @@ -108,6 +108,8 @@ For full details, see [ChangeLog](https://github.com/vladmandic/automatic/blob/m - `ipadapters` with offloading - `kanvas` outpaint - `network` preview handle invalid image + - `schedulers` improve *set_timesteps* handling + - `schedulers` improve *scale_noise* handling ## Update for 2026-04-28 diff --git a/installer.py b/installer.py index 525cc4400..1f658ae2e 100644 --- a/installer.py +++ b/installer.py @@ -494,7 +494,7 @@ def check_diffusers(): t_start = time.time() if args.skip_all: return - target_commit = "a851ce1058d5a465d7951687235cdaeac1978de2" # diffusers commit hash == 0.37.1.dev-0427 + target_commit = "015da50b40ee7a082ea8c17a8c43dff717c9653e" # diffusers commit hash == 0.37.1.dev-0427 # if args.use_rocm or args.use_zluda or args.use_directml: # sha = '043ab2520f6a19fce78e6e060a68dbc947edb9f9' # lock diffusers versions for now pkg = package_spec('diffusers') diff --git a/javascript/progressBar.js b/javascript/progressBar.js index 1ad591759..059b60800 100644 --- a/javascript/progressBar.js +++ b/javascript/progressBar.js @@ -116,7 +116,7 @@ function requestProgress(id_task = 'undefined', progressEl = null, galleryEl = n }; }; - const done = (ok = false) => { + const removeLivePreview = (ok = false) => { debug('taskEnd:', id_task); localStorage.removeItem('task'); setProgress(); @@ -142,7 +142,7 @@ function requestProgress(id_task = 'undefined', progressEl = null, galleryEl = n if (atEnd) atEnd(); }; - const start = (id_task, id_live_preview) => { // eslint-disable-line no-shadow + const startLivePreview = (id_task, id_live_preview) => { // eslint-disable-line no-shadow if (opts.live_preview_refresh_period === 0) return; const request_id = document.hidden ? -1 : id_live_preview; @@ -153,17 +153,17 @@ function requestProgress(id_task = 'undefined', progressEl = null, galleryEl = n hasStarted |= res.active; if (res.completed || (!res.active && (hasStarted || once))) { debug('progress', { end: res, reason: res.completed ? 'completed' : 'inactive' }); - if (!res.paused) done(true); // only abort if not paused + if (!res.paused) removeLivePreview(true); // only abort if not paused return; } if (elapsedFromStart > progressTimeout && !res.queued && res.progress === prevProgress) { debug('progress', { end: res, reason: 'progressSimeout' }); - if (!res.paused) done(false); // only abort if not paused + if (!res.paused) removeLivePreview(false); // only abort if not paused return; } if (elapsedFromStart > startTimeout && !res.queued && !res.active) { debug('progress', { end: res, reason: 'startTimeout' }); - if (!res.paused) done(false); // only abort if not paused + if (!res.paused) removeLivePreview(false); // only abort if not paused return; } if (res.progress !== prevProgress) { @@ -177,16 +177,16 @@ function requestProgress(id_task = 'undefined', progressEl = null, galleryEl = n id_live_preview = res.id_live_preview; } if (onProgress) onProgress(res); - setTimeout(() => start(id_task, id_live_preview), opts.live_preview_refresh_period || 500); + setTimeout(() => startLivePreview(id_task, id_live_preview), opts.live_preview_refresh_period || 500); }; const onProgressErrorHandler = (err) => { error('progress', { error: err }); - done(); + removeLivePreview(false); }; xhrPost('./internal/progress', { id_task, id_live_preview: request_id }, onProgressHandler, onProgressErrorHandler, false, 30000); }; debug('progress', { start: dateStart }); - start(id_task, 0); + startLivePreview(id_task, 0); } diff --git a/modules/schedulers/perflow/scheduler_perflow.py b/modules/schedulers/perflow/scheduler_perflow.py index e3a50feaf..3dcacc086 100644 --- a/modules/schedulers/perflow/scheduler_perflow.py +++ b/modules/schedulers/perflow/scheduler_perflow.py @@ -340,6 +340,27 @@ class PeRFlowScheduler(SchedulerMixin, ConfigMixin): return PeRFlowSchedulerOutput(prev_sample=prev_sample, pred_original_sample=None) + def scale_noise( + self, + sample: torch.FloatTensor, + timestep: Union[float, torch.FloatTensor], + noise: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + if noise is None: + noise = torch.randn_like(sample) + if not isinstance(timestep, torch.Tensor): + timestep = torch.tensor([timestep], device=sample.device) + else: + timestep = timestep.to(sample.device) + if timestep.ndim == 0: + timestep = timestep.unsqueeze(0) + if timestep.shape[0] != sample.shape[0]: + timestep = timestep.repeat(sample.shape[0]) + if torch.is_floating_point(timestep): + timestep = timestep.round().to(dtype=torch.long) + return self.add_noise(sample, noise, timestep) + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.add_noise def add_noise( self, diff --git a/modules/schedulers/scheduler_bdia.py b/modules/schedulers/scheduler_bdia.py index cffb5cb35..15b6b4caa 100644 --- a/modules/schedulers/scheduler_bdia.py +++ b/modules/schedulers/scheduler_bdia.py @@ -497,6 +497,26 @@ class BDIA_DDIMScheduler(SchedulerMixin, ConfigMixin): return DDIMSchedulerOutput(prev_sample=prev_sample, pred_original_sample=pred_original_sample) + def scale_noise( + self, + sample: torch.FloatTensor, + timestep: Union[float, torch.FloatTensor], + noise: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + if noise is None: + noise = torch.randn_like(sample) + if not isinstance(timestep, torch.Tensor): + timestep = torch.tensor([timestep], device=sample.device) + else: + timestep = timestep.to(sample.device) + if timestep.ndim == 0: + timestep = timestep.unsqueeze(0) + if timestep.shape[0] != sample.shape[0]: + timestep = timestep.repeat(sample.shape[0]) + if torch.is_floating_point(timestep): + timestep = timestep.round().to(dtype=torch.long) + return self.add_noise(sample, noise, timestep) + def add_noise( self, original_samples: torch.Tensor, diff --git a/modules/schedulers/scheduler_dc.py b/modules/schedulers/scheduler_dc.py index 992fc6531..4a352777a 100644 --- a/modules/schedulers/scheduler_dc.py +++ b/modules/schedulers/scheduler_dc.py @@ -1061,6 +1061,24 @@ class DCSolverMultistepScheduler(SchedulerMixin, ConfigMixin): """ return sample + def scale_noise( + self, + sample: torch.FloatTensor, + timestep: Union[float, torch.FloatTensor], + noise: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + if noise is None: + noise = torch.randn_like(sample) + if not isinstance(timestep, torch.Tensor): + timestep = torch.tensor([timestep], device=sample.device) + else: + timestep = timestep.to(sample.device) + if timestep.ndim == 0: + timestep = timestep.unsqueeze(0) + if timestep.shape[0] != sample.shape[0]: + timestep = timestep.repeat(sample.shape[0]) + return self.add_noise(sample, noise, timestep) + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.add_noise def add_noise( self, diff --git a/modules/schedulers/scheduler_dpm_flowmatch.py b/modules/schedulers/scheduler_dpm_flowmatch.py index d04705a89..3f6c6827c 100644 --- a/modules/schedulers/scheduler_dpm_flowmatch.py +++ b/modules/schedulers/scheduler_dpm_flowmatch.py @@ -295,6 +295,10 @@ class FlowMatchDPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin): else: num_inference_steps = len(sigmas) self.num_inference_steps = num_inference_steps + if isinstance(sigmas, torch.Tensor): + sigmas = sigmas.detach().cpu().numpy() + else: + sigmas = np.asarray(sigmas, dtype=np.float64) if self.config.sigma_schedule == "exponential": if self.use_beta_sigmas: diff --git a/modules/schedulers/scheduler_ersde.py b/modules/schedulers/scheduler_ersde.py index a81fe14df..d42cfb38d 100644 --- a/modules/schedulers/scheduler_ersde.py +++ b/modules/schedulers/scheduler_ersde.py @@ -226,7 +226,11 @@ class ERSDEScheduler(SchedulerMixin, ConfigMixin): def set_timesteps(self, num_inference_steps: Optional[int] = None, device: Union[str, torch.device] = None, timesteps: Optional[List[int]] = None, sigmas: Optional[List[float]] = None, mu: Optional[float] = None): if sigmas is not None: # Flow-matching path: sigmas provided externally - sigmas = np.array(sigmas, dtype=np.float64) if not isinstance(sigmas, np.ndarray) else sigmas.astype(np.float64) + if isinstance(sigmas, torch.Tensor): + sigmas = sigmas.detach().cpu().numpy() + elif not isinstance(sigmas, np.ndarray): + sigmas = np.asarray(sigmas, dtype=np.float64) + sigmas = sigmas.astype(np.float64, copy=False) self.num_inference_steps = len(sigmas) sigmas = torch.from_numpy(sigmas).to(dtype=torch.float64, device=device) self._setup_flow(sigmas, device, mu) diff --git a/modules/schedulers/scheduler_flashflow.py b/modules/schedulers/scheduler_flashflow.py index f7df144d2..7d42b4edd 100644 --- a/modules/schedulers/scheduler_flashflow.py +++ b/modules/schedulers/scheduler_flashflow.py @@ -208,7 +208,11 @@ class FlashFlowMatchEulerDiscreteScheduler(SchedulerMixin, ConfigMixin): sigmas = timesteps / self.config.num_train_timesteps else: - sigmas = np.array(sigmas).astype(np.float32) + if isinstance(sigmas, torch.Tensor): + sigmas = sigmas.detach().cpu().numpy() + else: + sigmas = np.asarray(sigmas, dtype=np.float32) + sigmas = sigmas.astype(np.float32, copy=False) num_inference_steps = len(sigmas) self.num_inference_steps = num_inference_steps diff --git a/modules/schedulers/scheduler_tcd.py b/modules/schedulers/scheduler_tcd.py index 83099217d..772602fc1 100644 --- a/modules/schedulers/scheduler_tcd.py +++ b/modules/schedulers/scheduler_tcd.py @@ -594,6 +594,26 @@ class TCDScheduler(SchedulerMixin, ConfigMixin): return TCDSchedulerOutput(prev_sample=prev_sample, pred_noised_sample=pred_noised_sample) + def scale_noise( + self, + sample: torch.FloatTensor, + timestep: Union[float, torch.FloatTensor], + noise: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + if noise is None: + noise = torch.randn_like(sample) + if not isinstance(timestep, torch.Tensor): + timestep = torch.tensor([timestep], device=sample.device) + else: + timestep = timestep.to(sample.device) + if timestep.ndim == 0: + timestep = timestep.unsqueeze(0) + if timestep.shape[0] != sample.shape[0]: + timestep = timestep.repeat(sample.shape[0]) + if torch.is_floating_point(timestep): + timestep = timestep.round().to(dtype=torch.long) + return self.add_noise(sample, noise, timestep) + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.add_noise def add_noise( self, diff --git a/modules/schedulers/scheduler_ufogen.py b/modules/schedulers/scheduler_ufogen.py index ff5f27eb3..908d9f95d 100644 --- a/modules/schedulers/scheduler_ufogen.py +++ b/modules/schedulers/scheduler_ufogen.py @@ -458,6 +458,26 @@ class UFOGenScheduler(SchedulerMixin, ConfigMixin): return UFOGenSchedulerOutput(prev_sample=pred_prev_sample, pred_original_sample=pred_original_sample) + def scale_noise( + self, + sample: torch.FloatTensor, + timestep: Union[float, torch.FloatTensor], + noise: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + if noise is None: + noise = torch.randn_like(sample) + if not isinstance(timestep, torch.Tensor): + timestep = torch.tensor([timestep], device=sample.device) + else: + timestep = timestep.to(sample.device) + if timestep.ndim == 0: + timestep = timestep.unsqueeze(0) + if timestep.shape[0] != sample.shape[0]: + timestep = timestep.repeat(sample.shape[0]) + if torch.is_floating_point(timestep): + timestep = timestep.round().to(dtype=torch.long) + return self.add_noise(sample, noise, timestep) + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.add_noise def add_noise( self, diff --git a/modules/schedulers/scheduler_unipc_flowmatch.py b/modules/schedulers/scheduler_unipc_flowmatch.py index d981ac8bd..f94c4cab2 100644 --- a/modules/schedulers/scheduler_unipc_flowmatch.py +++ b/modules/schedulers/scheduler_unipc_flowmatch.py @@ -181,6 +181,10 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin): sigmas = np.linspace(self.sigma_max, self.sigma_min, num_inference_steps + 1).copy()[:-1] # pyright: ignore + elif isinstance(sigmas, torch.Tensor): + sigmas = sigmas.detach().cpu().numpy() + else: + sigmas = np.asarray(sigmas, dtype=np.float32) if self.config.use_dynamic_shifting: sigmas = self.time_shift(mu, 1.0, sigmas) # pyright: ignore @@ -758,6 +762,41 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin): """ return sample + def scale_noise( + self, + sample: torch.FloatTensor, + timestep: Union[float, torch.FloatTensor], + noise: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + """Forward process in flow-matching.""" + sigmas = self.sigmas.to(device=sample.device, dtype=sample.dtype) + + if sample.device.type == "mps" and torch.is_floating_point(timestep): + # mps does not support float64 + schedule_timesteps = self.timesteps.to(sample.device, dtype=torch.float32) + timestep = timestep.to(sample.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(sample.device) + timestep = timestep.to(sample.device) + + if self.begin_index is None: + step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timestep] + elif self.step_index is not None: + # add_noise is called after first denoising step (for inpainting) + step_indices = [self.step_index] * timestep.shape[0] + else: + # add noise is called before first denoising step to create initial latent(img2img) + step_indices = [self.begin_index] * timestep.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(sample.shape): + sigma = sigma.unsqueeze(-1) + + if noise is None: + noise = torch.randn_like(sample) + + return sigma * noise + (1.0 - sigma) * sample + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.add_noise def add_noise( self, diff --git a/modules/schedulers/scheduler_vdm.py b/modules/schedulers/scheduler_vdm.py index 35aab6e41..64885ee30 100644 --- a/modules/schedulers/scheduler_vdm.py +++ b/modules/schedulers/scheduler_vdm.py @@ -386,6 +386,24 @@ class VDMScheduler(SchedulerMixin, ConfigMixin): return VDMSchedulerOutput(prev_sample=pred_prev_sample, pred_original_sample=pred_original_sample) + def scale_noise( + self, + sample: torch.Tensor, + timestep: Union[float, torch.Tensor], + noise: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if noise is None: + noise = torch.randn_like(sample) + if not isinstance(timestep, torch.Tensor): + timestep = torch.tensor([timestep], device=sample.device, dtype=sample.dtype) + else: + timestep = timestep.to(device=sample.device) + if timestep.ndim == 0: + timestep = timestep.unsqueeze(0) + if timestep.shape[0] != sample.shape[0]: + timestep = timestep.repeat(sample.shape[0]) + return self.add_noise(sample, noise, timestep) + def add_noise(self, original_samples: torch.Tensor, noise: torch.Tensor, timesteps: torch.Tensor) -> torch.Tensor: """ Adds noise to the original samples according to the noise schedule and the specified timesteps. diff --git a/modules/sd_models_compile.py b/modules/sd_models_compile.py index 13ee32317..411b2f912 100644 --- a/modules/sd_models_compile.py +++ b/modules/sd_models_compile.py @@ -216,6 +216,7 @@ def compile_torch(sd_model, apply_to_components=True, op="Model"): # configure torch.dynamo if hasattr(torch, '_logging'): torch._logging.set_logs(dynamo=log_level, aot=log_level, inductor=log_level) # pylint: disable=protected-access + setup_logging() # dynamo messes with logging so reset is needed torch._dynamo.config.verbose = verbose # pylint: disable=protected-access torch._dynamo.config.suppress_errors = not verbose # pylint: disable=protected-access if 'dynamic' in shared.opts.cuda_compile_options: diff --git a/pipelines/anima/anima_lora.py b/pipelines/anima/anima_lora.py index d75c40249..8454425de 100644 --- a/pipelines/anima/anima_lora.py +++ b/pipelines/anima/anima_lora.py @@ -114,7 +114,7 @@ def try_load_lora(name, network_on_disk, lora_scale): matched += 1 if matched == 0: return None - log.debug(f'Network load: type=LoRA name="{name}" native modules={matched} unmatched={unmatched} scale={lora_scale}') + log.debug(f'Network load: type=LoRA name="{name}" method=native modules={matched} unmatched={unmatched} scale={lora_scale}') if unmatched > 0 and l.debug: log.debug(f'Network load: type=LoRA name="{name}" unmatched_samples={unmatched_samples}') l.timer.activate += time.time() - t0 diff --git a/requirements.txt b/requirements.txt index ddbd65292..f1391e86a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ ftfy # versioned fastapi==0.124.4 rich==14.1.0 -safetensors==0.7.0 +safetensors==0.8.0rc0 peft==0.19.1 httpx==0.28.1 requests==2.32.3