diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fb09ed52..242b8365e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,7 +88,8 @@ We're back with another update with over 50 commits! - validate output before hires/refine - scheduler fix sigma index out of bounds - force pydantic version reinstall/reload - - multi-unit when using controlnet-union + - multi-unit when using controlnet-union + - pulid with hidiffusion ## Update for 2025-02-05 diff --git a/modules/hidiffusion/__init__.py b/modules/hidiffusion/__init__.py index 7ca7e253b..ac7dd9627 100644 --- a/modules/hidiffusion/__init__.py +++ b/modules/hidiffusion/__init__.py @@ -32,12 +32,14 @@ def apply(p, model_type): hidiffusion.switching_threshold_ratio_dict['sdxl_4096']['T2_ratio'] = t2 hidiffusion.switching_threshold_ratio_dict['sdxl_turbo_1024']['T2_ratio'] = t2 p.extra_generation_params['HiDiffusion Ratios'] = f'{shared.opts.hidiffusion_t1}/{shared.opts.hidiffusion_t2}' - hidiffusion.apply_hidiffusion(shared.sd_model, apply_raunet=shared.opts.hidiffusion_raunet, apply_window_attn=shared.opts.hidiffusion_attn, model_type=model_type) + pipe = shared.sd_model.pipe if hasattr(shared.sd_model, 'pipe') else shared.sd_model + hidiffusion.apply_hidiffusion(pipe, apply_raunet=shared.opts.hidiffusion_raunet, apply_window_attn=shared.opts.hidiffusion_attn, model_type=model_type, steps=p.steps) p.extra_generation_params['HiDiffusion'] = f'{shared.opts.hidiffusion_raunet}/{shared.opts.hidiffusion_attn}/{shared.opts.hidiffusion_steps > 0}:{shared.opts.hidiffusion_steps}' t1 = time.time() shared.log.debug(f'HiDiffusion apply: raunet={shared.opts.hidiffusion_raunet} attn={shared.opts.hidiffusion_attn} aggressive={shared.opts.hidiffusion_steps > 0}:{shared.opts.hidiffusion_steps} t1={shared.opts.hidiffusion_t1} t2={shared.opts.hidiffusion_t2} time={t1-t0:.2f} type={shared.sd_model_type} width={p.width} height={p.height}') def unapply(): - if hasattr(shared.sd_model, "unet"): - hidiffusion.remove_hidiffusion(shared.sd_model) + pipe = shared.sd_model.pipe if hasattr(shared.sd_model, 'pipe') else shared.sd_model + if hasattr(pipe, 'unet'): + hidiffusion.remove_hidiffusion(pipe) diff --git a/modules/hidiffusion/hidiffusion.py b/modules/hidiffusion/hidiffusion.py index d6f68eb15..c358c05af 100644 --- a/modules/hidiffusion/hidiffusion.py +++ b/modules/hidiffusion/hidiffusion.py @@ -6,6 +6,7 @@ from diffusers.utils.torch_utils import is_torch_version from diffusers.pipelines import auto_pipeline +current_steps = 50 def sd15_hidiffusion_key(): modified_key = dict() modified_key['down_module_key'] = ['down_blocks.0.downsamplers.0.conv'] @@ -163,12 +164,14 @@ def make_diffusers_transformer_block(block_class: Type[torch.nn.Module]) -> Type widow_size = (math.ceil(H/2), math.ceil(W/2)) if rand_num <= 0.25: shift_size = (0,0) - if rand_num > 0.25 and rand_num <= 0.5: + elif rand_num > 0.25 and rand_num <= 0.5: shift_size = (widow_size[0]//4, widow_size[1]//4) - if rand_num > 0.5 and rand_num <= 0.75: + elif rand_num > 0.5 and rand_num <= 0.75: shift_size = (widow_size[0]//4*2, widow_size[1]//4*2) - if rand_num > 0.75 and rand_num <= 1: + elif rand_num > 0.75 and rand_num <= 1: shift_size = (widow_size[0]//4*3, widow_size[1]//4*3) + else: + shift_size = (0,0) norm_hidden_states = window_partition(norm_hidden_states, widow_size, shift_size, H, W) # 1. Retrieve lora scale. @@ -261,7 +264,7 @@ def make_diffusers_cross_attn_down_block(block_class: Type[torch.nn.Module]) -> T1_start = 0 T1_end = 0 T1 = 0 # to avoid confict with sdxl-turbo - max_timestep = 50 + max_timestep = current_steps def forward( self, @@ -273,6 +276,8 @@ def make_diffusers_cross_attn_down_block(block_class: Type[torch.nn.Module]) -> encoder_attention_mask: Optional[torch.FloatTensor] = None, additional_residuals: Optional[torch.FloatTensor] = None, ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + if not hasattr(self.info['pipeline'], '_num_timesteps'): + self.info['pipeline']._num_timesteps = self.max_timestep # pylint: disable=protected-access self.max_timestep = self.info['pipeline']._num_timesteps # pylint: disable=protected-access # self.max_timestep = len(self.info['scheduler'].timesteps) ori_H, ori_W = self.info['size'] @@ -618,13 +623,15 @@ def apply_hidiffusion( model: torch.nn.Module, apply_raunet: bool = True, apply_window_attn: bool = True, - model_type: str = 'None'): + model_type: str = 'None', + steps: int = 50): """ model: diffusers model. We support SD 1.5, 2.1, XL, XL Turbo. apply_raunet: whether to apply RAU-Net apply_window_attn: whether to apply MSW-MSA. """ - + global current_steps # pylint: disable=global-statement + current_steps = steps if hasattr(model, 'controlnet'): from .hidiffusion_controlnet import make_diffusers_sdxl_contrtolnet_ppl, make_diffusers_unet_2d_condition make_ppl_fn = make_diffusers_sdxl_contrtolnet_ppl diff --git a/modules/pulid/pulid_sdxl.py b/modules/pulid/pulid_sdxl.py index 01ca660a7..0f4cf4f26 100644 --- a/modules/pulid/pulid_sdxl.py +++ b/modules/pulid/pulid_sdxl.py @@ -372,7 +372,7 @@ class StableDiffusionXLPuLIDPipeline: # sigmas sigmas = self.get_sigmas_karras(num_inference_steps).to(self.device) if image is not None and strength > 0: - _, num_inference_steps = self.pipe.get_timesteps(num_inference_steps, strength, self.device, None) # denoising_start disabled + _timesteps, num_inference_steps = self.pipe.get_timesteps(num_inference_steps, strength, self.device, None) # denoising_start disabled sigmas = sigmas[-(num_inference_steps + 1):].to(self.device) # shorten sigmas in i2i debug(f'PulID sigmas: sigmas={sigmas.shape} dtype={sigmas.dtype}') diff --git a/modules/ui_extensions.py b/modules/ui_extensions.py index 6df6de576..339a1e459 100644 --- a/modules/ui_extensions.py +++ b/modules/ui_extensions.py @@ -171,9 +171,6 @@ def install_extension_from_url(dirname, url, branch_name, search_text, sort_colu if ssh: args['env'] = {'GIT_SSH_COMMAND':ssh} shared.log.debug(f'GIT: {args}') - # from installer import run - # ssh_test = run('ssh', '-v -c chacha20-poly1305@openssh.com -T git@github.com') - # shared.log.debug('GIT SSH TEST', ssh_test) with git.Repo.clone_from(**args) as repo: repo.remote().fetch(verbose=True) for submodule in repo.submodules: