diff --git a/CHANGELOG.md b/CHANGELOG.md index bd3c14775..6f21380d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,7 @@ - add masking api endpoints GET:`/sdapi/v1/masking`, POST:`/sdapi/v1/mask`, sample script:`cli/simple-mask.py` - **Internal** + - **stable-fast** compatibility with torch 2.2.1 - remove obsolete textual inversion training code - remove obsolete hypernetworks training code - **Refiner** validated workflows: diff --git a/modules/control/run.py b/modules/control/run.py index 0c9abc76b..9a254e64b 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -27,9 +27,9 @@ def restore_pipeline(): global pipe, instance # pylint: disable=global-statement if instance is not None and hasattr(instance, 'restore'): instance.restore() - if original_pipeline is not None: + if original_pipeline is not None and (original_pipeline.__class__.__name__ != shared.sd_model.__class__.__name__): + shared.log.debug(f'Control restored pipeline: class={shared.sd_model.__class__.__name__} to={original_pipeline.__class__.__name__}') shared.sd_model = original_pipeline - shared.log.debug(f'Control restored pipeline: class={shared.sd_model.__class__.__name__}') pipe = None instance = None devices.torch_gc() @@ -44,6 +44,8 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ resize_mode_after, resize_name_after, width_after, height_after, scale_by_after, selected_scale_tab_after, resize_mode_mask, resize_name_mask, width_mask, height_mask, scale_by_mask, selected_scale_tab_mask, denoising_strength, batch_count, batch_size, + enable_hr, hr_sampler_index, hr_denoising_strength, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, + refiner_start, refiner_prompt, refiner_negative, video_skip_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, *input_script_args # pylint: disable=unused-argument ): @@ -66,13 +68,15 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ negative_prompt = negative, styles = styles, steps = steps, + n_iter = batch_count, + batch_size = batch_size, sampler_name = processing.get_sampler_name(sampler_index), - hr_sampler_name = processing.get_sampler_name(sampler_index), seed = seed, subseed = subseed, subseed_strength = subseed_strength, seed_resize_from_h = seed_resize_from_h, seed_resize_from_w = seed_resize_from_w, + # advanced cfg_scale = cfg_scale, clip_skip = clip_skip, image_cfg_scale = image_cfg_scale, @@ -81,29 +85,46 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ full_quality = full_quality, restore_faces = restore_faces, tiling = tiling, + # resize resize_mode = resize_mode_before if resize_name_before != 'None' else 0, resize_name = resize_name_before, scale_by = scale_by_before, selected_scale_tab = selected_scale_tab_before, denoising_strength = denoising_strength, - n_iter = batch_count, - batch_size = batch_size, + # inpaint inpaint_full_res = masking.opts.mask_only, - # inpaint_full_res_padding = masking.opts.mask_padding, inpainting_mask_invert = 1 if masking.opts.invert else 0, inpainting_fill = 1, + # hdr hdr_mode=hdr_mode, hdr_brightness=hdr_brightness, hdr_color=hdr_color, hdr_sharpen=hdr_sharpen, hdr_clamp=hdr_clamp, hdr_boundary=hdr_boundary, hdr_threshold=hdr_threshold, hdr_maximize=hdr_maximize, hdr_max_center=hdr_max_center, hdr_max_boundry=hdr_max_boundry, hdr_color_picker=hdr_color_picker, hdr_tint_ratio=hdr_tint_ratio, + # path outpath_samples=shared.opts.outdir_samples or shared.opts.outdir_control_samples, outpath_grids=shared.opts.outdir_grids or shared.opts.outdir_control_grids, ) processing.process_init(p) - + # set initial resolution if resize_mode_before != 0 or inputs is None or inputs == [None]: p.width, p.height = width_before, height_before # pylint: disable=attribute-defined-outside-init else: del p.width del p.height + # hires/refine defined outside of main init + p.enable_hr = enable_hr + p.hr_sampler_name = processing.get_sampler_name(hr_sampler_index) + p.hr_denoising_strength = hr_denoising_strength # TODO + p.hr_upscaler = hr_upscaler + p.hr_force = hr_force + p.hr_second_pass_steps = hr_second_pass_steps + p.hr_scale = hr_scale + p.hr_resize_x = hr_resize_x + p.hr_resize_y = hr_resize_y + p.refiner_steps = refiner_steps + p.refiner_start = refiner_start + p.refiner_prompt = refiner_prompt + p.refiner_negative = refiner_negative + if p.enable_hr and (p.hr_resize_x == 0 or p.hr_resize_y == 0): + p.hr_upscale_to_x, p.hr_upscale_to_y = 8 * int(p.width * p.hr_scale / 8), 8 * int(p.height * p.hr_scale / 8) t0 = time.time() num_units = 0 @@ -451,7 +472,7 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ p.init_images = [processed_image] shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) else: - p.init_hr() + p.init_hr(p.scale_by, p.resize_name) shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) elif has_models: # actual control p.is_control = True @@ -547,6 +568,7 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ else: image_str = [f'{image.width}x{image.height}' for image in output_images] image_txt = f'| Images {len(output_images)} | Size {" ".join(image_str)}' + p.init_images = output_images # may be used for hires if video_type != 'None' and isinstance(output_images, list): p.do_not_save_grid = True # pylint: disable=attribute-defined-outside-init @@ -554,7 +576,7 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ image_txt = f'| Frames {len(output_images)} | Size {output_images[0].width}x{output_images[0].height}' image_txt += f' | {util.dict2str(p.extra_generation_params)}' - # restore_pipeline() + restore_pipeline() debug(f'Control ready: {image_txt}') if is_generator: yield (output_images, processed_image, f'Control ready {image_txt}', output_filename) diff --git a/modules/processing_class.py b/modules/processing_class.py index 37b1610ce..d5fbe5551 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -1,5 +1,4 @@ import os -import math import hashlib from typing import Any, Dict, List from dataclasses import dataclass, field @@ -237,10 +236,12 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): self.width = self.width or 512 self.height = self.height or 512 - def init_hr(self): + def init_hr(self, scale = None, upscaler = None): + scale = scale or self.hr_scale + upscaler = upscaler or self.hr_upscaler if self.hr_resize_x == 0 and self.hr_resize_y == 0: - self.hr_upscale_to_x = int(self.width * self.hr_scale) - self.hr_upscale_to_y = int(self.height * self.hr_scale) + self.hr_upscale_to_x = int(self.width * scale) + self.hr_upscale_to_y = int(self.height * scale) else: if self.hr_resize_y == 0: self.hr_upscale_to_x = self.hr_resize_x @@ -262,7 +263,7 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): self.truncate_x = (self.hr_upscale_to_x - target_w) // 8 self.truncate_y = (self.hr_upscale_to_y - target_h) // 8 if shared.backend == shared.Backend.ORIGINAL: # diffusers are handled in processing_diffusers - if (self.hr_upscale_to_x == self.width and self.hr_upscale_to_y == self.height) or self.hr_upscaler is None or self.hr_upscaler == 'None': # special case: the user has chosen to do nothing + if (self.hr_upscale_to_x == self.width and self.hr_upscale_to_y == self.height) or upscaler is None or upscaler == 'None': # special case: the user has chosen to do nothing self.is_hr_pass = False return self.is_hr_pass = True @@ -283,6 +284,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): self.resize_mode: int = resize_mode self.resize_name: str = resize_name self.denoising_strength: float = denoising_strength + self.hr_denoising_strength: float = denoising_strength self.image_cfg_scale: float = image_cfg_scale self.init_latent = None self.image_mask = mask @@ -477,14 +479,15 @@ class StableDiffusionProcessingControl(StableDiffusionProcessingImg2Img): def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts): # abstract pass - def init_hr(self): - if self.resize_name == 'None' or self.scale_by == 1.0: + def init_hr(self, scale = None, upscaler = None): + scale = scale or self.scale_by + upscaler = upscaler or self.resize_name + if upscaler == 'None' or scale == 1.0: return self.is_hr_pass = True self.hr_force = True - self.hr_upscaler = self.resize_name - self.hr_upscale_to_x, self.hr_upscale_to_y = int(self.width * self.scale_by), int(self.height * self.scale_by) - self.hr_upscale_to_x, self.hr_upscale_to_y = 8 * math.ceil(self.hr_upscale_to_x / 8), 8 * math.ceil(self.hr_upscale_to_y / 8) + self.hr_upscaler = upscaler + self.hr_upscale_to_x, self.hr_upscale_to_y = 8 * int(self.width * scale / 8), 8 * int(self.height * scale / 8) # hypertile_set(self, hr=True) shared.state.job_count = 2 * self.n_iter - shared.log.debug(f'Control hires: upscaler="{self.hr_upscaler}" upscale={self.scale_by} size={self.hr_upscale_to_x}x{self.hr_upscale_to_y}') + shared.log.debug(f'Control hires: upscaler="{self.hr_upscaler}" upscale={scale} size={self.hr_upscale_to_x}x{self.hr_upscale_to_y}') diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 13c9b7b50..e1109c204 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -449,12 +449,15 @@ def process_diffusers(p: processing.StableDiffusionProcessing): return results # optional second pass - if p.enable_hr and len(getattr(p, 'init_images', [])) == 0: + if p.enable_hr: p.is_hr_pass = True - if p.is_hr_pass: - p.init_hr() + p.init_hr(p.hr_scale, p.hr_upscaler) prev_job = shared.state.job + # hires runs on original pipeline + if hasattr(shared.sd_model, 'restore_pipeline') and shared.sd_model.restore_pipeline is not None: + shared.sd_model.restore_pipeline() + # upscale if hasattr(p, 'height') and hasattr(p, 'width') and p.hr_upscaler is not None and p.hr_upscaler != 'None': shared.log.info(f'Upscale: upscaler="{p.hr_upscaler}" resize={p.hr_resize_x}x{p.hr_resize_y} upscale={p.hr_upscale_to_x}x{p.hr_upscale_to_y}') @@ -466,7 +469,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing): sd_hijack_hypertile.hypertile_set(p, hr=True) latent_upscale = shared.latent_upscale_modes.get(p.hr_upscaler, None) - if (latent_upscale is not None or p.hr_force) and p.denoising_strength > 0: + if (latent_upscale is not None or p.hr_force) and getattr(p, 'hr_denoising_strength', p.denoising_strength) > 0: p.ops.append('hires') sd_models_compile.openvino_recompile_model(p, hires=True, refiner=False) if shared.sd_model.__class__.__name__ == "OnnxRawPipeline": @@ -480,6 +483,8 @@ def process_diffusers(p: processing.StableDiffusionProcessing): update_sampler(shared.sd_model, second_pass=True) shared.log.info(f'HiRes: class={shared.sd_model.__class__.__name__} sampler="{p.hr_sampler_name}"') sd_models.move_model(shared.sd_model, devices.device) + orig_denoise = p.denoising_strength + p.denoising_strength = getattr(p, 'hr_denoising_strength', p.denoising_strength) hires_args = set_pipeline_args( model=shared.sd_model, prompts=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else p.prompts, @@ -507,7 +512,8 @@ def process_diffusers(p: processing.StableDiffusionProcessing): sd_models_compile.openvino_post_compile(op="base") except AssertionError as e: shared.log.info(e) - p.init_images = [] + p.denoising_strength = orig_denoise + # p.init_images = [] shared.state.job = prev_job shared.state.nextjob() p.is_hr_pass = False diff --git a/modules/ui_control.py b/modules/ui_control.py index 01f67c012..fc6c75e3a 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -124,6 +124,8 @@ def create_ui(_blocks: gr.Blocks=None): video_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False) video_type.change(fn=helpers.video_type_change, inputs=[video_type], outputs=[video_duration, video_loop, video_pad, video_interpolate]) + enable_hr, hr_sampler_index, hr_denoising_strength, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('txt2img') + with gr.Row(): override_settings = ui_common.create_override_inputs('control') @@ -501,6 +503,8 @@ def create_ui(_blocks: gr.Blocks=None): resize_mode_after, resize_name_after, width_after, height_after, scale_by_after, selected_scale_tab_after, resize_mode_mask, resize_name_mask, width_mask, height_mask, scale_by_mask, selected_scale_tab_mask, denoising_strength, batch_count, batch_size, + enable_hr, hr_sampler_index, hr_denoising_strength, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, + refiner_start, refiner_prompt, refiner_negative, video_skip_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, ] output_fields = [