diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a8d96737..e3838f32d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ - based on [VGen](https://huggingface.co/ali-vilab/i2vgen-xl) - **VQA** visual question & answer in interrogate - with support for multiple variations of base models: *GIT, BLIP, ViLT, PIX* +- **Second Pass / Refine** + - independent upscale and hires options: run hires without upscale or upscale without hires or both + - upscale can now run 0.1-8.0 scale and will also run if enabled at 1.0 to allow for upscalers that simply improve image quality + - update ui section to reflect changes + - *note*: behavior using backend:original is unchanged for backwards compatibilty - **Improvements** - **FaceID** extend support for LoRA, HyperTile and FreeU, thanks @Trojaner - **Tiling** now extends to both Unet and VAE producing smoother outputs, thanks @AI-Casanova diff --git a/TODO.md b/TODO.md index dab3a98fe..4e8741126 100644 --- a/TODO.md +++ b/TODO.md @@ -6,6 +6,7 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - defork - stable diffusion 3.0 +- EDMEulerScheduler: - stable cascade: - ipadapter masking: - x-adapter: diff --git a/modules/processing_class.py b/modules/processing_class.py index 05bfec177..6207317af 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -261,14 +261,14 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): self.hr_upscale_to_y = self.hr_resize_y self.truncate_x = (self.hr_upscale_to_x - target_w) // 8 self.truncate_y = (self.hr_upscale_to_y - target_h) // 8 - # special case: the user has chosen to do nothing - if (self.hr_upscale_to_x == self.width and self.hr_upscale_to_y == self.height) or self.hr_upscaler is None or self.hr_upscaler == 'None': - self.is_hr_pass = False - return - self.is_hr_pass = True - hypertile_set(self, hr=True) - shared.state.job_count = 2 * self.n_iter - shared.log.debug(f'Init hires: upscaler="{self.hr_upscaler}" sampler="{self.hr_sampler_name}" resize={self.hr_resize_x}x{self.hr_resize_y} upscale={self.hr_upscale_to_x}x{self.hr_upscale_to_y}') + if shared.backend == shared.Backend.ORIGINAL: # diffusers are handled in processing_diffusers + if (self.hr_upscale_to_x == self.width and self.hr_upscale_to_y == self.height) or self.hr_upscaler is None or self.hr_upscaler == 'None': # special case: the user has chosen to do nothing + self.is_hr_pass = False + return + self.is_hr_pass = True + hypertile_set(self, hr=True) + shared.state.job_count = 2 * self.n_iter + shared.log.debug(f'Init hires: upscaler="{self.hr_upscaler}" sampler="{self.hr_sampler_name}" resize={self.hr_resize_x}x{self.hr_resize_y} upscale={self.hr_upscale_to_x}x{self.hr_upscale_to_y}') def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts): from modules import processing_original diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index fc8f95c7a..b6d5c4c5e 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -388,6 +388,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing): use_denoise_start = not is_txt2img() and p.refiner_start > 0 and p.refiner_start < 1 shared.sd_model = update_pipeline(shared.sd_model, p) + shared.log.info(f'Base: class={shared.sd_model.__class__.__name__}') base_args = set_pipeline_args( model=shared.sd_model, prompts=p.prompts, @@ -448,54 +449,65 @@ def process_diffusers(p: processing.StableDiffusionProcessing): shared.sd_model = orig_pipeline return results - # optional hires pass - if p.enable_hr and getattr(p, 'hr_upscaler', 'None') != 'None' and len(getattr(p, 'init_images', [])) == 0: + # optional second pass + if p.enable_hr and len(getattr(p, 'init_images', [])) == 0: p.is_hr_pass = True - latent_scale_mode = shared.latent_upscale_modes.get(p.hr_upscaler, None) if (hasattr(p, "hr_upscaler") and p.hr_upscaler is not None) else shared.latent_upscale_modes.get(shared.latent_upscale_default_mode, "None") if p.is_hr_pass: p.init_hr() prev_job = shared.state.job - if hasattr(p, 'height') and hasattr(p, 'width') and (p.width != p.hr_upscale_to_x or p.height != p.hr_upscale_to_y): + + # upscale + if hasattr(p, 'height') and hasattr(p, 'width') and p.hr_upscaler is not None and p.hr_upscaler != 'None': + shared.log.info(f'Upscale: upscaler="{p.hr_upscaler}" resize={p.hr_resize_x}x{p.hr_resize_y} upscale={p.hr_upscale_to_x}x{p.hr_upscale_to_y}') p.ops.append('upscale') if shared.opts.save and not p.do_not_save_samples and shared.opts.save_images_before_highres_fix and hasattr(shared.sd_model, 'vae'): save_intermediate(latents=output.images, suffix="-before-hires") shared.state.job = 'upscale' output.images = resize_hires(p, latents=output.images) - if (latent_scale_mode is not None or p.hr_force) and p.denoising_strength > 0: - p.ops.append('hires') - sd_models_compile.openvino_recompile_model(p, hires=True, refiner=False) - shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) - if shared.sd_model.__class__.__name__ == "OnnxRawPipeline": - shared.sd_model = preprocess_onnx_pipeline(p) - update_sampler(shared.sd_model, second_pass=True) - hires_args = set_pipeline_args( - model=shared.sd_model, - prompts=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else p.prompts, - negative_prompts=[p.refiner_negative] if len(p.refiner_negative) > 0 else p.negative_prompts, - prompts_2=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else p.prompts, - negative_prompts_2=[p.refiner_negative] if len(p.refiner_negative) > 0 else p.negative_prompts, - num_inference_steps=calculate_hires_steps(p), - eta=shared.opts.scheduler_eta, - guidance_scale=p.image_cfg_scale if p.image_cfg_scale is not None else p.cfg_scale, - guidance_rescale=p.diffusers_guidance_rescale, - output_type='latent' if hasattr(shared.sd_model, 'vae') else 'np', - clip_skip=p.clip_skip, - image=output.images, - strength=p.denoising_strength, - desc='Hires', - ) - shared.state.job = 'hires' - shared.state.sampling_steps = hires_args['num_inference_steps'] - try: - sd_models_compile.check_deepcache(enable=True) - output = shared.sd_model(**hires_args) # pylint: disable=not-callable - if isinstance(output, dict): - output = SimpleNamespace(**output) - sd_models_compile.check_deepcache(enable=False) - sd_models_compile.openvino_post_compile(op="base") - except AssertionError as e: - shared.log.info(e) - p.init_images = [] + sd_hijack_hypertile.hypertile_set(p, hr=True) + + latent_upscale = shared.latent_upscale_modes.get(p.hr_upscaler, None) + if (latent_upscale is not None or p.hr_force) and p.denoising_strength > 0: + p.ops.append('hires') + sd_models_compile.openvino_recompile_model(p, hires=True, refiner=False) + if shared.sd_model.__class__.__name__ == "OnnxRawPipeline": + shared.sd_model = preprocess_onnx_pipeline(p) + p.hr_force = True + + # hires + if p.hr_force: + shared.state.job_count = 2 * p.n_iter + shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) + update_sampler(shared.sd_model, second_pass=True) + shared.log.info(f'HiRes: class={shared.sd_model.__class__.__name__} sampler="{p.hr_sampler_name}"') + hires_args = set_pipeline_args( + model=shared.sd_model, + prompts=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else p.prompts, + negative_prompts=[p.refiner_negative] if len(p.refiner_negative) > 0 else p.negative_prompts, + prompts_2=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else p.prompts, + negative_prompts_2=[p.refiner_negative] if len(p.refiner_negative) > 0 else p.negative_prompts, + num_inference_steps=calculate_hires_steps(p), + eta=shared.opts.scheduler_eta, + guidance_scale=p.image_cfg_scale if p.image_cfg_scale is not None else p.cfg_scale, + guidance_rescale=p.diffusers_guidance_rescale, + output_type='latent' if hasattr(shared.sd_model, 'vae') else 'np', + clip_skip=p.clip_skip, + image=output.images, + strength=p.denoising_strength, + desc='Hires', + ) + shared.state.job = 'hires' + shared.state.sampling_steps = hires_args['num_inference_steps'] + try: + sd_models_compile.check_deepcache(enable=True) + output = shared.sd_model(**hires_args) # pylint: disable=not-callable + if isinstance(output, dict): + output = SimpleNamespace(**output) + sd_models_compile.check_deepcache(enable=False) + sd_models_compile.openvino_post_compile(op="base") + except AssertionError as e: + shared.log.info(e) + p.init_images = [] shared.state.job = prev_job shared.state.nextjob() p.is_hr_pass = False @@ -529,6 +541,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing): image = processing_vae.vae_decode(latents=image, model=shared.sd_model, full_quality=p.full_quality, output_type='pil') p.extra_generation_params['Noise level'] = noise_level output_type = 'np' + shared.log.info(f'Refiner: class={shared.sd_refiner.__class__.__name__}') refiner_args = set_pipeline_args( model=shared.sd_refiner, prompts=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else p.prompts[i], diff --git a/modules/processing_helpers.py b/modules/processing_helpers.py index 0da7ab470..e9ecc43a6 100644 --- a/modules/processing_helpers.py +++ b/modules/processing_helpers.py @@ -352,7 +352,7 @@ def resize_hires(p, latents): # input=latents output=pil first_pass_images = processing_vae.vae_decode(latents=latents, model=shared.sd_model, full_quality=p.full_quality, output_type='pil') return first_pass_images latent_upscaler = shared.latent_upscale_modes.get(p.hr_upscaler, None) - shared.log.info(f'Hires: upscaler={p.hr_upscaler} width={p.hr_upscale_to_x} height={p.hr_upscale_to_y} images={latents.shape[0]}') + # shared.log.info(f'Hires: upscaler={p.hr_upscaler} width={p.hr_upscale_to_x} height={p.hr_upscale_to_y} images={latents.shape[0]}') if latent_upscaler is not None: latents = torch.nn.functional.interpolate(latents, size=(p.hr_upscale_to_y // 8, p.hr_upscale_to_x // 8), mode=latent_upscaler["mode"], antialias=latent_upscaler["antialias"]) first_pass_images = processing_vae.vae_decode(latents=latents, model=shared.sd_model, full_quality=p.full_quality, output_type='pil') diff --git a/modules/processing_info.py b/modules/processing_info.py index 4e1a860da..218e1accd 100644 --- a/modules/processing_info.py +++ b/modules/processing_info.py @@ -72,10 +72,10 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No args["Second pass"] = p.enable_hr args["Hires force"] = p.hr_force args["Hires steps"] = p.hr_second_pass_steps - args["Hires upscaler"] = p.hr_upscaler - args["Hires upscale"] = p.hr_scale - args["Hires resize"] = f"{p.hr_resize_x}x{p.hr_resize_y}" - args["Hires size"] = f"{p.hr_upscale_to_x}x{p.hr_upscale_to_y}" + args["Hires upscaler"] = p.hr_upscaler if p.hr_upscaler is not None and p.hr_upscaler != 'None' else None + args["Hires upscale"] = p.hr_scale if p.hr_upscaler is not None and p.hr_upscaler != 'None' else None + args["Hires resize"] = f"{p.hr_resize_x}x{p.hr_resize_y}" if p.hr_upscaler is not None and p.hr_upscaler != 'None' else None + args["Hires size"] = f"{p.hr_upscale_to_x}x{p.hr_upscale_to_y}" if p.hr_upscaler is not None and p.hr_upscaler != 'None' else None args["Denoising strength"] = p.denoising_strength args["Hires sampler"] = p.hr_sampler_name args["Image CFG scale"] = p.image_cfg_scale diff --git a/modules/ui_sections.py b/modules/ui_sections.py index 2d099eb4f..6035bcca7 100644 --- a/modules/ui_sections.py +++ b/modules/ui_sections.py @@ -185,24 +185,22 @@ def create_sampler_and_steps_selection(choices, tabname): def create_hires_inputs(tab): - with gr.Accordion(open=False, label="Second pass", elem_id=f"{tab}_second_pass", elem_classes=["small-accordion"]): + with gr.Accordion(open=False, label="Refine", elem_id=f"{tab}_second_pass", elem_classes=["small-accordion"]): with gr.Group(): with gr.Row(elem_id=f"{tab}_hires_row1"): enable_hr = gr.Checkbox(label='Enable second pass', value=False, elem_id=f"{tab}_enable_hr") - with gr.Row(elem_id=f"{tab}_hires_row2"): - hr_sampler_index = gr.Dropdown(label='Secondary sampler', elem_id=f"{tab}_sampling_alt", choices=[x.name for x in sd_samplers.samplers], value='Default', type="index") - denoising_strength = gr.Slider(minimum=0.0, maximum=0.99, step=0.01, label='Denoising strength', value=0.5, elem_id=f"{tab}_denoising_strength") - with gr.Row(elem_id=f"{tab}_hires_finalres", variant="compact"): - hr_final_resolution = gr.HTML(value="", elem_id=f"{tab}_hr_finalres", label="Upscaled resolution", interactive=False) with gr.Row(elem_id=f"{tab}_hires_fix_row1", variant="compact"): hr_upscaler = gr.Dropdown(label="Upscaler", elem_id=f"{tab}_hr_upscaler", choices=[*shared.latent_upscale_modes, *[x.name for x in shared.sd_upscalers]], value=shared.latent_upscale_default_mode) - hr_force = gr.Checkbox(label='Force Hires', value=False, elem_id=f"{tab}_hr_force") - with gr.Row(elem_id=f"{tab}_hires_fix_row2", variant="compact"): - hr_second_pass_steps = gr.Slider(minimum=0, maximum=99, step=1, label='Hires steps', elem_id=f"{tab}_steps_alt", value=20) - hr_scale = gr.Slider(minimum=1.0, maximum=8.0, step=0.05, label="Upscale by", value=2.0, elem_id=f"{tab}_hr_scale") + hr_scale = gr.Slider(minimum=0.1, maximum=8.0, step=0.05, label="Rescale by", value=2.0, elem_id=f"{tab}_hr_scale") with gr.Row(elem_id=f"{tab}_hires_fix_row3", variant="compact"): - hr_resize_x = gr.Slider(minimum=0, maximum=4096, step=8, label="Resize width to", value=0, elem_id=f"{tab}_hr_resize_x") - hr_resize_y = gr.Slider(minimum=0, maximum=4096, step=8, label="Resize height to", value=0, elem_id=f"{tab}_hr_resize_y") + hr_resize_x = gr.Slider(minimum=0, maximum=4096, step=8, label="Width resize", value=0, elem_id=f"{tab}_hr_resize_x") + hr_resize_y = gr.Slider(minimum=0, maximum=4096, step=8, label="Height resize", value=0, elem_id=f"{tab}_hr_resize_y") + with gr.Row(elem_id=f"{tab}_hires_fix_row2", variant="compact"): + hr_force = gr.Checkbox(label='Force HiRes', value=False, elem_id=f"{tab}_hr_force") + hr_sampler_index = gr.Dropdown(label='Secondary sampler', elem_id=f"{tab}_sampling_alt", choices=[x.name for x in sd_samplers.samplers], value='Default', type="index") + with gr.Row(elem_id=f"{tab}_hires_row2"): + hr_second_pass_steps = gr.Slider(minimum=0, maximum=99, step=1, label='HiRes steps', elem_id=f"{tab}_steps_alt", value=20) + denoising_strength = gr.Slider(minimum=0.0, maximum=0.99, step=0.01, label='Strength', value=0.5, elem_id=f"{tab}_denoising_strength") with gr.Group(visible=shared.backend == shared.Backend.DIFFUSERS): with gr.Row(elem_id=f"{tab}_refiner_row1", variant="compact"): refiner_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Refiner start', value=0.8, elem_id=f"{tab}_refiner_start") @@ -211,7 +209,7 @@ def create_hires_inputs(tab): refiner_prompt = gr.Textbox(value='', label='Secondary prompt', elem_id=f"{tab}_refiner_prompt") with gr.Row(elem_id="txt2img_refiner_row4", variant="compact"): refiner_negative = gr.Textbox(value='', label='Secondary negative prompt', elem_id=f"{tab}_refiner_neg_prompt") - return enable_hr, hr_sampler_index, denoising_strength, hr_final_resolution, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative + return enable_hr, hr_sampler_index, denoising_strength, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative def create_resize_inputs(tab, images, scale_visible=True, mode=None, accordion=True, latent=False): diff --git a/modules/ui_txt2img.py b/modules/ui_txt2img.py index 025d45c44..ca6920a77 100644 --- a/modules/ui_txt2img.py +++ b/modules/ui_txt2img.py @@ -47,22 +47,12 @@ def create_ui(): seed, reuse_seed, subseed, reuse_subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w = ui_sections.create_seed_inputs('txt2img') cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, sag_scale, cfg_end, full_quality, restore_faces, tiling = ui_sections.create_advanced_inputs('txt2img') hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundry, hdr_color_picker, hdr_tint_ratio, = ui_sections.create_correction_inputs('txt2img') - enable_hr, hr_sampler_index, denoising_strength, hr_final_resolution, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('txt2img') + enable_hr, hr_sampler_index, denoising_strength, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('txt2img') override_settings = ui_common.create_override_inputs('txt2img') with gr.Group(elem_id="txt2img_script_container"): txt2img_script_inputs = modules.scripts.scripts_txt2img.setup_ui(parent='txt2img', accordion=True) - hr_resolution_preview_inputs = [width, height, hr_scale, hr_resize_x, hr_resize_y, hr_upscaler] - for preview_input in hr_resolution_preview_inputs: - preview_input.change( - fn=calc_resolution_hires, - _js="onCalcResolutionHires", - inputs=hr_resolution_preview_inputs, - outputs=[hr_final_resolution], - show_progress=False, - ) - txt2img_gallery, txt2img_generation_info, txt2img_html_info, _txt2img_html_info_formatted, txt2img_html_log = ui_common.create_output_panel("txt2img", preview=True, prompt=None) ui_common.connect_reuse_seed(seed, reuse_seed, txt2img_generation_info, is_subseed=False) ui_common.connect_reuse_seed(subseed, reuse_subseed, txt2img_generation_info, is_subseed=True)