diff --git a/CHANGELOG.md b/CHANGELOG.md index 363d3974a..beebef73e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ And it also includes fixes for all reported issues so far - add **outpaint** support applies to both *img2img* and *controlnet* workflows *note*: increase denoising strength since outpainted area is blank by default + - allow **resize** both *before* and *after* generate operation + this allows for workflows such as: *image -> upscale or downscale -> generate -> upscale or downscale -> output* + providing more flexibility and than standard hires workflow + *note*: resizing before generate can be done using standard upscalers or latent - add **marigold** depth map processor this is state-of-the-art depth estimation model, but its quite heavy on resources - add **openpose xl** controlnet @@ -32,6 +36,8 @@ And it also includes fixes for all reported issues so far - faster json parsing - **offline deployment**: allow deployment without git clone for example, you can now deploy a zip of the sdnext folder + - **latent upscale**: updated latent upscalers (some are new) + *nearest, nearest-exact, area, bilinear, bicubic, bilinear-antialias, bicubic-antialias* - **xyz grid**: continue on error now you can use xyz grid with different params and test which ones work and which dont - **hypertile** diff --git a/modules/control/run.py b/modules/control/run.py index 1c065c4ff..056090494 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -1,6 +1,5 @@ import os import time -import math from typing import List, Union import cv2 import numpy as np @@ -66,7 +65,8 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w, cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, sag_scale, full_quality, restore_faces, tiling, hdr_clamp, hdr_boundary, hdr_threshold, hdr_center, hdr_channel_shift, hdr_full_shift, hdr_maximize, hdr_max_center, hdr_max_boundry, - resize_mode, resize_name, width, height, scale_by, selected_scale_tab, resize_time, + resize_mode_before, resize_name_before, width_before, height_before, scale_by_before, selected_scale_tab_before, + resize_mode_after, resize_name_after, width_after, height_after, scale_by_after, selected_scale_tab_after, denoising_strength, batch_count, batch_size, mask_blur, mask_overlap, video_skip_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, ip_adapter, ip_scale, ip_image, ip_type, @@ -82,8 +82,6 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ active_start: List[float] = [] # start step for all active models active_end: List[float] = [] # end step for all active models processed_image: Image.Image = None # last processed image - width = 8 * math.ceil(width / 8) - height = 8 * math.ceil(height / 8) if mask is not None and input_type == 0: input_type = 1 # inpaint always requires control_image @@ -116,10 +114,10 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ hdr_maximize = hdr_maximize, hdr_max_center = hdr_max_center, hdr_max_boundry = hdr_max_boundry, - resize_mode = resize_mode if resize_name != 'None' else 0, - resize_name = resize_name, - scale_by = scale_by, - selected_scale_tab = selected_scale_tab, + resize_mode = resize_mode_before if resize_name_before != 'None' else 0, + resize_name = resize_name_before, + scale_by = scale_by_before, + selected_scale_tab = selected_scale_tab_before, denoising_strength = denoising_strength, n_iter = batch_count, batch_size = batch_size, @@ -129,9 +127,9 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ ) processing.process_init(p) - if resize_mode != 0 or inputs is None or inputs == [None]: - p.width = width # pylint: disable=attribute-defined-outside-init - p.height = height # pylint: disable=attribute-defined-outside-init + if resize_mode_before != 0 or inputs is None or inputs == [None]: + p.width = width_before # pylint: disable=attribute-defined-outside-init + p.height = height_before # pylint: disable=attribute-defined-outside-init else: del p.width del p.height @@ -342,15 +340,15 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ if video is not None and index % (video_skip_frames + 1) != 0: continue - # resize - if p.resize_mode != 0 and input_image is not None: - p.extra_generation_params["Control resize"] = f'{resize_time}: {resize_name}' - if selected_scale_tab == 1: - width = int(input_image.width * scale_by) - height = int(input_image.height * scale_by) - if p.resize_mode != 0 and input_image is not None and resize_time == 'Before': - debug(f'Control resize: image={input_image} width={width} height={height} mode={p.resize_mode} name={resize_name} sequence={resize_time}') - input_image = images.resize_image(p.resize_mode, input_image, width, height, resize_name) + # resize before + if resize_mode_before != 0 and resize_name_before != 'None': + if selected_scale_tab_before == 1: + width_before = int(input_image.width * scale_by_before) + height_before = int(input_image.height * scale_by_before) + if input_image is not None: + p.extra_generation_params["Control resize"] = f'{resize_name_before}' + debug(f'Control resize: op=before image={input_image} width={width_before} height={height_before} mode={resize_mode_before} name={resize_name_before}') + input_image = images.resize_image(resize_mode_before, input_image, width_before, height_before, resize_name_before) if input_image is not None: p.width = input_image.width p.height = input_image.height @@ -485,10 +483,14 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ if output is not None and len(output) > 0: output_image = output[0] if output_image is not None: - # resize - if p.resize_mode != 0 and resize_time == 'After': - debug(f'Control resize: image={input_image} width={width} height={height} mode={p.resize_mode} name={resize_name} sequence={resize_time}') - output_image = images.resize_image(p.resize_mode, output_image, width, height, resize_name) + + # resize after + if selected_scale_tab_after == 1: + width_after = int(output_image.width * scale_by_after) + height_after = int(output_image.height * scale_by_after) + if resize_mode_after != 0 and resize_name_after != 'None': + debug(f'Control resize: op=after image={output_image} width={width_after} height={height_after} mode={resize_mode_after} name={resize_name_after}') + output_image = images.resize_image(resize_mode_after, output_image, width_after, height_after, resize_name_after) elif hasattr(p, 'width') and hasattr(p, 'height'): output_image = output_image.resize((p.width, p.height), Image.Resampling.LANCZOS) diff --git a/modules/images.py b/modules/images.py index 87a6b43d0..e117b6860 100644 --- a/modules/images.py +++ b/modules/images.py @@ -226,18 +226,29 @@ def resize_image(resize_mode, im, width, height, upscaler_name=None, output_type """ upscaler_name = upscaler_name or shared.opts.upscaler_for_img2img + def latent(im, w, h, upscaler): + from modules.processing_vae import vae_encode, vae_decode + import torch + latents = vae_encode(im, shared.sd_model, full_quality=False) # TODO enable full VAE mode + latents = torch.nn.functional.interpolate(latents, size=(h // 8, w // 8), mode=upscaler["mode"], antialias=upscaler["antialias"]) + im = vae_decode(latents, shared.sd_model, output_type='pil', full_quality=False)[0] + return im + def resize(im, w, h): if upscaler_name is None or upscaler_name == "None" or im.mode == 'L': return im.resize((w, h), resample=Image.Resampling.LANCZOS) scale = max(w / im.width, h / im.height) if scale > 1.0: upscalers = [x for x in shared.sd_upscalers if x.name == upscaler_name] - if len(upscalers) == 0: - upscaler = shared.sd_upscalers[0] - shared.log.warning(f"Could not find upscaler: {upscaler_name or ''} using fallback: {upscaler.name}") - else: + if len(upscalers) > 0: upscaler = upscalers[0] - im = upscaler.scaler.upscale(im, scale, upscaler.data_path) + im = upscaler.scaler.upscale(im, scale, upscaler.data_path) + else: + upscaler = shared.latent_upscale_modes.get(upscaler_name, None) + if upscaler is not None: + im = latent(im, w, h, upscaler) + else: + shared.log.warning(f"Could not find upscaler: {upscaler_name or ''} using fallback: {upscaler.name}") if im.width != w or im.height != h: im = im.resize((w, h), resample=Image.Resampling.LANCZOS) return im diff --git a/modules/shared.py b/modules/shared.py index 7315b84d2..135b39766 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -45,12 +45,15 @@ loaded_hypernetworks = [] settings_components = None latent_upscale_default_mode = "None" latent_upscale_modes = { - "Latent": {"mode": "bilinear", "antialias": False}, - "Latent (antialiased)": {"mode": "bilinear", "antialias": True}, - "Latent (bicubic)": {"mode": "bicubic", "antialias": False}, - "Latent (bicubic antialiased)": {"mode": "bicubic", "antialias": True}, - "Latent (nearest)": {"mode": "nearest", "antialias": False}, - "Latent (nearest-exact)": {"mode": "nearest-exact", "antialias": False}, + "Latent Nearest": {"mode": "nearest", "antialias": False}, + "Latent Nearest-exact": {"mode": "nearest-exact", "antialias": False}, + "Latent Area": {"mode": "area", "antialias": False}, + "Latent Bilinear": {"mode": "bilinear", "antialias": False}, + "Latent Bicubic": {"mode": "bicubic", "antialias": False}, + "Latent Bilinear antialias": {"mode": "bilinear", "antialias": True}, + "Latent Bicubic antialias": {"mode": "bicubic", "antialias": True}, + # "Latent Linear": {"mode": "linear", "antialias": False}, # not supported for latents with channels=4 + # "Latent Trilinear": {"mode": "trilinear", "antialias": False}, # not supported for latents with channels=4 } restricted_opts = { "samples_filename_pattern", diff --git a/modules/ui_control.py b/modules/ui_control.py index 28184c020..eb8a4889c 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -319,7 +319,12 @@ def create_ui(_blocks: gr.Blocks=None): mask_blur = gr.Slider(minimum=0, maximum=100, step=1, label='Blur', value=8, elem_id="control_mask_blur") mask_overlap = gr.Slider(minimum=0, maximum=100, step=1, label='Overlap', value=64, elem_id="control_mask_overlap") - resize_mode, resize_name, width, height, scale_by, selected_scale_tab, resize_time = ui_sections.create_resize_inputs('control', [], time_selector=True, scale_visible=False, mode='Fixed') + with gr.Accordion(open=False, label="Size", elem_id="control_size", elem_classes=["small-accordion"]): + with gr.Tabs(): + with gr.Tab('Before'): + resize_mode_before, resize_name_before, width_before, height_before, scale_by_before, selected_scale_tab_before = ui_sections.create_resize_inputs('control', [], scale_visible=False, mode='Fixed', accordion=False, latent=True) + with gr.Tab('After'): + resize_mode_after, resize_name_after, width_after, height_after, scale_by_after, selected_scale_tab_after = ui_sections.create_resize_inputs('control', [], scale_visible=False, mode='Fixed', accordion=False, latent=False) with gr.Accordion(open=False, label="Sampler", elem_id="control_sampler", elem_classes=["small-accordion"]): sd_samplers.set_samplers() @@ -698,7 +703,8 @@ def create_ui(_blocks: gr.Blocks=None): steps, sampler_index, seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w, cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, sag_scale, full_quality, restore_faces, tiling, hdr_clamp, hdr_boundary, hdr_threshold, hdr_center, hdr_channel_shift, hdr_full_shift, hdr_maximize, hdr_max_center, hdr_max_boundry, - resize_mode, resize_name, width, height, scale_by, selected_scale_tab, resize_time, + resize_mode_before, resize_name_before, width_before, height_before, scale_by_before, selected_scale_tab_before, + resize_mode_after, resize_name_after, width_after, height_after, scale_by_after, selected_scale_tab_after, denoising_strength, batch_count, batch_size, mask_blur, mask_overlap, video_skip_frames, video_type, video_duration, video_loop, video_pad, video_interpolate, ip_adapter, ip_scale, ip_image, ip_type, diff --git a/modules/ui_img2img.py b/modules/ui_img2img.py index 0b073d075..42916abde 100644 --- a/modules/ui_img2img.py +++ b/modules/ui_img2img.py @@ -133,7 +133,7 @@ def create_ui(): with FormGroup(elem_classes="settings-accordion"): steps, sampler_index = ui_sections.create_sampler_inputs('img2img') - resize_mode, resize_name, width, height, scale_by, selected_scale_tab, _resize_time = ui_sections.create_resize_inputs('img2img', [init_img, sketch]) + resize_mode, resize_name, width, height, scale_by, selected_scale_tab = ui_sections.create_resize_inputs('img2img', [init_img, sketch]) batch_count, batch_size = ui_sections.create_batch_inputs('img2img') seed, reuse_seed, subseed, reuse_subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w = ui_sections.create_seed_inputs('img2img') diff --git a/modules/ui_sections.py b/modules/ui_sections.py index 8474459cf..44f0df1ea 100644 --- a/modules/ui_sections.py +++ b/modules/ui_sections.py @@ -108,7 +108,8 @@ def create_advanced_inputs(tab): image_cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='Secondary CFG scale', value=6.0, elem_id=f"{tab}_image_cfg_scale") with FormRow(): diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Guidance rescale', value=0.7, elem_id=f"{tab}_image_cfg_rescale", visible=shared.backend == shared.Backend.DIFFUSERS) - # diffusers_sag_scale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Self-attention guidance', value=0.0, elem_id=f"{tab}_image_sag_scale", visible=shared.backend == shared.Backend.DIFFUSERS) # TODO enable SAG once fixed in diffusers + # TODO enable SAG once fixed in diffusers + # diffusers_sag_scale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Self-attention guidance', value=0.0, elem_id=f"{tab}_image_sag_scale", visible=shared.backend == shared.Backend.DIFFUSERS) diffusers_sag_scale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Self-attention guidance', value=0.0, elem_id=f"{tab}_image_sag_scale", visible=False) with FormRow(): clip_skip = gr.Slider(label='CLIP skip', value=1, minimum=1, maximum=14, step=1, elem_id=f"{tab}_clip_skip", interactive=True) @@ -205,7 +206,7 @@ def create_hires_inputs(tab): return enable_hr, hr_sampler_index, denoising_strength, hr_final_resolution, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative -def create_resize_inputs(tab, images, time_selector=False, scale_visible=True, mode=None): +def create_resize_inputs(tab, images, scale_visible=True, mode=None, accordion=True, latent=False): def resize_from_to_html(width, height, scale_by): target_width = int(width * scale_by) target_height = int(height * scale_by) @@ -214,15 +215,14 @@ def create_resize_inputs(tab, images, time_selector=False, scale_visible=True, m return f"Hires resize: from {width}x{height} to {target_width}x{target_height}" dummy_component = gr.Number(visible=False, value=0) - with gr.Accordion(open=False, label="Resize", elem_classes=["small-accordion"], elem_id=f"{tab}_resize_group"): + with gr.Accordion(open=False, label="Resize", elem_classes=["small-accordion"], elem_id=f"{tab}_resize_group") if accordion else gr.Group(): with gr.Row(): if mode is not None: resize_mode = gr.Radio(label="Resize mode", elem_id=f"{tab}_resize_mode", choices=shared.resize_modes, type="index", value=mode, visible=False) else: resize_mode = gr.Radio(label="Resize mode", elem_id=f"{tab}_resize_mode", choices=shared.resize_modes, type="index", value='None') - resize_time = gr.Radio(label="Resize order", elem_id=f"{tab}_resize_order", choices=['Before', 'After'], value="Before", visible=time_selector) with gr.Row(): - resize_name = gr.Dropdown(label="Resize method", elem_id=f"{tab}_resize_name", choices=[x.name for x in shared.sd_upscalers], value=shared.opts.upscaler_for_img2img) + resize_name = gr.Dropdown(label="Resize method", elem_id=f"{tab}_resize_name", choices=([] if not latent else list(shared.latent_upscale_modes)) + [x.name for x in shared.sd_upscalers], value=shared.latent_upscale_default_mode) ui_common.create_refresh_button(resize_name, modelloader.load_upscalers, lambda: {"choices": modelloader.load_upscalers()}, 'refresh_upscalers') with FormRow(visible=True) as _resize_group: @@ -258,4 +258,4 @@ def create_resize_inputs(tab, images, time_selector=False, scale_visible=True, m tab_scale_to.select(fn=lambda: 0, inputs=[], outputs=[selected_scale_tab]) tab_scale_by.select(fn=lambda: 1, inputs=[], outputs=[selected_scale_tab]) # resize_mode.change(fn=lambda x: gr.update(visible=x != 0), inputs=[resize_mode], outputs=[_resize_group]) - return resize_mode, resize_name, width, height, scale_by, selected_scale_tab, resize_time + return resize_mode, resize_name, width, height, scale_by, selected_scale_tab