control add latent upscale

CHANGELOG.md

@@ -12,6 +12,10 @@ And it also includes fixes for all reported issues so far
   - add **outpaint** support  
     applies to both *img2img* and *controlnet* workflows  
     *note*: increase denoising strength since outpainted area is blank by default  
+  - allow **resize** both *before* and *after* generate operation  
+    this allows for workflows such as: *image -> upscale or downscale -> generate -> upscale or downscale -> output*  
+    providing more flexibility and than standard hires workflow  
+    *note*: resizing before generate can be done using standard upscalers or latent
   - add **marigold** depth map processor  
     this is state-of-the-art depth estimation model, but its quite heavy on resources  
   - add **openpose xl** controlnet  
@@ -32,6 +36,8 @@ And it also includes fixes for all reported issues so far
     - faster json parsing
   - **offline deployment**: allow deployment without git clone  
     for example, you can now deploy a zip of the sdnext folder  
+  - **latent upscale**: updated latent upscalers (some are new)  
+    *nearest, nearest-exact, area, bilinear, bicubic, bilinear-antialias, bicubic-antialias*
   - **xyz grid**: continue on error  
     now you can use xyz grid with different params and test which ones work and which dont  
   - **hypertile**  

modules/control/run.py

@@ -1,6 +1,5 @@
 import os
 import time
-import math
 from typing import List, Union
 import cv2
 import numpy as np
@@ -66,7 +65,8 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
                 seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w,
                 cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, sag_scale, full_quality, restore_faces, tiling,
                 hdr_clamp, hdr_boundary, hdr_threshold, hdr_center, hdr_channel_shift, hdr_full_shift, hdr_maximize, hdr_max_center, hdr_max_boundry,
-                resize_mode, resize_name, width, height, scale_by, selected_scale_tab, resize_time,
+                resize_mode_before, resize_name_before, width_before, height_before, scale_by_before, selected_scale_tab_before,
+                resize_mode_after, resize_name_after, width_after, height_after, scale_by_after, selected_scale_tab_after,
                 denoising_strength, batch_count, batch_size, mask_blur, mask_overlap,
                 video_skip_frames, video_type, video_duration, video_loop, video_pad, video_interpolate,
                 ip_adapter, ip_scale, ip_image, ip_type,
@@ -82,8 +82,6 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
     active_start: List[float] = [] # start step for all active models
     active_end: List[float] = [] # end step for all active models
     processed_image: Image.Image = None # last processed image
-    width = 8 * math.ceil(width / 8)
-    height = 8 * math.ceil(height / 8)
     if mask is not None and input_type == 0:
         input_type = 1 # inpaint always requires control_image
 
@@ -116,10 +114,10 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
         hdr_maximize = hdr_maximize,
         hdr_max_center = hdr_max_center,
         hdr_max_boundry = hdr_max_boundry,
-        resize_mode = resize_mode if resize_name != 'None' else 0,
-        resize_name = resize_name,
-        scale_by = scale_by,
-        selected_scale_tab = selected_scale_tab,
+        resize_mode = resize_mode_before if resize_name_before != 'None' else 0,
+        resize_name = resize_name_before,
+        scale_by = scale_by_before,
+        selected_scale_tab = selected_scale_tab_before,
         denoising_strength = denoising_strength,
         n_iter = batch_count,
         batch_size = batch_size,
@@ -129,9 +127,9 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
     )
     processing.process_init(p)
 
-    if resize_mode != 0 or inputs is None or inputs == [None]:
-        p.width = width # pylint: disable=attribute-defined-outside-init
-        p.height = height # pylint: disable=attribute-defined-outside-init
+    if resize_mode_before != 0 or inputs is None or inputs == [None]:
+        p.width = width_before # pylint: disable=attribute-defined-outside-init
+        p.height = height_before # pylint: disable=attribute-defined-outside-init
     else:
         del p.width
         del p.height
@@ -342,15 +340,15 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
                     if video is not None and index % (video_skip_frames + 1) != 0:
                         continue
 
-                    # resize
-                    if p.resize_mode != 0 and input_image is not None:
-                        p.extra_generation_params["Control resize"] = f'{resize_time}: {resize_name}'
-                        if selected_scale_tab == 1:
-                            width = int(input_image.width * scale_by)
-                            height = int(input_image.height * scale_by)
-                    if p.resize_mode != 0 and input_image is not None and resize_time == 'Before':
-                        debug(f'Control resize: image={input_image} width={width} height={height} mode={p.resize_mode} name={resize_name} sequence={resize_time}')
-                        input_image = images.resize_image(p.resize_mode, input_image, width, height, resize_name)
+                    # resize before
+                    if resize_mode_before != 0 and resize_name_before != 'None':
+                        if selected_scale_tab_before == 1:
+                            width_before = int(input_image.width * scale_by_before)
+                            height_before = int(input_image.height * scale_by_before)
+                        if input_image is not None:
+                            p.extra_generation_params["Control resize"] = f'{resize_name_before}'
+                            debug(f'Control resize: op=before image={input_image} width={width_before} height={height_before} mode={resize_mode_before} name={resize_name_before}')
+                            input_image = images.resize_image(resize_mode_before, input_image, width_before, height_before, resize_name_before)
                     if input_image is not None:
                         p.width = input_image.width
                         p.height = input_image.height
@@ -485,10 +483,14 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
                     if output is not None and len(output) > 0:
                         output_image = output[0]
                         if output_image is not None:
-                            # resize
-                            if p.resize_mode != 0 and resize_time == 'After':
-                                debug(f'Control resize: image={input_image} width={width} height={height} mode={p.resize_mode} name={resize_name} sequence={resize_time}')
-                                output_image = images.resize_image(p.resize_mode, output_image, width, height, resize_name)
+
+                            # resize after
+                            if selected_scale_tab_after == 1:
+                                width_after = int(output_image.width * scale_by_after)
+                                height_after = int(output_image.height * scale_by_after)
+                            if resize_mode_after != 0 and resize_name_after != 'None':
+                                debug(f'Control resize: op=after image={output_image} width={width_after} height={height_after} mode={resize_mode_after} name={resize_name_after}')
+                                output_image = images.resize_image(resize_mode_after, output_image, width_after, height_after, resize_name_after)
                             elif hasattr(p, 'width') and hasattr(p, 'height'):
                                 output_image = output_image.resize((p.width, p.height), Image.Resampling.LANCZOS)
 

modules/images.py

@@ -226,18 +226,29 @@ def resize_image(resize_mode, im, width, height, upscaler_name=None, output_type
     """
     upscaler_name = upscaler_name or shared.opts.upscaler_for_img2img
 
+    def latent(im, w, h, upscaler):
+        from modules.processing_vae import vae_encode, vae_decode
+        import torch
+        latents = vae_encode(im, shared.sd_model, full_quality=False) # TODO enable full VAE mode
+        latents = torch.nn.functional.interpolate(latents, size=(h // 8, w // 8), mode=upscaler["mode"], antialias=upscaler["antialias"])
+        im = vae_decode(latents, shared.sd_model, output_type='pil', full_quality=False)[0]
+        return im
+
     def resize(im, w, h):
         if upscaler_name is None or upscaler_name == "None" or im.mode == 'L':
             return im.resize((w, h), resample=Image.Resampling.LANCZOS)
         scale = max(w / im.width, h / im.height)
         if scale > 1.0:
             upscalers = [x for x in shared.sd_upscalers if x.name == upscaler_name]
-            if len(upscalers) == 0:
-                upscaler = shared.sd_upscalers[0]
-                shared.log.warning(f"Could not find upscaler: {upscaler_name or '<empty string>'} using fallback: {upscaler.name}")
-            else:
+            if len(upscalers) > 0:
                 upscaler = upscalers[0]
-            im = upscaler.scaler.upscale(im, scale, upscaler.data_path)
+                im = upscaler.scaler.upscale(im, scale, upscaler.data_path)
+            else:
+                upscaler = shared.latent_upscale_modes.get(upscaler_name, None)
+                if upscaler is not None:
+                    im = latent(im, w, h, upscaler)
+                else:
+                    shared.log.warning(f"Could not find upscaler: {upscaler_name or '<empty string>'} using fallback: {upscaler.name}")
         if im.width != w or im.height != h:
             im = im.resize((w, h), resample=Image.Resampling.LANCZOS)
         return im

modules/shared.py

@@ -45,12 +45,15 @@ loaded_hypernetworks = []
 settings_components = None
 latent_upscale_default_mode = "None"
 latent_upscale_modes = {
-    "Latent": {"mode": "bilinear", "antialias": False},
-    "Latent (antialiased)": {"mode": "bilinear", "antialias": True},
-    "Latent (bicubic)": {"mode": "bicubic", "antialias": False},
-    "Latent (bicubic antialiased)": {"mode": "bicubic", "antialias": True},
-    "Latent (nearest)": {"mode": "nearest", "antialias": False},
-    "Latent (nearest-exact)": {"mode": "nearest-exact", "antialias": False},
+    "Latent Nearest": {"mode": "nearest", "antialias": False},
+    "Latent Nearest-exact": {"mode": "nearest-exact", "antialias": False},
+    "Latent Area": {"mode": "area", "antialias": False},
+    "Latent Bilinear": {"mode": "bilinear", "antialias": False},
+    "Latent Bicubic": {"mode": "bicubic", "antialias": False},
+    "Latent Bilinear antialias": {"mode": "bilinear", "antialias": True},
+    "Latent Bicubic antialias": {"mode": "bicubic", "antialias": True},
+    # "Latent Linear": {"mode": "linear", "antialias": False}, # not supported for latents with channels=4
+    # "Latent Trilinear": {"mode": "trilinear", "antialias": False}, # not supported for latents with channels=4
 }
 restricted_opts = {
     "samples_filename_pattern",

modules/ui_control.py

@@ -319,7 +319,12 @@ def create_ui(_blocks: gr.Blocks=None):
                         mask_blur = gr.Slider(minimum=0, maximum=100, step=1, label='Blur', value=8, elem_id="control_mask_blur")
                         mask_overlap = gr.Slider(minimum=0, maximum=100, step=1, label='Overlap', value=64, elem_id="control_mask_overlap")
 
-                resize_mode, resize_name, width, height, scale_by, selected_scale_tab, resize_time = ui_sections.create_resize_inputs('control', [], time_selector=True, scale_visible=False, mode='Fixed')
+                with gr.Accordion(open=False, label="Size", elem_id="control_size", elem_classes=["small-accordion"]):
+                    with gr.Tabs():
+                        with gr.Tab('Before'):
+                            resize_mode_before, resize_name_before, width_before, height_before, scale_by_before, selected_scale_tab_before = ui_sections.create_resize_inputs('control', [], scale_visible=False, mode='Fixed', accordion=False, latent=True)
+                        with gr.Tab('After'):
+                            resize_mode_after, resize_name_after, width_after, height_after, scale_by_after, selected_scale_tab_after = ui_sections.create_resize_inputs('control', [], scale_visible=False, mode='Fixed', accordion=False, latent=False)
 
                 with gr.Accordion(open=False, label="Sampler", elem_id="control_sampler", elem_classes=["small-accordion"]):
                     sd_samplers.set_samplers()
@@ -698,7 +703,8 @@ def create_ui(_blocks: gr.Blocks=None):
                     steps, sampler_index,
                     seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w,
                     cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, sag_scale, full_quality, restore_faces, tiling, hdr_clamp, hdr_boundary, hdr_threshold, hdr_center, hdr_channel_shift, hdr_full_shift, hdr_maximize, hdr_max_center, hdr_max_boundry,
-                    resize_mode, resize_name, width, height, scale_by, selected_scale_tab, resize_time,
+                    resize_mode_before, resize_name_before, width_before, height_before, scale_by_before, selected_scale_tab_before,
+                    resize_mode_after, resize_name_after, width_after, height_after, scale_by_after, selected_scale_tab_after,
                     denoising_strength, batch_count, batch_size, mask_blur, mask_overlap,
                     video_skip_frames, video_type, video_duration, video_loop, video_pad, video_interpolate,
                     ip_adapter, ip_scale, ip_image, ip_type,

modules/ui_img2img.py

@@ -133,7 +133,7 @@ def create_ui():
                 with FormGroup(elem_classes="settings-accordion"):
 
                     steps, sampler_index = ui_sections.create_sampler_inputs('img2img')
-                    resize_mode, resize_name, width, height, scale_by, selected_scale_tab, _resize_time = ui_sections.create_resize_inputs('img2img', [init_img, sketch])
+                    resize_mode, resize_name, width, height, scale_by, selected_scale_tab = ui_sections.create_resize_inputs('img2img', [init_img, sketch])
                     batch_count, batch_size = ui_sections.create_batch_inputs('img2img')
                     seed, reuse_seed, subseed, reuse_subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w = ui_sections.create_seed_inputs('img2img')
 

modules/ui_sections.py

@@ -108,7 +108,8 @@ def create_advanced_inputs(tab):
                 image_cfg_scale = gr.Slider(minimum=0.0, maximum=30.0, step=0.1, label='Secondary CFG scale', value=6.0, elem_id=f"{tab}_image_cfg_scale")
             with FormRow():
                 diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Guidance rescale', value=0.7, elem_id=f"{tab}_image_cfg_rescale", visible=shared.backend == shared.Backend.DIFFUSERS)
-                # diffusers_sag_scale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Self-attention guidance', value=0.0, elem_id=f"{tab}_image_sag_scale", visible=shared.backend == shared.Backend.DIFFUSERS) # TODO enable SAG once fixed in diffusers
+                # TODO enable SAG once fixed in diffusers
+                # diffusers_sag_scale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Self-attention guidance', value=0.0, elem_id=f"{tab}_image_sag_scale", visible=shared.backend == shared.Backend.DIFFUSERS)
                 diffusers_sag_scale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Self-attention guidance', value=0.0, elem_id=f"{tab}_image_sag_scale", visible=False)
             with FormRow():
                 clip_skip = gr.Slider(label='CLIP skip', value=1, minimum=1, maximum=14, step=1, elem_id=f"{tab}_clip_skip", interactive=True)
@@ -205,7 +206,7 @@ def create_hires_inputs(tab):
     return enable_hr, hr_sampler_index, denoising_strength, hr_final_resolution, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative
 
 
-def create_resize_inputs(tab, images, time_selector=False, scale_visible=True, mode=None):
+def create_resize_inputs(tab, images, scale_visible=True, mode=None, accordion=True, latent=False):
     def resize_from_to_html(width, height, scale_by):
         target_width = int(width * scale_by)
         target_height = int(height * scale_by)
@@ -214,15 +215,14 @@ def create_resize_inputs(tab, images, time_selector=False, scale_visible=True, m
         return f"Hires resize: from <span class='resolution'>{width}x{height}</span> to <span class='resolution'>{target_width}x{target_height}</span>"
 
     dummy_component = gr.Number(visible=False, value=0)
-    with gr.Accordion(open=False, label="Resize", elem_classes=["small-accordion"], elem_id=f"{tab}_resize_group"):
+    with gr.Accordion(open=False, label="Resize", elem_classes=["small-accordion"], elem_id=f"{tab}_resize_group") if accordion else gr.Group():
         with gr.Row():
             if mode is not None:
                 resize_mode = gr.Radio(label="Resize mode", elem_id=f"{tab}_resize_mode", choices=shared.resize_modes, type="index", value=mode, visible=False)
             else:
                 resize_mode = gr.Radio(label="Resize mode", elem_id=f"{tab}_resize_mode", choices=shared.resize_modes, type="index", value='None')
-            resize_time = gr.Radio(label="Resize order", elem_id=f"{tab}_resize_order", choices=['Before', 'After'], value="Before", visible=time_selector)
         with gr.Row():
-            resize_name = gr.Dropdown(label="Resize method", elem_id=f"{tab}_resize_name", choices=[x.name for x in shared.sd_upscalers], value=shared.opts.upscaler_for_img2img)
+            resize_name = gr.Dropdown(label="Resize method", elem_id=f"{tab}_resize_name", choices=([] if not latent else list(shared.latent_upscale_modes)) + [x.name for x in shared.sd_upscalers], value=shared.latent_upscale_default_mode)
             ui_common.create_refresh_button(resize_name, modelloader.load_upscalers, lambda: {"choices": modelloader.load_upscalers()}, 'refresh_upscalers')
 
         with FormRow(visible=True) as _resize_group:
@@ -258,4 +258,4 @@ def create_resize_inputs(tab, images, time_selector=False, scale_visible=True, m
             tab_scale_to.select(fn=lambda: 0, inputs=[], outputs=[selected_scale_tab])
             tab_scale_by.select(fn=lambda: 1, inputs=[], outputs=[selected_scale_tab])
             # resize_mode.change(fn=lambda x: gr.update(visible=x != 0), inputs=[resize_mode], outputs=[_resize_group])
-    return resize_mode, resize_name, width, height, scale_by, selected_scale_tab, resize_time
+    return resize_mode, resize_name, width, height, scale_by, selected_scale_tab