From f03994c44fa8e151d7dcc5c765b1b98ee976fa73 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 6 Jan 2024 17:00:21 -0500 Subject: [PATCH] control module hires --- CHANGELOG.md | 6 +++++- extensions-builtin/sd-extension-chainner | 2 +- modules/control/run.py | 24 +++++++++++++++++------- modules/ui_control.py | 8 ++++---- 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b0202491..ad72968e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Update for 2023-01-06 -Following-up on a major release, some more functionality in new Control module +Following-up on a major release, here is a lot more functionality in new Control module and FaceID module And it also includes fixes for all reported issues so far - **Control**: @@ -16,6 +16,10 @@ And it also includes fixes for all reported issues so far this allows for workflows such as: *image -> upscale or downscale -> generate -> upscale or downscale -> output* providing more flexibility and than standard hires workflow *note*: resizing before generate can be done using standard upscalers or latent + - implicit **hires** + since hires is only used for txt2img, control reuses existing resize functionality + any image size is used as txt2img target size + but if resize scale is also set its used to additionally upscale image after initial txt2img and for hires pass - add **marigold** depth map processor this is state-of-the-art depth estimation model, but its quite heavy on resources - add **openpose xl** controlnet diff --git a/extensions-builtin/sd-extension-chainner b/extensions-builtin/sd-extension-chainner index e48020d03..929593daf 160000 --- a/extensions-builtin/sd-extension-chainner +++ b/extensions-builtin/sd-extension-chainner @@ -1 +1 @@ -Subproject commit e48020d035b0d4daffec37a2bbce61c3ea04798c +Subproject commit 929593daff4bb88712bda19e876e8ef69cf2dc6b diff --git a/modules/control/run.py b/modules/control/run.py index 742cde298..9c174bcd0 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -1,5 +1,6 @@ import os import time +import math from typing import List, Union import cv2 import numpy as np @@ -50,6 +51,18 @@ class ControlProcessing(processing.StableDiffusionProcessingImg2Img): def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts): # abstract pass + def init_hr(self): + if self.resize_name == 'None' or self.scale_by == 1.0: + return + self.is_hr_pass = True + self.hr_force = True + self.hr_upscaler = self.resize_name + self.hr_upscale_to_x, self.hr_upscale_to_y = int(self.width * self.scale_by), int(self.height * self.scale_by) + self.hr_upscale_to_x, self.hr_upscale_to_y = 8 * math.ceil(self.hr_upscale_to_x / 8), 8 * math.ceil(self.hr_upscale_to_y / 8) + # hypertile_set(self, hr=True) + shared.state.job_count = 2 * self.n_iter + shared.log.debug(f'Control hires: upscaler="{self.hr_upscaler}" upscale={self.scale_by} size={self.hr_upscale_to_x}x{self.hr_upscale_to_y}') + def restore_pipeline(): global pipe # pylint: disable=global-statement @@ -128,8 +141,7 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ processing.process_init(p) if resize_mode_before != 0 or inputs is None or inputs == [None]: - p.width = width_before # pylint: disable=attribute-defined-outside-init - p.height = height_before # pylint: disable=attribute-defined-outside-init + p.width, p.height = width_before, height_before # pylint: disable=attribute-defined-outside-init else: del p.width del p.height @@ -342,9 +354,8 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ # resize before if resize_mode_before != 0 and resize_name_before != 'None': - if selected_scale_tab_before == 1: - width_before = int(input_image.width * scale_by_before) - height_before = int(input_image.height * scale_by_before) + if selected_scale_tab_before == 1 and input_image is not None: + width_before, height_before = int(input_image.width * scale_by_before), int(input_image.height * scale_by_before) if input_image is not None: p.extra_generation_params["Control resize"] = f'{resize_name_before}' debug(f'Control resize: op=before image={input_image} width={width_before} height={height_before} mode={resize_mode_before} name={resize_name_before}') @@ -444,6 +455,7 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ p.init_images = [processed_image] shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) else: + p.init_hr() shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) elif unit_type == 'reference': p.is_control = True @@ -489,8 +501,6 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ if resize_mode_after != 0 and resize_name_after != 'None': debug(f'Control resize: op=after image={output_image} width={width_after} height={height_after} mode={resize_mode_after} name={resize_name_after}') output_image = images.resize_image(resize_mode_after, output_image, width_after, height_after, resize_name_after) - elif hasattr(p, 'width') and hasattr(p, 'height'): - output_image = output_image.resize((p.width, p.height), Image.Resampling.LANCZOS) output_images.append(output_image) if is_generator: diff --git a/modules/ui_control.py b/modules/ui_control.py index 9642340af..1f4c845e1 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -421,7 +421,7 @@ def create_ui(_blocks: gr.Blocks=None): num_controlnet_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) controlnet_ui_units = [] # list of hidable accordions for i in range(max_units): - with gr.Accordion(f'Control unit {i+1}', visible= i < num_controlnet_units.value, elem_classes='control-unit') as unit_ui: + with gr.Accordion(f'ControlNet unit {i+1}', visible= i < num_controlnet_units.value, elem_classes='control-unit') as unit_ui: with gr.Row(): enabled_cb = gr.Checkbox(value= i==0, label="") process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') @@ -466,7 +466,7 @@ def create_ui(_blocks: gr.Blocks=None): num_adapter_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) adapter_ui_units = [] # list of hidable accordions for i in range(max_units): - with gr.Accordion(f'Adapter unit {i+1}', visible= i < num_adapter_units.value, elem_classes='control-unit') as unit_ui: + with gr.Accordion(f'T2I-Adapter unit {i+1}', visible= i < num_adapter_units.value, elem_classes='control-unit') as unit_ui: with gr.Row(): enabled_cb = gr.Checkbox(value= i == 0, label="Enabled") process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') @@ -507,7 +507,7 @@ def create_ui(_blocks: gr.Blocks=None): num_controlnet_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) controlnetxs_ui_units = [] # list of hidable accordions for i in range(max_units): - with gr.Accordion(f'Control unit {i+1}', visible= i < num_controlnet_units.value, elem_classes='control-unit') as unit_ui: + with gr.Accordion(f'ControlNet-XS unit {i+1}', visible= i < num_controlnet_units.value, elem_classes='control-unit') as unit_ui: with gr.Row(): enabled_cb = gr.Checkbox(value= i==0, label="") process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None') @@ -551,7 +551,7 @@ def create_ui(_blocks: gr.Blocks=None): num_lite_units = gr.Slider(label="Units", minimum=1, maximum=max_units, step=1, value=1, scale=1) lite_ui_units = [] # list of hidable accordions for i in range(max_units): - with gr.Accordion(f'Control unit {i+1}', visible= i < num_lite_units.value, elem_classes='control-unit') as unit_ui: + with gr.Accordion(f'Control-LLLite unit {i+1}', visible= i < num_lite_units.value, elem_classes='control-unit') as unit_ui: with gr.Row(): enabled_cb = gr.Checkbox(value= i == 0, label="Enabled") process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None')