Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-19 09:14:35 +02:00
Code Issues Packages Projects Releases Wiki Activity

control refine/secondpass/hires

Browse Source
This commit is contained in:
Vladimir Mandic
2024-03-08 10:44:21 -05:00
parent 116e6b3f64
commit e46c5d2995
5 changed files with 61 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+1
View File
@@ -83,6 +83,7 @@
- add masking api endpoints
GET:`/sdapi/v1/masking`, POST:`/sdapi/v1/mask`, sample script:`cli/simple-mask.py`
- **Internal**
- **stable-fast** compatibility with torch 2.2.1
- remove obsolete textual inversion training code
- remove obsolete hypernetworks training code
- **Refiner** validated workflows:
modules/control/run.py
+31 -9
View File
@@ -27,9 +27,9 @@ def restore_pipeline():
global pipe, instance # pylint: disable=global-statement
if instance is not None and hasattr(instance, 'restore'):
instance.restore()
if original_pipeline is not None:
if original_pipeline is not None and (original_pipeline.__class__.__name__ != shared.sd_model.__class__.__name__):
shared.log.debug(f'Control restored pipeline: class={shared.sd_model.__class__.__name__} to={original_pipeline.__class__.__name__}')
shared.sd_model = original_pipeline
shared.log.debug(f'Control restored pipeline: class={shared.sd_model.__class__.__name__}')
pipe = None
instance = None
devices.torch_gc()
@@ -44,6 +44,8 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
resize_mode_after, resize_name_after, width_after, height_after, scale_by_after, selected_scale_tab_after,
resize_mode_mask, resize_name_mask, width_mask, height_mask, scale_by_mask, selected_scale_tab_mask,
denoising_strength, batch_count, batch_size,
enable_hr, hr_sampler_index, hr_denoising_strength, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps,
refiner_start, refiner_prompt, refiner_negative,
video_skip_frames, video_type, video_duration, video_loop, video_pad, video_interpolate,
*input_script_args # pylint: disable=unused-argument
):
@@ -66,13 +68,15 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
negative_prompt = negative,
styles = styles,
steps = steps,
n_iter = batch_count,
batch_size = batch_size,
sampler_name = processing.get_sampler_name(sampler_index),
hr_sampler_name = processing.get_sampler_name(sampler_index),
seed = seed,
subseed = subseed,
subseed_strength = subseed_strength,
seed_resize_from_h = seed_resize_from_h,
seed_resize_from_w = seed_resize_from_w,
# advanced
cfg_scale = cfg_scale,
clip_skip = clip_skip,
image_cfg_scale = image_cfg_scale,
@@ -81,29 +85,46 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
full_quality = full_quality,
restore_faces = restore_faces,
tiling = tiling,
# resize
resize_mode = resize_mode_before if resize_name_before != 'None' else 0,
resize_name = resize_name_before,
scale_by = scale_by_before,
selected_scale_tab = selected_scale_tab_before,
denoising_strength = denoising_strength,
n_iter = batch_count,
batch_size = batch_size,
# inpaint
inpaint_full_res = masking.opts.mask_only,
# inpaint_full_res_padding = masking.opts.mask_padding,
inpainting_mask_invert = 1 if masking.opts.invert else 0,
inpainting_fill = 1,
# hdr
hdr_mode=hdr_mode, hdr_brightness=hdr_brightness, hdr_color=hdr_color, hdr_sharpen=hdr_sharpen, hdr_clamp=hdr_clamp,
hdr_boundary=hdr_boundary, hdr_threshold=hdr_threshold, hdr_maximize=hdr_maximize, hdr_max_center=hdr_max_center, hdr_max_boundry=hdr_max_boundry, hdr_color_picker=hdr_color_picker, hdr_tint_ratio=hdr_tint_ratio,
# path
outpath_samples=shared.opts.outdir_samples or shared.opts.outdir_control_samples,
outpath_grids=shared.opts.outdir_grids or shared.opts.outdir_control_grids,
)
processing.process_init(p)
# set initial resolution
if resize_mode_before != 0 or inputs is None or inputs == [None]:
p.width, p.height = width_before, height_before # pylint: disable=attribute-defined-outside-init
else:
del p.width
del p.height
# hires/refine defined outside of main init
p.enable_hr = enable_hr
p.hr_sampler_name = processing.get_sampler_name(hr_sampler_index)
p.hr_denoising_strength = hr_denoising_strength # TODO
p.hr_upscaler = hr_upscaler
p.hr_force = hr_force
p.hr_second_pass_steps = hr_second_pass_steps
p.hr_scale = hr_scale
p.hr_resize_x = hr_resize_x
p.hr_resize_y = hr_resize_y
p.refiner_steps = refiner_steps
p.refiner_start = refiner_start
p.refiner_prompt = refiner_prompt
p.refiner_negative = refiner_negative
if p.enable_hr and (p.hr_resize_x == 0 or p.hr_resize_y == 0):
p.hr_upscale_to_x, p.hr_upscale_to_y = 8 * int(p.width * p.hr_scale / 8), 8 * int(p.height * p.hr_scale / 8)
t0 = time.time()
num_units = 0
@@ -451,7 +472,7 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
p.init_images = [processed_image]
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE)
else:
p.init_hr()
p.init_hr(p.scale_by, p.resize_name)
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
elif has_models: # actual control
p.is_control = True
@@ -547,6 +568,7 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
else:
image_str = [f'{image.width}x{image.height}' for image in output_images]
image_txt = f'| Images {len(output_images)} | Size {" ".join(image_str)}'
p.init_images = output_images # may be used for hires
if video_type != 'None' and isinstance(output_images, list):
p.do_not_save_grid = True # pylint: disable=attribute-defined-outside-init
@@ -554,7 +576,7 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
image_txt = f'| Frames {len(output_images)} | Size {output_images[0].width}x{output_images[0].height}'
image_txt += f' | {util.dict2str(p.extra_generation_params)}'
# restore_pipeline()
restore_pipeline()
debug(f'Control ready: {image_txt}')
if is_generator:
yield (output_images, processed_image, f'Control ready {image_txt}', output_filename)
modules/processing_class.py
+14 -11
View File
@@ -1,5 +1,4 @@
import os
import math
import hashlib
from typing import Any, Dict, List
from dataclasses import dataclass, field
@@ -237,10 +236,12 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
self.width = self.width or 512
self.height = self.height or 512
def init_hr(self):
def init_hr(self, scale = None, upscaler = None):
scale = scale or self.hr_scale
upscaler = upscaler or self.hr_upscaler
if self.hr_resize_x == 0 and self.hr_resize_y == 0:
self.hr_upscale_to_x = int(self.width * self.hr_scale)
self.hr_upscale_to_y = int(self.height * self.hr_scale)
self.hr_upscale_to_x = int(self.width * scale)
self.hr_upscale_to_y = int(self.height * scale)
else:
if self.hr_resize_y == 0:
self.hr_upscale_to_x = self.hr_resize_x
@@ -262,7 +263,7 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
self.truncate_x = (self.hr_upscale_to_x - target_w) // 8
self.truncate_y = (self.hr_upscale_to_y - target_h) // 8
if shared.backend == shared.Backend.ORIGINAL: # diffusers are handled in processing_diffusers
if (self.hr_upscale_to_x == self.width and self.hr_upscale_to_y == self.height) or self.hr_upscaler is None or self.hr_upscaler == 'None': # special case: the user has chosen to do nothing
if (self.hr_upscale_to_x == self.width and self.hr_upscale_to_y == self.height) or upscaler is None or upscaler == 'None': # special case: the user has chosen to do nothing
self.is_hr_pass = False
return
self.is_hr_pass = True
@@ -283,6 +284,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
self.resize_mode: int = resize_mode
self.resize_name: str = resize_name
self.denoising_strength: float = denoising_strength
self.hr_denoising_strength: float = denoising_strength
self.image_cfg_scale: float = image_cfg_scale
self.init_latent = None
self.image_mask = mask
@@ -477,14 +479,15 @@ class StableDiffusionProcessingControl(StableDiffusionProcessingImg2Img):
def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts): # abstract
pass
def init_hr(self):
if self.resize_name == 'None' or self.scale_by == 1.0:
def init_hr(self, scale = None, upscaler = None):
scale = scale or self.scale_by
upscaler = upscaler or self.resize_name
if upscaler == 'None' or scale == 1.0:
return
self.is_hr_pass = True
self.hr_force = True
self.hr_upscaler = self.resize_name
self.hr_upscale_to_x, self.hr_upscale_to_y = int(self.width * self.scale_by), int(self.height * self.scale_by)
self.hr_upscale_to_x, self.hr_upscale_to_y = 8 * math.ceil(self.hr_upscale_to_x / 8), 8 * math.ceil(self.hr_upscale_to_y / 8)
self.hr_upscaler = upscaler
self.hr_upscale_to_x, self.hr_upscale_to_y = 8 * int(self.width * scale / 8), 8 * int(self.height * scale / 8)
# hypertile_set(self, hr=True)
shared.state.job_count = 2 * self.n_iter
shared.log.debug(f'Control hires: upscaler="{self.hr_upscaler}" upscale={self.scale_by} size={self.hr_upscale_to_x}x{self.hr_upscale_to_y}')
shared.log.debug(f'Control hires: upscaler="{self.hr_upscaler}" upscale={scale} size={self.hr_upscale_to_x}x{self.hr_upscale_to_y}')
modules/processing_diffusers.py
+11 -5
View File
@@ -449,12 +449,15 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
return results
# optional second pass
if p.enable_hr and len(getattr(p, 'init_images', [])) == 0:
if p.enable_hr:
p.is_hr_pass = True
if p.is_hr_pass:
p.init_hr()
p.init_hr(p.hr_scale, p.hr_upscaler)
prev_job = shared.state.job
# hires runs on original pipeline
if hasattr(shared.sd_model, 'restore_pipeline') and shared.sd_model.restore_pipeline is not None:
shared.sd_model.restore_pipeline()
# upscale
if hasattr(p, 'height') and hasattr(p, 'width') and p.hr_upscaler is not None and p.hr_upscaler != 'None':
shared.log.info(f'Upscale: upscaler="{p.hr_upscaler}" resize={p.hr_resize_x}x{p.hr_resize_y} upscale={p.hr_upscale_to_x}x{p.hr_upscale_to_y}')
@@ -466,7 +469,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
sd_hijack_hypertile.hypertile_set(p, hr=True)
latent_upscale = shared.latent_upscale_modes.get(p.hr_upscaler, None)
if (latent_upscale is not None or p.hr_force) and p.denoising_strength > 0:
if (latent_upscale is not None or p.hr_force) and getattr(p, 'hr_denoising_strength', p.denoising_strength) > 0:
p.ops.append('hires')
sd_models_compile.openvino_recompile_model(p, hires=True, refiner=False)
if shared.sd_model.__class__.__name__ == "OnnxRawPipeline":
@@ -480,6 +483,8 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
update_sampler(shared.sd_model, second_pass=True)
shared.log.info(f'HiRes: class={shared.sd_model.__class__.__name__} sampler="{p.hr_sampler_name}"')
sd_models.move_model(shared.sd_model, devices.device)
orig_denoise = p.denoising_strength
p.denoising_strength = getattr(p, 'hr_denoising_strength', p.denoising_strength)
hires_args = set_pipeline_args(
model=shared.sd_model,
prompts=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else p.prompts,
@@ -507,7 +512,8 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
sd_models_compile.openvino_post_compile(op="base")
except AssertionError as e:
shared.log.info(e)
p.init_images = []
p.denoising_strength = orig_denoise
# p.init_images = []
shared.state.job = prev_job
shared.state.nextjob()
p.is_hr_pass = False
modules/ui_control.py
+4
View File
@@ -124,6 +124,8 @@ def create_ui(_blocks: gr.Blocks=None):
video_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False)
video_type.change(fn=helpers.video_type_change, inputs=[video_type], outputs=[video_duration, video_loop, video_pad, video_interpolate])
enable_hr, hr_sampler_index, hr_denoising_strength, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('txt2img')
with gr.Row():
override_settings = ui_common.create_override_inputs('control')
@@ -501,6 +503,8 @@ def create_ui(_blocks: gr.Blocks=None):
resize_mode_after, resize_name_after, width_after, height_after, scale_by_after, selected_scale_tab_after,
resize_mode_mask, resize_name_mask, width_mask, height_mask, scale_by_mask, selected_scale_tab_mask,
denoising_strength, batch_count, batch_size,
enable_hr, hr_sampler_index, hr_denoising_strength, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps,
refiner_start, refiner_prompt, refiner_negative,
video_skip_frames, video_type, video_duration, video_loop, video_pad, video_interpolate,
]
output_fields = [