diff --git a/CHANGELOG.md b/CHANGELOG.md index b7106236e..6dd331f57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,14 +4,17 @@ Mostly a service release - tons of fixes +- changes to **hires** + - enable non-latent upscale modes (standard upscalers) + - when using latent upscale, hires pass is run automatically + - when using non-latent upscalers, hires pass is skipped by default + enabled using **force hires** option in ui + hires was not designed to work with standard upscalers, but i understand this is a common workflow + - when using refiner, upscale/hires runs before refiner pass - update **ui hints** - updated **models -> civitai** - search and download loras - find previews for already downloaded models or loras -- **hires** enable non-latent upscale modes (standard upscalers) - for both *original* and *diffusers* backend - - when using non-latent upscalers, hires is now skipped - hires is only used for latent upscale - - when using refiner, latent upscale works before refiner pass, but non-latent upscale works after refiner pass - new option **inference mode** - default is standard `torch.no_grad` new option is `torch.inference_only` which is slightly faster and uses less vram, but only works on some gpus diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index b15636ed3..e67e01773 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit b15636ed35eff934af69985bcdfbc407cfedfe7d +Subproject commit e67e017731aad05796b9615dc6eadce911298ea1 diff --git a/extensions-builtin/stable-diffusion-webui-images-browser b/extensions-builtin/stable-diffusion-webui-images-browser index b8f6e05d1..9f95e6d48 160000 --- a/extensions-builtin/stable-diffusion-webui-images-browser +++ b/extensions-builtin/stable-diffusion-webui-images-browser @@ -1 +1 @@ -Subproject commit b8f6e05d1d23b3d0d17b0a9cfbf824bf4a1f98e9 +Subproject commit 9f95e6d4812ca7acb3ec56cbf45fbc85aef0236d diff --git a/html/locale_en.json b/html/locale_en.json index 99fa6b8bc..25a6516b9 100644 --- a/html/locale_en.json +++ b/html/locale_en.json @@ -140,6 +140,7 @@ {"id":"","label":"Hires steps","localized":"","hint":"Number of sampling steps for upscaled picture. If 0, uses same as for original"}, {"id":"","label":"Upscaler","localized":"","hint":"Which pre-trained model to use for the upscaling process."}, {"id":"","label":"Upscale by","localized":"","hint":"Adjusts the size of the image by multiplying the original width and height by the selected value. Ignored if either Resize width to or Resize height to are non-zero"}, + {"id":"","label":"Force Hires","localized":"","hint":"Hires runs automatically when Latent upscale is selected, but its skipped when using non-latent upscalers. Enable force hires to run hires with non-latent upscalers"}, {"id":"","label":"Resize width to","localized":"","hint":"Resizes image to this width. If 0, width is inferred from either of two nearby sliders"}, {"id":"","label":"Resize height to","localized":"","hint":"Resizes image to this height. If 0, height is inferred from either of two nearby sliders"}, {"id":"","label":"Secondary sampler","localized":"","hint":"Use specific sampler as fallback sampler if primary is not supported for specific operation"}, diff --git a/javascript/setHints.js b/javascript/setHints.js index f755e6dee..7398c7a5e 100644 --- a/javascript/setHints.js +++ b/javascript/setHints.js @@ -52,6 +52,7 @@ async function setHints() { const res = await fetch('/file=html/locale_en.json'); const json = await res.json(); localeData.data = Object.values(json).flat(); + for (const e of localeData.data) e.label = e.label.toLowerCase().trim(); } const elements = [ ...Array.from(gradioApp().querySelectorAll('button')), @@ -65,7 +66,7 @@ async function setHints() { localeData.finished = true; const t0 = performance.now(); for (const el of elements) { - const found = localeData.data.find((l) => l.label === el.textContent.trim()); + const found = localeData.data.find((l) => l.label === el.textContent.toLowerCase().trim()); if (found?.localized?.length > 0) { localized++; el.textContent = found.localized; diff --git a/javascript/style.css b/javascript/style.css index 5b7ed81a6..b8b4edbb8 100644 --- a/javascript/style.css +++ b/javascript/style.css @@ -6,7 +6,7 @@ div.tabitem { padding: 0 !important; } div.form { border-width: 0; box-shadow: none; background: transparent; overflow: visible; gap: 0.5em 1em; flex-grow: 1 !important; } div.compact{ gap: 1em; } div.gradio-html.min{ min-height: 0; } -.block.gradio-checkbox { margin: 0.75em 1.5em 0 0; } +.block.gradio-checkbox { margin: 0.75em 1.5em 0 0; align-self: center; } .block.gradio-dropdown, .block.gradio-slider, .block.gradio-checkbox, .block.gradio-textbox, .block.gradio-radio, .block.gradio-checkboxgroup, .block.gradio-number, .block.gradio-colorpicker { border-width: 0 !important; box-shadow: none !important;} .block.padded:not(.gradio-accordion) { padding: 0 !important; margin-right: 0; min-width: 100px !important; } .compact{ background: transparent !important; padding: 0 !important; } diff --git a/modules/processing.py b/modules/processing.py index 09cfee1ee..a07434ed7 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -158,6 +158,7 @@ class StableDiffusionProcessing: self.clip_skip = clip_skip self.iteration = 0 self.is_hr_pass = False + self.hr_force = False self.enable_hr = None self.refiner_steps = 5 self.refiner_start = 0 @@ -895,13 +896,14 @@ def old_hires_fix_first_pass_dimensions(width, height): class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): - def __init__(self, enable_hr: bool = False, denoising_strength: float = 0.75, firstphase_width: int = 0, firstphase_height: int = 0, hr_scale: float = 2.0, hr_upscaler: str = None, hr_second_pass_steps: int = 0, hr_resize_x: int = 0, hr_resize_y: int = 0, refiner_steps: int = 5, refiner_start: float = 0, refiner_prompt: str = '', refiner_negative: str = '', **kwargs): + def __init__(self, enable_hr: bool = False, denoising_strength: float = 0.75, firstphase_width: int = 0, firstphase_height: int = 0, hr_scale: float = 2.0, hr_force: bool = False, hr_upscaler: str = None, hr_second_pass_steps: int = 0, hr_resize_x: int = 0, hr_resize_y: int = 0, refiner_steps: int = 5, refiner_start: float = 0, refiner_prompt: str = '', refiner_negative: str = '', **kwargs): super().__init__(**kwargs) self.enable_hr = enable_hr self.denoising_strength = denoising_strength self.hr_scale = hr_scale self.hr_upscaler = hr_upscaler + self.hr_force = hr_force self.hr_second_pass_steps = hr_second_pass_steps self.hr_resize_x = hr_resize_x self.hr_resize_y = hr_resize_y @@ -983,13 +985,14 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): if shared.backend == shared.Backend.DIFFUSERS: modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.TEXT_2_IMAGE) + latent_scale_mode = shared.latent_upscale_modes.get(self.hr_upscaler, None) if self.hr_upscaler is not None else shared.latent_upscale_modes.get(shared.latent_upscale_default_mode, "None") + if self.enable_hr and (latent_scale_mode is None or self.hr_force): + if len([x for x in shared.sd_upscalers if x.name == self.hr_upscaler]) == 0: + shared.log.warning(f"Cannot find upscaler for hires: {self.hr_upscaler}") + self.enable_hr = False + self.ops.append('txt2img') self.sampler = modules.sd_samplers.create_sampler(self.sampler_name, self.sd_model) - latent_scale_mode = shared.latent_upscale_modes.get(self.hr_upscaler, None) if self.hr_upscaler is not None else shared.latent_upscale_modes.get(shared.latent_upscale_default_mode, "None") - if self.enable_hr and latent_scale_mode is None: - if len([x for x in shared.sd_upscalers if x.name == self.hr_upscaler]) == 0: - shared.log.warning("Could not find upscaler to use with hrfix") - self.enable_hr = False x = create_random_tensors([4, self.height // 8, self.width // 8], seeds=seeds, subseeds=subseeds, subseed_strength=self.subseed_strength, seed_resize_from_h=self.seed_resize_from_h, seed_resize_from_w=self.seed_resize_from_w, p=self) samples = self.sampler.sample(self, x, conditioning, unconditional_conditioning, image_conditioning=self.txt2img_image_conditioning(x)) if not self.enable_hr or shared.state.interrupted or shared.state.skipped: @@ -1000,25 +1003,9 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): self.ops.append('hires') target_width = self.hr_upscale_to_x target_height = self.hr_upscale_to_y - if latent_scale_mode is not None: - for i in range(samples.shape[0]): - save_intermediate(samples, i) - samples = torch.nn.functional.interpolate(samples, size=(target_height // 8, target_width // 8), mode=latent_scale_mode["mode"], antialias=latent_scale_mode["antialias"]) - if getattr(self, "inpainting_mask_weight", shared.opts.inpainting_mask_weight) < 1.0: - image_conditioning = self.img2img_image_conditioning(decode_first_stage(self.sd_model, samples.to(dtype=devices.dtype_vae)), samples) - else: - image_conditioning = self.txt2img_image_conditioning(samples.to(dtype=devices.dtype_vae)) - if self.latent_sampler == "PLMS": - self.latent_sampler = 'UniPC' - self.sampler = modules.sd_samplers.create_sampler(self.latent_sampler or self.sampler_name, self.sd_model) - samples = samples[:, :, self.truncate_y//2:samples.shape[2]-(self.truncate_y+1)//2, self.truncate_x//2:samples.shape[3]-(self.truncate_x+1)//2] - noise = create_random_tensors(samples.shape[1:], seeds=seeds, subseeds=subseeds, subseed_strength=subseed_strength, p=self) - x = None - devices.torch_gc() # GC now before running the next img2img to prevent running out of memory - modules.sd_models.apply_token_merging(self.sd_model, self.get_token_merging_ratio(for_hr=True)) - samples = self.sampler.sample_img2img(self, samples, noise, conditioning, unconditional_conditioning, steps=self.hr_second_pass_steps or self.steps, image_conditioning=image_conditioning) - modules.sd_models.apply_token_merging(self.sd_model, self.get_token_merging_ratio()) - else: + for i in range(samples.shape[0]): + save_intermediate(samples, i) + if latent_scale_mode is None or self.hr_force: # non-latent upscaling decoded_samples = decode_first_stage(self.sd_model, samples.to(dtype=devices.dtype_vae)) lowres_samples = torch.clamp((decoded_samples + 1.0) / 2.0, min=0.0, max=1.0) batch_images = [] @@ -1043,6 +1030,23 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): else: samples = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(decoded_samples)) image_conditioning = self.img2img_image_conditioning(decoded_samples, samples) + else: + samples = torch.nn.functional.interpolate(samples, size=(target_height // 8, target_width // 8), mode=latent_scale_mode["mode"], antialias=latent_scale_mode["antialias"]) + if getattr(self, "inpainting_mask_weight", shared.opts.inpainting_mask_weight) < 1.0: + image_conditioning = self.img2img_image_conditioning(decode_first_stage(self.sd_model, samples.to(dtype=devices.dtype_vae)), samples) + else: + image_conditioning = self.txt2img_image_conditioning(samples.to(dtype=devices.dtype_vae)) + if self.latent_sampler == "PLMS": + self.latent_sampler = 'UniPC' + if self.hr_force or latent_scale_mode is not None: + devices.torch_gc() # GC now before running the next img2img to prevent running out of memory + self.sampler = modules.sd_samplers.create_sampler(self.latent_sampler or self.sampler_name, self.sd_model) + samples = samples[:, :, self.truncate_y//2:samples.shape[2]-(self.truncate_y+1)//2, self.truncate_x//2:samples.shape[3]-(self.truncate_x+1)//2] + noise = create_random_tensors(samples.shape[1:], seeds=seeds, subseeds=subseeds, subseed_strength=subseed_strength, p=self) + modules.sd_models.apply_token_merging(self.sd_model, self.get_token_merging_ratio(for_hr=True)) + samples = self.sampler.sample_img2img(self, samples, noise, conditioning, unconditional_conditioning, steps=self.hr_second_pass_steps or self.steps, image_conditioning=image_conditioning) + modules.sd_models.apply_token_merging(self.sd_model, self.get_token_merging_ratio()) + x = None shared.state.nextjob() self.is_hr_pass = False diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 13be1752e..915acd316 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -2,8 +2,6 @@ import time import inspect import typing import torch -import numpy as np -from PIL import Image import modules.devices as devices import modules.shared as shared import modules.sd_samplers as sd_samplers @@ -26,7 +24,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro results = [] if p.enable_hr and p.hr_upscaler != 'None' and p.denoising_strength > 0 and len(getattr(p, 'init_images', [])) == 0: p.is_hr_pass = True - is_refiner_enabled = p.enable_hr and p.refiner_steps > 0 and shared.sd_refiner is not None + is_refiner_enabled = p.enable_hr and p.refiner_steps > 0 and p.refiner_start > 0 and p.refiner_start < 1 and shared.sd_refiner is not None def hires_resize(latents): # input=latents output=pil latent_upscaler = shared.latent_upscale_modes.get(p.hr_upscaler, None) @@ -36,10 +34,11 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro first_pass_images = vae_decode(latents=latents, model=shared.sd_model, full_quality=True, output_type='pil') p.init_images = [] for first_pass_image in first_pass_images: - init_image = images.resize_image(1, first_pass_image, p.hr_upscale_to_x, p.hr_upscale_to_y, upscaler_name=p.hr_upscaler) if latent_upscaler is None else first_pass_image + if latent_upscaler is None: + init_image = images.resize_image(1, first_pass_image, p.hr_upscale_to_x, p.hr_upscale_to_y, upscaler_name=p.hr_upscaler) + else: + init_image = first_pass_image p.init_images.append(init_image) - p.width = p.hr_upscale_to_x - p.height = p.hr_upscale_to_y def save_intermediate(latents, suffix): for i in range(len(latents)): @@ -321,12 +320,12 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro if p.is_hr_pass: p.init_hr() if p.width != p.hr_upscale_to_x or p.height != p.hr_upscale_to_y: + p.ops.append('upscale') if shared.opts.save and not p.do_not_save_samples and shared.opts.save_images_before_highres_fix and hasattr(shared.sd_model, 'vae'): save_intermediate(latents=output.images, suffix="-before-hires") - if latent_scale_mode is not None: - p.ops.append('hires') + hires_resize(latents=output.images) + if latent_scale_mode is not None or p.hr_force: recompile_model(hires=True) - hires_resize(latents=output.images) sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) hires_args = set_pipeline_args( model=shared.sd_model, @@ -372,6 +371,11 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro refiner_is_sdxl = bool("StableDiffusionXL" in shared.sd_refiner.__class__.__name__) p.ops.append('refine') for i in range(len(output.images)): + image = output.images[i] + if (image.shape[2] == 3) and (image.shape[0] % 8 != 0 or image.shape[1] % 8 != 0): + shared.log.warning(f'Refiner requires image size to be divisible by 8: {image.shape}') + results.append(image) + return results refiner_args = set_pipeline_args( model=shared.sd_refiner, prompts=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts[i], @@ -383,7 +387,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro guidance_rescale=p.diffusers_guidance_rescale, denoising_start=p.refiner_start if p.refiner_start > 0 and p.refiner_start < 1 else None, denoising_end=1 if p.refiner_start > 0 and p.refiner_start < 1 else None, - image=output.images[i], + image=image, output_type='latent' if hasattr(shared.sd_refiner, 'vae') else 'np', clip_skip=p.clip_skip, desc='Refiner', @@ -403,20 +407,6 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro shared.sd_refiner.to(devices.cpu) devices.torch_gc() - if p.is_hr_pass and latent_scale_mode is None: - if p.width != p.hr_upscale_to_x or p.height != p.hr_upscale_to_y: - p.ops.append('upscale') - if not is_refiner_enabled: - results = vae_decode(latents=output.images, model=shared.sd_model, full_quality=p.full_quality) - upscaled = [] - for image in results: - image = (image * 255.0).astype(np.uint8) - image = Image.fromarray(image) - image = images.resize_image(1, image, p.hr_upscale_to_x, p.hr_upscale_to_y, upscaler_name=p.hr_upscaler) - image = np.array(image).astype(np.float32) / 255.0 - upscaled.append(image) - return upscaled - # final decode since there is no refiner if not is_refiner_enabled: results = vae_decode(latents=output.images, model=shared.sd_model, full_quality=p.full_quality) diff --git a/modules/txt2img.py b/modules/txt2img.py index 51b09192e..899af07db 100644 --- a/modules/txt2img.py +++ b/modules/txt2img.py @@ -4,9 +4,9 @@ from modules.generation_parameters_copypaste import create_override_settings_dic from modules.ui import plaintext_to_html -def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, steps: int, sampler_index: int, latent_index: int, full_quality: bool, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, image_cfg_scale: float, diffusers_guidance_rescale: float, clip_skip: int, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, height: int, width: int, enable_hr: bool, denoising_strength: float, hr_scale: float, hr_upscaler: str, hr_second_pass_steps: int, hr_resize_x: int, hr_resize_y: int, refiner_steps: int, refiner_start: int, refiner_prompt: str, refiner_negative: str, override_settings_texts, *args): # pylint: disable=unused-argument +def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, steps: int, sampler_index: int, latent_index: int, full_quality: bool, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, image_cfg_scale: float, diffusers_guidance_rescale: float, clip_skip: int, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, height: int, width: int, enable_hr: bool, denoising_strength: float, hr_scale: float, hr_upscaler: str, hr_force: bool, hr_second_pass_steps: int, hr_resize_x: int, hr_resize_y: int, refiner_steps: int, refiner_start: int, refiner_prompt: str, refiner_negative: str, override_settings_texts, *args): # pylint: disable=unused-argument - shared.log.debug(f'txt2img: id_task={id_task}|prompt={prompt}|negative_prompt={negative_prompt}|prompt_styles={prompt_styles}|steps={steps}|sampler_index={sampler_index}|latent_index={latent_index}|full_quality={full_quality}|restore_faces={restore_faces}|tiling={tiling}|n_iter={n_iter}|batch_size={batch_size}|cfg_scale={cfg_scale}|clip_skip={clip_skip}|seed={seed}|subseed={subseed}|subseed_strength={subseed_strength}|seed_resize_from_h={seed_resize_from_h}|seed_resize_from_w={seed_resize_from_w}||height={height}|width={width}|enable_hr={enable_hr}|denoising_strength={denoising_strength}|hr_scale={hr_scale}|hr_upscaler={hr_upscaler}|hr_second_pass_steps={hr_second_pass_steps}|hr_resize_x={hr_resize_x}|hr_resize_y={hr_resize_y}|image_cfg_scale={image_cfg_scale}|diffusers_guidance_rescale={diffusers_guidance_rescale}|refiner_steps={refiner_steps}|refiner_start={refiner_start}||refiner_prompt={refiner_prompt}|refiner_negative={refiner_negative}|override_settings_texts={override_settings_texts}') + shared.log.debug(f'txt2img: id_task={id_task}|prompt={prompt}|negative_prompt={negative_prompt}|prompt_styles={prompt_styles}|steps={steps}|sampler_index={sampler_index}|latent_index={latent_index}|full_quality={full_quality}|restore_faces={restore_faces}|tiling={tiling}|n_iter={n_iter}|batch_size={batch_size}|cfg_scale={cfg_scale}|clip_skip={clip_skip}|seed={seed}|subseed={subseed}|subseed_strength={subseed_strength}|seed_resize_from_h={seed_resize_from_h}|seed_resize_from_w={seed_resize_from_w}||height={height}|width={width}|enable_hr={enable_hr}|denoising_strength={denoising_strength}|hr_scale={hr_scale}|hr_upscaler={hr_upscaler}|hr_force={hr_force}|hr_second_pass_steps={hr_second_pass_steps}|hr_resize_x={hr_resize_x}|hr_resize_y={hr_resize_y}|image_cfg_scale={image_cfg_scale}|diffusers_guidance_rescale={diffusers_guidance_rescale}|refiner_steps={refiner_steps}|refiner_start={refiner_start}|refiner_prompt={refiner_prompt}|refiner_negative={refiner_negative}|override_settings_texts={override_settings_texts}') if shared.sd_model is None: shared.log.warning('Model not loaded') @@ -49,6 +49,7 @@ def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, step denoising_strength=denoising_strength, hr_scale=hr_scale, hr_upscaler=hr_upscaler, + hr_force=hr_force, hr_second_pass_steps=hr_second_pass_steps, hr_resize_x=hr_resize_x, hr_resize_y=hr_resize_y, diff --git a/modules/ui.py b/modules/ui.py index d1b7a6b2a..1b12839a3 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -396,8 +396,9 @@ def create_ui(startup_timer = None): hr_final_resolution = FormHTML(value="", elem_id="txtimg_hr_finalres", label="Upscaled resolution", interactive=False) with FormRow(elem_id="txt2img_hires_fix_row1", variant="compact"): hr_upscaler = gr.Dropdown(label="Upscaler", elem_id="txt2img_hr_upscaler", choices=[*modules.shared.latent_upscale_modes, *[x.name for x in modules.shared.sd_upscalers]], value=modules.shared.latent_upscale_default_mode) - hr_second_pass_steps = gr.Slider(minimum=0, maximum=99, step=1, label='Hires steps', elem_id="txt2img_steps_alt", value=20) + hr_force = gr.Checkbox(label='Force Hires', value=False, elem_id="txt2img_hr_force") with FormRow(elem_id="txt2img_hires_fix_row2", variant="compact"): + hr_second_pass_steps = gr.Slider(minimum=0, maximum=99, step=1, label='Hires steps', elem_id="txt2img_steps_alt", value=20) hr_scale = gr.Slider(minimum=1.0, maximum=4.0, step=0.05, label="Upscale by", value=2.0, elem_id="txt2img_hr_scale") with FormRow(elem_id="txt2img_hires_fix_row3", variant="compact"): hr_resize_x = gr.Slider(minimum=0, maximum=4096, step=8, label="Resize width to", value=0, elem_id="txt2img_hr_resize_x") @@ -450,7 +451,7 @@ def create_ui(startup_timer = None): seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w, height, width, show_second_pass, denoising_strength, - hr_scale, hr_upscaler, hr_second_pass_steps, hr_resize_x, hr_resize_y, + hr_scale, hr_upscaler, hr_force, hr_second_pass_steps, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative, override_settings, ] + custom_inputs,