update hires logic

CHANGELOG.md

@@ -4,14 +4,17 @@
 
 Mostly a service release  
 - tons of fixes  
+- changes to **hires**
+  - enable non-latent upscale modes (standard upscalers)  
+  - when using latent upscale, hires pass is run automatically  
+  - when using non-latent upscalers, hires pass is skipped by default  
+    enabled using **force hires** option in ui  
+    hires was not designed to work with standard upscalers, but i understand this is a common workflow  
+  - when using refiner, upscale/hires runs before refiner pass  
 - update **ui hints**
 - updated **models -> civitai**
   - search and download loras  
   - find previews for already downloaded models or loras  
-- **hires** enable non-latent upscale modes (standard upscalers)  
-  for both *original* and *diffusers* backend  
-  - when using non-latent upscalers, hires is now skipped - hires is only used for latent upscale  
-  - when using refiner, latent upscale works before refiner pass, but non-latent upscale works after refiner pass  
 - new option **inference mode**  
   - default is standard `torch.no_grad`  
     new option is `torch.inference_only` which is slightly faster and uses less vram, but only works on some gpus  

html/locale_en.json

@@ -140,6 +140,7 @@
   {"id":"","label":"Hires steps","localized":"","hint":"Number of sampling steps for upscaled picture. If 0, uses same as for original"},
   {"id":"","label":"Upscaler","localized":"","hint":"Which pre-trained model to use for the upscaling process."},
   {"id":"","label":"Upscale by","localized":"","hint":"Adjusts the size of the image by multiplying the original width and height by the selected value. Ignored if either Resize width to or Resize height to are non-zero"},
+  {"id":"","label":"Force Hires","localized":"","hint":"Hires runs automatically when Latent upscale is selected, but its skipped when using non-latent upscalers. Enable force hires to run hires with non-latent upscalers"},
   {"id":"","label":"Resize width to","localized":"","hint":"Resizes image to this width. If 0, width is inferred from either of two nearby sliders"},
   {"id":"","label":"Resize height to","localized":"","hint":"Resizes image to this height. If 0, height is inferred from either of two nearby sliders"},
   {"id":"","label":"Secondary sampler","localized":"","hint":"Use specific sampler as fallback sampler if primary is not supported for specific operation"},

javascript/setHints.js

@@ -52,6 +52,7 @@ async function setHints() {
     const res = await fetch('/file=html/locale_en.json');
     const json = await res.json();
     localeData.data = Object.values(json).flat();
+    for (const e of localeData.data) e.label = e.label.toLowerCase().trim();
   }
   const elements = [
     ...Array.from(gradioApp().querySelectorAll('button')),
@@ -65,7 +66,7 @@ async function setHints() {
   localeData.finished = true;
   const t0 = performance.now();
   for (const el of elements) {
-    const found = localeData.data.find((l) => l.label === el.textContent.trim());
+    const found = localeData.data.find((l) => l.label === el.textContent.toLowerCase().trim());
     if (found?.localized?.length > 0) {
       localized++;
       el.textContent = found.localized;

javascript/style.css

@@ -6,7 +6,7 @@ div.tabitem { padding: 0 !important; }
 div.form { border-width: 0; box-shadow: none; background: transparent; overflow: visible; gap: 0.5em 1em; flex-grow: 1 !important; }
 div.compact{ gap: 1em; }
 div.gradio-html.min{ min-height: 0; }
-.block.gradio-checkbox { margin: 0.75em 1.5em 0 0; }
+.block.gradio-checkbox { margin: 0.75em 1.5em 0 0; align-self: center; }
 .block.gradio-dropdown, .block.gradio-slider, .block.gradio-checkbox, .block.gradio-textbox, .block.gradio-radio, .block.gradio-checkboxgroup, .block.gradio-number, .block.gradio-colorpicker { border-width: 0 !important; box-shadow: none !important;}
 .block.padded:not(.gradio-accordion) { padding: 0 !important; margin-right: 0; min-width: 100px !important; }
 .compact{ background: transparent !important; padding: 0 !important; }

modules/processing.py

@@ -158,6 +158,7 @@ class StableDiffusionProcessing:
         self.clip_skip = clip_skip
         self.iteration = 0
         self.is_hr_pass = False
+        self.hr_force = False
         self.enable_hr = None
         self.refiner_steps = 5
         self.refiner_start = 0
@@ -895,13 +896,14 @@ def old_hires_fix_first_pass_dimensions(width, height):
 
 class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
 
-    def __init__(self, enable_hr: bool = False, denoising_strength: float = 0.75, firstphase_width: int = 0, firstphase_height: int = 0, hr_scale: float = 2.0, hr_upscaler: str = None, hr_second_pass_steps: int = 0, hr_resize_x: int = 0, hr_resize_y: int = 0, refiner_steps: int = 5, refiner_start: float = 0, refiner_prompt: str = '', refiner_negative: str = '', **kwargs):
+    def __init__(self, enable_hr: bool = False, denoising_strength: float = 0.75, firstphase_width: int = 0, firstphase_height: int = 0, hr_scale: float = 2.0, hr_force: bool = False, hr_upscaler: str = None, hr_second_pass_steps: int = 0, hr_resize_x: int = 0, hr_resize_y: int = 0, refiner_steps: int = 5, refiner_start: float = 0, refiner_prompt: str = '', refiner_negative: str = '', **kwargs):
 
         super().__init__(**kwargs)
         self.enable_hr = enable_hr
         self.denoising_strength = denoising_strength
         self.hr_scale = hr_scale
         self.hr_upscaler = hr_upscaler
+        self.hr_force = hr_force
         self.hr_second_pass_steps = hr_second_pass_steps
         self.hr_resize_x = hr_resize_x
         self.hr_resize_y = hr_resize_y
@@ -983,13 +985,14 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
         if shared.backend == shared.Backend.DIFFUSERS:
             modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.TEXT_2_IMAGE)
 
+        latent_scale_mode = shared.latent_upscale_modes.get(self.hr_upscaler, None) if self.hr_upscaler is not None else shared.latent_upscale_modes.get(shared.latent_upscale_default_mode, "None")
+        if self.enable_hr and (latent_scale_mode is None or self.hr_force):
+            if len([x for x in shared.sd_upscalers if x.name == self.hr_upscaler]) == 0:
+                shared.log.warning(f"Cannot find upscaler for hires: {self.hr_upscaler}")
+                self.enable_hr = False
+
         self.ops.append('txt2img')
         self.sampler = modules.sd_samplers.create_sampler(self.sampler_name, self.sd_model)
-        latent_scale_mode = shared.latent_upscale_modes.get(self.hr_upscaler, None) if self.hr_upscaler is not None else shared.latent_upscale_modes.get(shared.latent_upscale_default_mode, "None")
-        if self.enable_hr and latent_scale_mode is None:
-            if len([x for x in shared.sd_upscalers if x.name == self.hr_upscaler]) == 0:
-                shared.log.warning("Could not find upscaler to use with hrfix")
-                self.enable_hr = False
         x = create_random_tensors([4, self.height // 8, self.width // 8], seeds=seeds, subseeds=subseeds, subseed_strength=self.subseed_strength, seed_resize_from_h=self.seed_resize_from_h, seed_resize_from_w=self.seed_resize_from_w, p=self)
         samples = self.sampler.sample(self, x, conditioning, unconditional_conditioning, image_conditioning=self.txt2img_image_conditioning(x))
         if not self.enable_hr or shared.state.interrupted or shared.state.skipped:
@@ -1000,25 +1003,9 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
             self.ops.append('hires')
             target_width = self.hr_upscale_to_x
             target_height = self.hr_upscale_to_y
-            if latent_scale_mode is not None:
-                for i in range(samples.shape[0]):
-                    save_intermediate(samples, i)
-                samples = torch.nn.functional.interpolate(samples, size=(target_height // 8, target_width // 8), mode=latent_scale_mode["mode"], antialias=latent_scale_mode["antialias"])
-                if getattr(self, "inpainting_mask_weight", shared.opts.inpainting_mask_weight) < 1.0:
-                    image_conditioning = self.img2img_image_conditioning(decode_first_stage(self.sd_model, samples.to(dtype=devices.dtype_vae)), samples)
-                else:
-                    image_conditioning = self.txt2img_image_conditioning(samples.to(dtype=devices.dtype_vae))
-                if self.latent_sampler == "PLMS":
-                    self.latent_sampler = 'UniPC'
-                self.sampler = modules.sd_samplers.create_sampler(self.latent_sampler or self.sampler_name, self.sd_model)
-                samples = samples[:, :, self.truncate_y//2:samples.shape[2]-(self.truncate_y+1)//2, self.truncate_x//2:samples.shape[3]-(self.truncate_x+1)//2]
-                noise = create_random_tensors(samples.shape[1:], seeds=seeds, subseeds=subseeds, subseed_strength=subseed_strength, p=self)
-                x = None
-                devices.torch_gc() # GC now before running the next img2img to prevent running out of memory
-                modules.sd_models.apply_token_merging(self.sd_model, self.get_token_merging_ratio(for_hr=True))
-                samples = self.sampler.sample_img2img(self, samples, noise, conditioning, unconditional_conditioning, steps=self.hr_second_pass_steps or self.steps, image_conditioning=image_conditioning)
-                modules.sd_models.apply_token_merging(self.sd_model, self.get_token_merging_ratio())
-            else:
+            for i in range(samples.shape[0]):
+                save_intermediate(samples, i)
+            if latent_scale_mode is None or self.hr_force: # non-latent upscaling
                 decoded_samples = decode_first_stage(self.sd_model, samples.to(dtype=devices.dtype_vae))
                 lowres_samples = torch.clamp((decoded_samples + 1.0) / 2.0, min=0.0, max=1.0)
                 batch_images = []
@@ -1043,6 +1030,23 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
                 else:
                     samples = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(decoded_samples))
                 image_conditioning = self.img2img_image_conditioning(decoded_samples, samples)
+            else:
+                samples = torch.nn.functional.interpolate(samples, size=(target_height // 8, target_width // 8), mode=latent_scale_mode["mode"], antialias=latent_scale_mode["antialias"])
+                if getattr(self, "inpainting_mask_weight", shared.opts.inpainting_mask_weight) < 1.0:
+                    image_conditioning = self.img2img_image_conditioning(decode_first_stage(self.sd_model, samples.to(dtype=devices.dtype_vae)), samples)
+                else:
+                    image_conditioning = self.txt2img_image_conditioning(samples.to(dtype=devices.dtype_vae))
+                if self.latent_sampler == "PLMS":
+                    self.latent_sampler = 'UniPC'
+            if self.hr_force or latent_scale_mode is not None:
+                devices.torch_gc() # GC now before running the next img2img to prevent running out of memory
+                self.sampler = modules.sd_samplers.create_sampler(self.latent_sampler or self.sampler_name, self.sd_model)
+                samples = samples[:, :, self.truncate_y//2:samples.shape[2]-(self.truncate_y+1)//2, self.truncate_x//2:samples.shape[3]-(self.truncate_x+1)//2]
+                noise = create_random_tensors(samples.shape[1:], seeds=seeds, subseeds=subseeds, subseed_strength=subseed_strength, p=self)
+                modules.sd_models.apply_token_merging(self.sd_model, self.get_token_merging_ratio(for_hr=True))
+                samples = self.sampler.sample_img2img(self, samples, noise, conditioning, unconditional_conditioning, steps=self.hr_second_pass_steps or self.steps, image_conditioning=image_conditioning)
+                modules.sd_models.apply_token_merging(self.sd_model, self.get_token_merging_ratio())
+            x = None
             shared.state.nextjob()
             self.is_hr_pass = False
 

modules/processing_diffusers.py

@@ -2,8 +2,6 @@ import time
 import inspect
 import typing
 import torch
-import numpy as np
-from PIL import Image
 import modules.devices as devices
 import modules.shared as shared
 import modules.sd_samplers as sd_samplers
@@ -26,7 +24,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
     results = []
     if p.enable_hr and p.hr_upscaler != 'None' and p.denoising_strength > 0 and len(getattr(p, 'init_images', [])) == 0:
         p.is_hr_pass = True
-    is_refiner_enabled = p.enable_hr and p.refiner_steps > 0 and shared.sd_refiner is not None
+    is_refiner_enabled = p.enable_hr and p.refiner_steps > 0 and p.refiner_start > 0 and p.refiner_start < 1 and shared.sd_refiner is not None
 
     def hires_resize(latents): # input=latents output=pil
         latent_upscaler = shared.latent_upscale_modes.get(p.hr_upscaler, None)
@@ -36,10 +34,11 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
         first_pass_images = vae_decode(latents=latents, model=shared.sd_model, full_quality=True, output_type='pil')
         p.init_images = []
         for first_pass_image in first_pass_images:
-            init_image = images.resize_image(1, first_pass_image, p.hr_upscale_to_x, p.hr_upscale_to_y, upscaler_name=p.hr_upscaler) if latent_upscaler is None else first_pass_image
+            if latent_upscaler is None:
+                init_image = images.resize_image(1, first_pass_image, p.hr_upscale_to_x, p.hr_upscale_to_y, upscaler_name=p.hr_upscaler)
+            else:
+                init_image = first_pass_image
             p.init_images.append(init_image)
-        p.width = p.hr_upscale_to_x
-        p.height = p.hr_upscale_to_y
 
     def save_intermediate(latents, suffix):
         for i in range(len(latents)):
@@ -321,12 +320,12 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
     if p.is_hr_pass:
         p.init_hr()
         if p.width != p.hr_upscale_to_x or p.height != p.hr_upscale_to_y:
+            p.ops.append('upscale')
             if shared.opts.save and not p.do_not_save_samples and shared.opts.save_images_before_highres_fix and hasattr(shared.sd_model, 'vae'):
                 save_intermediate(latents=output.images, suffix="-before-hires")
-            if latent_scale_mode is not None:
-                p.ops.append('hires')
+            hires_resize(latents=output.images)
+            if latent_scale_mode is not None or p.hr_force:
                 recompile_model(hires=True)
-                hires_resize(latents=output.images)
                 sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE)
                 hires_args = set_pipeline_args(
                     model=shared.sd_model,
@@ -372,6 +371,11 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
         refiner_is_sdxl = bool("StableDiffusionXL" in shared.sd_refiner.__class__.__name__)
         p.ops.append('refine')
         for i in range(len(output.images)):
+            image = output.images[i]
+            if (image.shape[2] == 3) and (image.shape[0] % 8 != 0 or image.shape[1] % 8 != 0):
+                shared.log.warning(f'Refiner requires image size to be divisible by 8: {image.shape}')
+                results.append(image)
+                return results
             refiner_args = set_pipeline_args(
                 model=shared.sd_refiner,
                 prompts=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts[i],
@@ -383,7 +387,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
                 guidance_rescale=p.diffusers_guidance_rescale,
                 denoising_start=p.refiner_start if p.refiner_start > 0 and p.refiner_start < 1 else None,
                 denoising_end=1 if p.refiner_start > 0 and p.refiner_start < 1 else None,
-                image=output.images[i],
+                image=image,
                 output_type='latent' if hasattr(shared.sd_refiner, 'vae') else 'np',
                 clip_skip=p.clip_skip,
                 desc='Refiner',
@@ -403,20 +407,6 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
             shared.sd_refiner.to(devices.cpu)
             devices.torch_gc()
 
-    if p.is_hr_pass and latent_scale_mode is None:
-        if p.width != p.hr_upscale_to_x or p.height != p.hr_upscale_to_y:
-            p.ops.append('upscale')
-            if not is_refiner_enabled:
-                results = vae_decode(latents=output.images, model=shared.sd_model, full_quality=p.full_quality)
-            upscaled = []
-            for image in results:
-                image = (image * 255.0).astype(np.uint8)
-                image = Image.fromarray(image)
-                image = images.resize_image(1, image, p.hr_upscale_to_x, p.hr_upscale_to_y, upscaler_name=p.hr_upscaler)
-                image = np.array(image).astype(np.float32) / 255.0
-                upscaled.append(image)
-            return upscaled
-
     # final decode since there is no refiner
     if not is_refiner_enabled:
         results = vae_decode(latents=output.images, model=shared.sd_model, full_quality=p.full_quality)

modules/txt2img.py

@@ -4,9 +4,9 @@ from modules.generation_parameters_copypaste import create_override_settings_dic
 from modules.ui import plaintext_to_html
 
 
-def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, steps: int, sampler_index: int, latent_index: int, full_quality: bool, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, image_cfg_scale: float, diffusers_guidance_rescale: float, clip_skip: int, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, height: int, width: int, enable_hr: bool, denoising_strength: float, hr_scale: float, hr_upscaler: str, hr_second_pass_steps: int, hr_resize_x: int, hr_resize_y: int, refiner_steps: int, refiner_start: int, refiner_prompt: str, refiner_negative: str, override_settings_texts, *args): # pylint: disable=unused-argument
+def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, steps: int, sampler_index: int, latent_index: int, full_quality: bool, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, image_cfg_scale: float, diffusers_guidance_rescale: float, clip_skip: int, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, height: int, width: int, enable_hr: bool, denoising_strength: float, hr_scale: float, hr_upscaler: str, hr_force: bool, hr_second_pass_steps: int, hr_resize_x: int, hr_resize_y: int, refiner_steps: int, refiner_start: int, refiner_prompt: str, refiner_negative: str, override_settings_texts, *args): # pylint: disable=unused-argument
 
-    shared.log.debug(f'txt2img: id_task={id_task}|prompt={prompt}|negative_prompt={negative_prompt}|prompt_styles={prompt_styles}|steps={steps}|sampler_index={sampler_index}|latent_index={latent_index}|full_quality={full_quality}|restore_faces={restore_faces}|tiling={tiling}|n_iter={n_iter}|batch_size={batch_size}|cfg_scale={cfg_scale}|clip_skip={clip_skip}|seed={seed}|subseed={subseed}|subseed_strength={subseed_strength}|seed_resize_from_h={seed_resize_from_h}|seed_resize_from_w={seed_resize_from_w}||height={height}|width={width}|enable_hr={enable_hr}|denoising_strength={denoising_strength}|hr_scale={hr_scale}|hr_upscaler={hr_upscaler}|hr_second_pass_steps={hr_second_pass_steps}|hr_resize_x={hr_resize_x}|hr_resize_y={hr_resize_y}|image_cfg_scale={image_cfg_scale}|diffusers_guidance_rescale={diffusers_guidance_rescale}|refiner_steps={refiner_steps}|refiner_start={refiner_start}||refiner_prompt={refiner_prompt}|refiner_negative={refiner_negative}|override_settings_texts={override_settings_texts}')
+    shared.log.debug(f'txt2img: id_task={id_task}|prompt={prompt}|negative_prompt={negative_prompt}|prompt_styles={prompt_styles}|steps={steps}|sampler_index={sampler_index}|latent_index={latent_index}|full_quality={full_quality}|restore_faces={restore_faces}|tiling={tiling}|n_iter={n_iter}|batch_size={batch_size}|cfg_scale={cfg_scale}|clip_skip={clip_skip}|seed={seed}|subseed={subseed}|subseed_strength={subseed_strength}|seed_resize_from_h={seed_resize_from_h}|seed_resize_from_w={seed_resize_from_w}||height={height}|width={width}|enable_hr={enable_hr}|denoising_strength={denoising_strength}|hr_scale={hr_scale}|hr_upscaler={hr_upscaler}|hr_force={hr_force}|hr_second_pass_steps={hr_second_pass_steps}|hr_resize_x={hr_resize_x}|hr_resize_y={hr_resize_y}|image_cfg_scale={image_cfg_scale}|diffusers_guidance_rescale={diffusers_guidance_rescale}|refiner_steps={refiner_steps}|refiner_start={refiner_start}|refiner_prompt={refiner_prompt}|refiner_negative={refiner_negative}|override_settings_texts={override_settings_texts}')
 
     if shared.sd_model is None:
         shared.log.warning('Model not loaded')
@@ -49,6 +49,7 @@ def txt2img(id_task: str, prompt: str, negative_prompt: str, prompt_styles, step
         denoising_strength=denoising_strength,
         hr_scale=hr_scale,
         hr_upscaler=hr_upscaler,
+        hr_force=hr_force,
         hr_second_pass_steps=hr_second_pass_steps,
         hr_resize_x=hr_resize_x,
         hr_resize_y=hr_resize_y,

modules/ui.py

@@ -396,8 +396,9 @@ def create_ui(startup_timer = None):
                         hr_final_resolution = FormHTML(value="", elem_id="txtimg_hr_finalres", label="Upscaled resolution", interactive=False)
                     with FormRow(elem_id="txt2img_hires_fix_row1", variant="compact"):
                         hr_upscaler = gr.Dropdown(label="Upscaler", elem_id="txt2img_hr_upscaler", choices=[*modules.shared.latent_upscale_modes, *[x.name for x in modules.shared.sd_upscalers]], value=modules.shared.latent_upscale_default_mode)
-                        hr_second_pass_steps = gr.Slider(minimum=0, maximum=99, step=1, label='Hires steps', elem_id="txt2img_steps_alt", value=20)
+                        hr_force = gr.Checkbox(label='Force Hires', value=False, elem_id="txt2img_hr_force")
                     with FormRow(elem_id="txt2img_hires_fix_row2", variant="compact"):
+                        hr_second_pass_steps = gr.Slider(minimum=0, maximum=99, step=1, label='Hires steps', elem_id="txt2img_steps_alt", value=20)
                         hr_scale = gr.Slider(minimum=1.0, maximum=4.0, step=0.05, label="Upscale by", value=2.0, elem_id="txt2img_hr_scale")
                     with FormRow(elem_id="txt2img_hires_fix_row3", variant="compact"):
                         hr_resize_x = gr.Slider(minimum=0, maximum=4096, step=8, label="Resize width to", value=0, elem_id="txt2img_hr_resize_x")
@@ -450,7 +451,7 @@ def create_ui(startup_timer = None):
                     seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w,
                     height, width,
                     show_second_pass, denoising_strength,
-                    hr_scale, hr_upscaler, hr_second_pass_steps, hr_resize_x, hr_resize_y,
+                    hr_scale, hr_upscaler, hr_force, hr_second_pass_steps, hr_resize_x, hr_resize_y,
                     refiner_steps, refiner_start, refiner_prompt, refiner_negative,
                     override_settings,
                 ] + custom_inputs,