base and refiner mix and match

modules/generation_parameters_copypaste.py

@@ -202,39 +202,6 @@ def find_hypernetwork_key(hypernet_name, hypernet_hash=None):
     return None
 
 
-def restore_old_hires_fix_params(res):
-    """for infotexts that specify old First pass size parameter, convert it into
-    width, height, and hr scale"""
-
-    firstpass_width = res.get('First pass size-1', None)
-    firstpass_height = res.get('First pass size-2', None)
-
-    if shared.opts.use_old_hires_fix_width_height:
-        hires_width = int(res.get("Hires resize-1", 0))
-        hires_height = int(res.get("Hires resize-2", 0))
-
-        if hires_width and hires_height:
-            res['Size-1'] = hires_width
-            res['Size-2'] = hires_height
-            return
-
-    if firstpass_width is None or firstpass_height is None:
-        return
-
-    firstpass_width, firstpass_height = int(firstpass_width), int(firstpass_height)
-    width = int(res.get("Size-1", 512))
-    height = int(res.get("Size-2", 512))
-
-    if firstpass_width == 0 or firstpass_height == 0:
-        from modules import processing
-        firstpass_width, firstpass_height = processing.old_hires_fix_first_pass_dimensions(width, height)
-
-    res['Size-1'] = firstpass_width
-    res['Size-2'] = firstpass_height
-    res['Hires resize-1'] = width
-    res['Hires resize-2'] = height
-
-
 def parse_generation_parameters(x: str):
     """parses generation parameters string, the one you see in text field under the picture in UI:
 ```
@@ -288,7 +255,6 @@ Steps: 20, Sampler: Euler a, CFG scale: 7, Seed: 965400086, Size: 512x512, Model
     if "Hires resize-1" not in res:
         res["Hires resize-1"] = 0
         res["Hires resize-2"] = 0
-    restore_old_hires_fix_params(res)
     return res
 
 

modules/images.py

@@ -385,6 +385,8 @@ class FilenameGenerator:
 
     def apply(self, x):
         res = ''
+        if self.p is None:
+            return res
         for m in re_pattern.finditer(x):
             text, pattern = m.groups()
             if pattern is None:
@@ -629,7 +631,7 @@ def read_info_from_image(image):
                 for key, val in subkey.items():
                     if isinstance(val, bytes): # decode bytestring
                         val = safe_decode_string(val)
-                    if isinstance(val, tuple) and isinstance(val[0], int) and isinstance(val[1], int): # convert camera ratios
+                    if isinstance(val, tuple) and isinstance(val[0], int) and isinstance(val[1], int) and val[1] > 0: # convert camera ratios
                         val = round(val[0] / val[1], 2)
                     if val is not None and key in ExifTags.TAGS: # add known tags
                         if ExifTags.TAGS[key] == 'UserComment': # add geninfo from UserComment

modules/processing.py

@@ -455,7 +455,8 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts, all_seeds, all_su
     vae = (None if not shared.opts.add_model_name_to_info or sd_vae.loaded_vae_file is None else os.path.splitext(os.path.basename(sd_vae.loaded_vae_file))[0]) if p.full_quality else 'TAESD'
     comment = ', '.join(comments) if comments is not None and type(comments) is list else None
 
-    generation_params = {
+    args = {
+        # basic
         "Steps": p.steps,
         "Seed": all_seeds[index],
         "Sampler": p.sampler_name,
@@ -465,42 +466,66 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts, all_seeds, all_su
         "Parser": shared.opts.prompt_attention,
         "Model": None if (not shared.opts.add_model_name_to_info) or (not shared.sd_model.sd_checkpoint_info.model_name) else shared.sd_model.sd_checkpoint_info.model_name.replace(',', '').replace(':', ''),
         "Model hash": getattr(p, 'sd_model_hash', None if (not shared.opts.add_model_hash_to_info) or (not shared.sd_model.sd_model_hash) else shared.sd_model.sd_model_hash),
-        "Refiner": None if (not shared.opts.add_model_name_to_info) or (not shared.sd_refiner) or (not shared.sd_refiner.sd_checkpoint_info.model_name) else shared.sd_refiner.sd_checkpoint_info.model_name.replace(',', '').replace(':', ''),
         "VAE": vae,
-        # subseed
         "Variation seed": None if p.subseed_strength == 0 else all_subseeds[index],
         "Variation strength": None if p.subseed_strength == 0 else p.subseed_strength,
-        # seed resize
         "Seed resize from": None if p.seed_resize_from_w == 0 or p.seed_resize_from_h == 0 else f"{p.seed_resize_from_w}x{p.seed_resize_from_h}",
-        "Init image hash": getattr(p, 'init_img_hash', None),
-        "Conditional mask weight": getattr(p, "inpainting_mask_weight", shared.opts.inpainting_mask_weight) if p.is_using_inpainting_conditioning else None,
-        # clip skip
         "Clip skip": p.clip_skip if p.clip_skip > 1 else None,
-        # ensd
+        "Prompt2": p.refiner_prompt if len(p.refiner_prompt) > 0 else None,
+        "Negative2": p.refiner_negative if len(p.refiner_negative) > 0 else None,
+        # other
         "ENSD": shared.opts.eta_noise_seed_delta if shared.opts.eta_noise_seed_delta != 0 and sd_samplers_common.is_sampler_using_eta_noise_seed_delta(p) else None,
-        # restore_faces, tiling
-        "Face restoration": shared.opts.face_restoration_model if p.restore_faces else None,
         "Tiling": p.tiling if p.tiling else None,
-        # enable_hr
-        "Prompt2": p.refiner_prompt if p.enable_hr and len(p.refiner_prompt) > 0 else None,
-        "Negative2": p.refiner_negative if p.enable_hr and len(p.refiner_negative) > 0 else None,
-        "Latent sampler": p.latent_sampler if p.enable_hr and p.latent_sampler != p.sampler_name else None,
-        "Denoising strength": p.denoising_strength if p.enable_hr else None,
-        "Image CFG Scale": p.image_cfg_scale,
         # sdnext
         "Backend": 'Diffusers' if shared.backend == shared.Backend.DIFFUSERS else 'Original',
         "Version": git_commit,
         "Comment": comment,
-        "Operations": ', '.join(list(set(p.ops))) if len(p.ops) > 0 else None,
+        "Operations": ', '.join(list(set(p.ops))).replace('"', '') if len(p.ops) > 0 else None,
     }
+    if 'txt2img' in p.ops:
+        pass
+    if 'hires' in p.ops:
+        args["Hires steps"] = p.hr_second_pass_steps
+        args["Hires upscaler"] = p.hr_upscaler
+        args["Hires upscale"] = p.hr_scale
+        args["Hires resize"] = f"{p.hr_resize_x}x{p.hr_resize_y}"
+        args["Hires size"] = f"{p.hr_upscale_to_x}x{p.hr_upscale_to_y}"
+        args["Denoising strength"] = p.denoising_strength
+        args["Latent sampler"] = p.latent_sampler
+        args["Image CFG scale"] = p.image_cfg_scale
+        args["CFG rescale"] = p.diffusers_guidance_rescale if shared.backend == shared.Backend.DIFFUSERS else None
+    if 'refine' in p.ops:
+        args["Refiner"] = None if (not shared.opts.add_model_name_to_info) or (not shared.sd_refiner) or (not shared.sd_refiner.sd_checkpoint_info.model_name) else shared.sd_refiner.sd_checkpoint_info.model_name.replace(',', '').replace(':', '')
+        args['Image CFG scale'] = p.image_cfg_scale
+        args['Refiner steps'] = p.refiner_steps
+        args['Refiner start'] = p.refiner_start
+        args["Hires steps"] = p.hr_second_pass_steps
+        args["Latent sampler"] = p.latent_sampler
+        args["CFG rescale"] = p.diffusers_guidance_rescale if shared.backend == shared.Backend.DIFFUSERS else None
+    if 'img2img' in p.ops or 'inpaint' in p.ops:
+        args["Init image size"] = f"{getattr(p, 'init_img_width', 0)}x{getattr(p, 'init_img_height', 0)}"
+        args["Init image hash"] = getattr(p, 'init_img_hash', None)
+        args["Conditional mask weight"] = getattr(p, "inpainting_mask_weight", shared.opts.inpainting_mask_weight) if p.is_using_inpainting_conditioning else None
+        args['Resize mode'] = p.resize_mode
+        args["Mask blur"] = p.mask_blur if p.mask is not None and p.mask_blur > 0 else None
+        args["Noise multiplier"] = p.initial_noise_multiplier if p.initial_noise_multiplier != 1.0 else None
+        args["Denoising strength"] = p.denoising_strength
+    if 'face' in p.ops:
+        args["Face restoration"] = shared.opts.face_restoration_model
+    if 'color' in p.ops:
+        args["Color correction"] = True
+
+
+    # tome
     token_merging_ratio = p.get_token_merging_ratio()
     token_merging_ratio_hr = p.get_token_merging_ratio(for_hr=True) if p.enable_hr else None
-    generation_params['Token merging ratio'] = token_merging_ratio if token_merging_ratio != 0 else None
-    generation_params['Token merging ratio hr'] = token_merging_ratio_hr if token_merging_ratio_hr != 0 else None
-    generation_params.update(p.extra_generation_params)
-    generation_params_text = ", ".join([k if k == v else f'{k}: {generation_parameters_copypaste.quote(v)}' for k, v in generation_params.items() if v is not None])
+    args['Token merging ratio'] = token_merging_ratio if token_merging_ratio != 0 else None
+    args['Token merging ratio hr'] = token_merging_ratio_hr if token_merging_ratio_hr != 0 else None
+
+    args.update(p.extra_generation_params)
+    params_text = ", ".join([k if k == v else f'{k}: {generation_parameters_copypaste.quote(v)}' for k, v in args.items() if v is not None])
     negative_prompt_text = f"\nNegative prompt: {all_negative_prompts[index]}" if all_negative_prompts[index] else ""
-    infotext = f"{all_prompts[index]}{negative_prompt_text}\n{generation_params_text}".strip()
+    infotext = f"{all_prompts[index]}{negative_prompt_text}\n{params_text}".strip()
     return infotext
 
 
@@ -886,19 +911,10 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
         self.height = self.height or 512
 
     def init_hr(self):
-        if shared.opts.use_old_hires_fix_width_height and self.applied_old_hires_behavior_to != (self.width, self.height):
-            self.hr_resize_x = self.width
-            self.hr_resize_y = self.height
-            self.hr_upscale_to_x = self.width
-            self.hr_upscale_to_y = self.height
-            self.width, self.height = old_hires_fix_first_pass_dimensions(self.width, self.height)
-            self.applied_old_hires_behavior_to = (self.width, self.height)
         if self.hr_resize_x == 0 and self.hr_resize_y == 0:
-            self.extra_generation_params["Hires upscale"] = self.hr_scale
             self.hr_upscale_to_x = int(self.width * self.hr_scale)
             self.hr_upscale_to_y = int(self.height * self.hr_scale)
         else:
-            self.extra_generation_params["Hires resize"] = f"{self.hr_resize_x}x{self.hr_resize_y}"
             if self.hr_resize_y == 0:
                 self.hr_upscale_to_x = self.hr_resize_x
                 self.hr_upscale_to_y = self.hr_resize_x * self.height // self.width
@@ -908,10 +924,6 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
             else:
                 target_w = self.hr_resize_x
                 target_h = self.hr_resize_y
-                """
-                self.hr_upscale_to_x = self.hr_resize_x
-                self.hr_upscale_to_y = self.hr_resize_y
-                """
                 src_ratio = self.width / self.height
                 dst_ratio = self.hr_resize_x / self.hr_resize_y
                 if src_ratio < dst_ratio:
@@ -923,21 +935,16 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
                 self.truncate_x = (self.hr_upscale_to_x - target_w) // 8
                 self.truncate_y = (self.hr_upscale_to_y - target_h) // 8
         # special case: the user has chosen to do nothing
-        if self.hr_upscale_to_x == self.width and self.hr_upscale_to_y == self.height:
-            self.extra_generation_params.pop("Hires upscale", None)
-            self.extra_generation_params.pop("Hires resize", None)
+        if (self.hr_upscale_to_x == self.width and self.hr_upscale_to_y == self.height) or self.hr_upscaler is None or self.hr_upscaler == 'None':
+            self.is_hr_pass = False
             return
+        self.is_hr_pass = True
         if not shared.state.processing_has_refined_job_count:
             if shared.state.job_count == -1:
                 shared.state.job_count = self.n_iter
             shared.state.job_count = shared.state.job_count * 2
             shared.state.processing_has_refined_job_count = True
-        if self.hr_second_pass_steps:
-            self.extra_generation_params["Hires steps"] = self.hr_second_pass_steps
-        if self.hr_upscaler is not None:
-            self.extra_generation_params["Hires upscaler"] = self.hr_upscaler
-            shared.log.debug(f'Init hires: upscaler={self.hr_upscaler} sampler={self.latent_sampler} resize={self.hr_resize_x}x{self.hr_resize_y} upscale={self.hr_upscale_to_x}x{self.hr_upscale_to_y}')
-        self.extra_generation_params["Secondary sampler"] = self.latent_sampler
+        shared.log.debug(f'Init hires: upscaler={self.hr_upscaler} sampler={self.latent_sampler} resize={self.hr_resize_x}x{self.hr_resize_y} upscale={self.hr_upscale_to_x}x{self.hr_upscale_to_y}')
 
     def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts):
 
@@ -961,7 +968,7 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
 
         self.ops.append('txt2img')
         self.sampler = sd_samplers.create_sampler(self.sampler_name, self.sd_model)
-        latent_scale_mode = shared.latent_upscale_modes.get(self.hr_upscaler, None) if self.hr_upscaler is not None else shared.latent_upscale_modes.get(shared.latent_upscale_default_mode, "nearest")
+        latent_scale_mode = shared.latent_upscale_modes.get(self.hr_upscaler, None) if self.hr_upscaler is not None else shared.latent_upscale_modes.get(shared.latent_upscale_default_mode, "None")
         if self.enable_hr and latent_scale_mode is None:
             if len([x for x in shared.sd_upscalers if x.name == self.hr_upscaler]) == 0:
                 shared.log.warning("Could not find upscaler to use with hrfix")
@@ -970,57 +977,59 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
         samples = self.sampler.sample(self, x, conditioning, unconditional_conditioning, image_conditioning=self.txt2img_image_conditioning(x))
         if not self.enable_hr or shared.state.interrupted or shared.state.skipped:
             return samples
-        self.is_hr_pass = True
-        self.init_hr()
-        self.ops.append('hires')
-        target_width = self.hr_upscale_to_x
-        target_height = self.hr_upscale_to_y
 
-        if latent_scale_mode is not None:
-            for i in range(samples.shape[0]):
-                save_intermediate(samples, i)
-            samples = torch.nn.functional.interpolate(samples, size=(target_height // 8, target_width // 8), mode=latent_scale_mode["mode"], antialias=latent_scale_mode["antialias"])
-            if getattr(self, "inpainting_mask_weight", shared.opts.inpainting_mask_weight) < 1.0:
-                image_conditioning = self.img2img_image_conditioning(decode_first_stage(self.sd_model, samples.to(dtype=devices.dtype_vae)), samples)
+        self.init_hr()
+        if self.is_hr_pass:
+            self.ops.append('hires')
+            target_width = self.hr_upscale_to_x
+            target_height = self.hr_upscale_to_y
+
+            if latent_scale_mode is not None:
+                for i in range(samples.shape[0]):
+                    save_intermediate(samples, i)
+                samples = torch.nn.functional.interpolate(samples, size=(target_height // 8, target_width // 8), mode=latent_scale_mode["mode"], antialias=latent_scale_mode["antialias"])
+                if getattr(self, "inpainting_mask_weight", shared.opts.inpainting_mask_weight) < 1.0:
+                    image_conditioning = self.img2img_image_conditioning(decode_first_stage(self.sd_model, samples.to(dtype=devices.dtype_vae)), samples)
+                else:
+                    image_conditioning = self.txt2img_image_conditioning(samples.to(dtype=devices.dtype_vae))
             else:
-                image_conditioning = self.txt2img_image_conditioning(samples.to(dtype=devices.dtype_vae))
-        else:
-            decoded_samples = decode_first_stage(self.sd_model, samples.to(dtype=devices.dtype_vae))
-            lowres_samples = torch.clamp((decoded_samples + 1.0) / 2.0, min=0.0, max=1.0)
-            batch_images = []
-            for i, x_sample in enumerate(lowres_samples):
-                x_sample = 255. * np.moveaxis(x_sample.cpu().numpy(), 0, 2)
-                x_sample = validate_sample(x_sample)
-                image = Image.fromarray(x_sample)
-                save_intermediate(image, i)
-                image = images.resize_image(1, image, target_width, target_height, upscaler_name=self.hr_upscaler)
-                image = np.array(image).astype(np.float32) / 255.0
-                image = np.moveaxis(image, 2, 0)
-                batch_images.append(image)
-            decoded_samples = torch.from_numpy(np.array(batch_images))
-            decoded_samples = decoded_samples.to(device=shared.device, dtype=devices.dtype_vae)
-            decoded_samples = 2. * decoded_samples - 1.
-            if shared.opts.sd_vae_sliced_encode and len(decoded_samples) > 1:
-                samples = torch.stack([
-                    self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(torch.unsqueeze(decoded_sample, 0)))[0]
-                    for decoded_sample
-                    in decoded_samples
-                ])
-            else:
-                samples = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(decoded_samples))
-            image_conditioning = self.img2img_image_conditioning(decoded_samples, samples)
-        shared.state.nextjob()
-        if self.latent_sampler == "PLMS":
-            self.latent_sampler = 'UniPC'
-        self.sampler = sd_samplers.create_sampler(self.latent_sampler or self.sampler_name, self.sd_model)
-        samples = samples[:, :, self.truncate_y//2:samples.shape[2]-(self.truncate_y+1)//2, self.truncate_x//2:samples.shape[3]-(self.truncate_x+1)//2]
-        noise = create_random_tensors(samples.shape[1:], seeds=seeds, subseeds=subseeds, subseed_strength=subseed_strength, p=self)
-        x = None
-        devices.torch_gc() # GC now before running the next img2img to prevent running out of memory
-        sd_models.apply_token_merging(self.sd_model, self.get_token_merging_ratio(for_hr=True))
-        samples = self.sampler.sample_img2img(self, samples, noise, conditioning, unconditional_conditioning, steps=self.hr_second_pass_steps or self.steps, image_conditioning=image_conditioning)
-        sd_models.apply_token_merging(self.sd_model, self.get_token_merging_ratio())
-        self.is_hr_pass = False
+                decoded_samples = decode_first_stage(self.sd_model, samples.to(dtype=devices.dtype_vae))
+                lowres_samples = torch.clamp((decoded_samples + 1.0) / 2.0, min=0.0, max=1.0)
+                batch_images = []
+                for i, x_sample in enumerate(lowres_samples):
+                    x_sample = 255. * np.moveaxis(x_sample.cpu().numpy(), 0, 2)
+                    x_sample = validate_sample(x_sample)
+                    image = Image.fromarray(x_sample)
+                    save_intermediate(image, i)
+                    image = images.resize_image(1, image, target_width, target_height, upscaler_name=self.hr_upscaler)
+                    image = np.array(image).astype(np.float32) / 255.0
+                    image = np.moveaxis(image, 2, 0)
+                    batch_images.append(image)
+                decoded_samples = torch.from_numpy(np.array(batch_images))
+                decoded_samples = decoded_samples.to(device=shared.device, dtype=devices.dtype_vae)
+                decoded_samples = 2. * decoded_samples - 1.
+                if shared.opts.sd_vae_sliced_encode and len(decoded_samples) > 1:
+                    samples = torch.stack([
+                        self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(torch.unsqueeze(decoded_sample, 0)))[0]
+                        for decoded_sample
+                        in decoded_samples
+                    ])
+                else:
+                    samples = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(decoded_samples))
+                image_conditioning = self.img2img_image_conditioning(decoded_samples, samples)
+            shared.state.nextjob()
+            if self.latent_sampler == "PLMS":
+                self.latent_sampler = 'UniPC'
+            self.sampler = sd_samplers.create_sampler(self.latent_sampler or self.sampler_name, self.sd_model)
+            samples = samples[:, :, self.truncate_y//2:samples.shape[2]-(self.truncate_y+1)//2, self.truncate_x//2:samples.shape[3]-(self.truncate_x+1)//2]
+            noise = create_random_tensors(samples.shape[1:], seeds=seeds, subseeds=subseeds, subseed_strength=subseed_strength, p=self)
+            x = None
+            devices.torch_gc() # GC now before running the next img2img to prevent running out of memory
+            sd_models.apply_token_merging(self.sd_model, self.get_token_merging_ratio(for_hr=True))
+            samples = self.sampler.sample_img2img(self, samples, noise, conditioning, unconditional_conditioning, steps=self.hr_second_pass_steps or self.steps, image_conditioning=image_conditioning)
+            sd_models.apply_token_merging(self.sd_model, self.get_token_merging_ratio())
+            self.is_hr_pass = False
+
         return samples
 
 
@@ -1099,9 +1108,10 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
         imgs = []
         unprocessed = []
         for img in self.init_images:
-            # Save init image
+            self.init_img_hash = hashlib.sha256(img.tobytes()).hexdigest()[0:8] # pylint: disable=attribute-defined-outside-init
+            self.init_img_width = img.width # pylint: disable=attribute-defined-outside-init
+            self.init_img_height = img.height # pylint: disable=attribute-defined-outside-init
             if shared.opts.save_init_img:
-                self.init_img_hash = hashlib.sha256(img.tobytes()).hexdigest()[0:8] # pylint: disable=attribute-defined-outside-init
                 images.save_image(img, path=shared.opts.outdir_init_images, basename=None, forced_filename=self.init_img_hash, save_to_dirs=False)
             image = images.flatten(img, shared.opts.img2img_background_color)
             if crop_region is None and self.resize_mode != 4:
@@ -1174,9 +1184,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
                 self.sd_model.dtype = self.sd_model.unet.dtype
 
         x = create_random_tensors([4, self.height // 8, self.width // 8], seeds=seeds, subseeds=subseeds, subseed_strength=self.subseed_strength, seed_resize_from_h=self.seed_resize_from_h, seed_resize_from_w=self.seed_resize_from_w, p=self)
-        if self.initial_noise_multiplier != 1.0:
-            self.extra_generation_params["Noise multiplier"] = self.initial_noise_multiplier
-            x *= self.initial_noise_multiplier
+        x *= self.initial_noise_multiplier
         samples = self.sampler.sample_img2img(self, self.init_latent, x, conditioning, unconditional_conditioning, image_conditioning=self.image_conditioning)
         if self.mask is not None:
             samples = samples * self.nmask + self.init_latent * self.mask

modules/processing_diffusers.py

@@ -44,8 +44,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
             from modules.processing import create_infotext
             info=create_infotext(p, p.all_prompts, p.all_seeds, p.all_subseeds, [], iteration=p.iteration, position_in_batch=i)
             decoded = vae_decode(latents=latents, model=shared.sd_model, output_type='pil', full_quality=p.full_quality)
-            for i in range(len(decoded)):
-                images.save_image(decoded[i], path=p.outpath_samples, basename="", seed=seeds[i], prompt=prompts[i], extension=shared.opts.samples_format, info=info, p=p, suffix=suffix)
+            for j in range(len(decoded)):
+                images.save_image(decoded[j], path=p.outpath_samples, basename="", seed=seeds[i], prompt=prompts[i], extension=shared.opts.samples_format, info=info, p=p, suffix=suffix)
 
     def diffusers_callback(_step: int, _timestep: int, latents: torch.FloatTensor):
         shared.state.sampling_step += 1
@@ -123,12 +123,15 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
                 negative_prompts_2.append(negative_prompts_2[-1])
         return prompts, negative_prompts, prompts_2, negative_prompts_2
 
-    def set_pipeline_args(model, prompts: list, negative_prompts: list, prompts_2: typing.Optional[list]=None, negative_prompts_2: typing.Optional[list]=None, is_refiner: bool=False, desc:str='', **kwargs):
+    def set_pipeline_args(model, prompts: list, negative_prompts: list, prompts_2: typing.Optional[list]=None, negative_prompts_2: typing.Optional[list]=None, desc:str='', **kwargs):
+        try:
+            is_refiner = model.text_encoder.__class__.__name__ != 'CLIPTextModel'
+        except Exception:
+            is_refiner = False
         if hasattr(model, "set_progress_bar_config"):
             model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} '+desc, ncols=80, colour='#327fba')
         args = {}
-        pipeline = model
-        signature = inspect.signature(type(pipeline).__call__)
+        signature = inspect.signature(type(model).__call__)
         possible = signature.parameters.keys()
         generator_device = devices.cpu if shared.opts.diffusers_generator_device == "cpu" else shared.device
         generator = [torch.Generator(generator_device).manual_seed(s) for s in seeds]
@@ -141,24 +144,20 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
             prompt_embed, pooled, negative_embed, negative_pooled = prompt_parser_diffusers.compel_encode_prompts(model, prompts, negative_prompts, prompts_2, negative_prompts_2, is_refiner, kwargs.pop("clip_skip", None))
         if 'prompt' in possible:
             if hasattr(model, 'text_encoder') and 'prompt_embeds' in possible and prompt_embed is not None:
+                if type(pooled) == list:
+                    pooled = pooled[0]
+                if type(negative_pooled) == list:
+                    negative_pooled = negative_pooled[0]
                 args['prompt_embeds'] = prompt_embed
-                if not is_refiner and shared.sd_model_type == "sdxl":
+                if 'XL' in model.__class__.__name__:
                     args['pooled_prompt_embeds'] = pooled
-                    # args['prompt_2'] = None # Cannot pass prompts when passing embeds
-                if is_refiner and shared.sd_refiner_type == "sdxl":
-                    args['pooled_prompt_embeds'] = pooled
-                    # args['prompt_2'] = None # Cannot pass prompts when passing embeds
             else:
                 args['prompt'] = prompts
         if 'negative_prompt' in possible:
             if hasattr(model, 'text_encoder') and 'negative_prompt_embeds' in possible and negative_embed is not None:
                 args['negative_prompt_embeds'] = negative_embed
-                if not is_refiner and shared.sd_model_type == "sdxl":
+                if 'XL' in model.__class__.__name__:
                     args['negative_pooled_prompt_embeds'] = negative_pooled
-                    # args['negative_prompt_2'] = None
-                if is_refiner and shared.sd_refiner_type == "sdxl":
-                    args['negative_pooled_prompt_embeds'] = negative_pooled
-                    # args['negative_prompt_2'] = None
             else:
                 args['negative_prompt'] = negative_prompts
         if 'guidance_scale' in possible:
@@ -198,7 +197,7 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
         if 'negative_pooled_prompt_embeds' in clean:
             clean['negative_pooled_prompt_embeds'] = clean['negative_pooled_prompt_embeds'].shape if torch.is_tensor(clean['negative_pooled_prompt_embeds']) else type(clean['negative_pooled_prompt_embeds'])
         clean['generator'] = generator_device
-        shared.log.debug(f'Diffuser pipeline: {pipeline.__class__.__name__} task={sd_models.get_diffusers_task(model)} set={clean}')
+        shared.log.debug(f'Diffuser pipeline: {model.__class__.__name__} task={sd_models.get_diffusers_task(model)} set={clean}')
         return args
 
     def recompile_model(hires=False):
@@ -294,7 +293,6 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
         denoising_start=0 if use_refiner_start else p.refiner_start if use_denoise_start else None,
         denoising_end=p.refiner_start if use_refiner_start else 1 if use_denoise_start else None,
         output_type='latent' if hasattr(shared.sd_model, 'vae') else 'np',
-        is_refiner=False,
         clip_skip=p.clip_skip,
         desc='Base',
         **task_specific_kwargs
@@ -334,7 +332,6 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
                 guidance_scale=p.image_cfg_scale if p.image_cfg_scale is not None else p.cfg_scale,
                 guidance_rescale=p.diffusers_guidance_rescale,
                 output_type='latent' if hasattr(shared.sd_model, 'vae') else 'np',
-                is_refiner=False,
                 clip_skip=p.clip_skip,
                 image=p.init_images,
                 strength=p.denoising_strength,
@@ -381,7 +378,6 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
                 denoising_end=1 if p.refiner_start > 0 and p.refiner_start < 1 else None,
                 image=output.images[i],
                 output_type='latent' if hasattr(shared.sd_refiner, 'vae') else 'np',
-                is_refiner=True,
                 clip_skip=p.clip_skip,
                 desc='Refiner',
             )
@@ -390,12 +386,6 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
             except AssertionError as e:
                 shared.log.info(e)
 
-            p.extra_generation_params['Image CFG scale'] = p.image_cfg_scale if p.image_cfg_scale is not None else None
-            p.extra_generation_params['Refiner steps'] = p.refiner_steps
-            p.extra_generation_params['Refiner start'] = p.refiner_start
-            p.extra_generation_params["Hires steps"] = p.hr_second_pass_steps
-            p.extra_generation_params["Secondary sampler"] = p.latent_sampler
-
             if not shared.state.interrupted and not shared.state.skipped:
                 refiner_images = vae_decode(latents=refiner_output.images, model=shared.sd_refiner, full_quality=True)
                 for refiner_image in refiner_images:

modules/prompt_parser_diffusers.py

@@ -61,9 +61,9 @@ def compel_encode_prompts(
         prompt_embeds = torch.cat(prompt_embeds, dim=0)
     if negative_embeds is not None:
         negative_embeds = torch.cat(negative_embeds, dim=0)
-    if positive_pooleds is not None and shared.sd_model_type == "sdxl":
+    if positive_pooleds is not None and 'XL' in pipeline.__class__.__name__:
         positive_pooleds = torch.cat(positive_pooleds, dim=0)
-    if negative_pooleds is not None and shared.sd_model_type == "sdxl":
+    if negative_pooleds is not None and 'XL' in pipeline.__class__.__name__:
         negative_pooleds = torch.cat(negative_pooleds, dim=0)
     return prompt_embeds, positive_pooleds, negative_embeds, negative_pooleds
 
@@ -81,11 +81,11 @@ def compel_encode_prompt(
         shared.log.warning(f"Prompt parser: Compel not supported: {type(pipeline).__name__}")
         return (None, None, None, None)
 
-    if not is_refiner and shared.sd_model_type == "sdxl":
+    if not is_refiner and 'XL' in pipeline.__class__.__name__:
         embedding_type = ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NON_NORMALIZED
         if clip_skip is not None and clip_skip > 1:
             shared.log.warning(f"Prompt parser SDXL unsupported: clip_skip={clip_skip}")
-    elif is_refiner and shared.sd_refiner_type == "sdxl":
+    elif is_refiner and 'XL' in pipeline.__class__.__name__:
         embedding_type = ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NON_NORMALIZED
         if clip_skip is not None and clip_skip > 1:
             shared.log.warning(f"Prompt parser SDXL unsupported: clip_skip={clip_skip}")
@@ -109,7 +109,7 @@ def compel_encode_prompt(
         device=shared.device
     )
 
-    if not is_refiner and shared.sd_model_type == "sdxl":
+    if 'XL' in pipeline.__class__.__name__ and not is_refiner:
         compel_te2 = Compel(tokenizer=pipeline.tokenizer_2, text_encoder=pipeline.text_encoder_2, returned_embeddings_type=embedding_type, requires_pooled=True, device=shared.device)
         positive_te1 = compel_te1(prompt)
         positive_te2, positive_pooled = compel_te2(prompt_2)
@@ -123,7 +123,7 @@ def compel_encode_prompt(
         [prompt_embed, negative_embed] = compel_te2.pad_conditioning_tensors_to_same_length([positive, negative])
         return prompt_embed, positive_pooled, negative_embed, negative_pooled
 
-    if is_refiner and shared.sd_refiner_type == "sdxl":
+    elif 'XL' in pipeline.__class__.__name__ and is_refiner:
         compel_te2 = Compel(tokenizer=pipeline.tokenizer_2, text_encoder=pipeline.text_encoder_2, returned_embeddings_type=embedding_type, requires_pooled=True, device=shared.device)
         positive, positive_pooled = compel_te2(prompt)
         negative, negative_pooled = compel_te2(negative_prompt)
@@ -133,7 +133,7 @@ def compel_encode_prompt(
         [prompt_embed, negative_embed] = compel_te2.pad_conditioning_tensors_to_same_length([positive, negative])
         return prompt_embed, positive_pooled, negative_embed, negative_pooled
 
-    # neither base+sdxl nor refiner+sdxl
-    positive, negative = compel_te1(prompt), compel_te1(negative_prompt)
-    [prompt_embed, negative_embed] = compel_te1.pad_conditioning_tensors_to_same_length([positive, negative])
-    return prompt_embed, None, negative_embed, None
+    else:
+        positive, negative = compel_te1(prompt), compel_te1(negative_prompt)
+        [prompt_embed, negative_embed] = compel_te1.pad_conditioning_tensors_to_same_length([positive, negative])
+        return prompt_embed, None, negative_embed, None

modules/sd_models.py

@@ -518,7 +518,7 @@ class ModelData:
                     elif shared.backend == shared.Backend.DIFFUSERS:
                         load_diffuser(op='model')
                     else:
-                        shared.log.error(f"Unknown Stable Diffusion backend: {shared.backend}")
+                        shared.log.error(f"Unknown Execution backend: {shared.backend}")
                     self.initial = False
                 except Exception as e:
                     shared.log.error("Failed to load stable diffusion model")
@@ -538,7 +538,7 @@ class ModelData:
                     elif shared.backend == shared.Backend.DIFFUSERS:
                         load_diffuser(op='refiner')
                     else:
-                        shared.log.error(f"Unknown Stable Diffusion backend: {shared.backend}")
+                        shared.log.error(f"Unknown Execution backend: {shared.backend}")
                     self.initial = False
                 except Exception as e:
                     shared.log.error("Failed to load stable diffusion model")
@@ -547,7 +547,6 @@ class ModelData:
         return self.sd_refiner
 
     def set_sd_refiner(self, v):
-        shared.log.debug(f"Class refiner: {v}")
         self.sd_refiner = v
 
 model_data = ModelData()
@@ -580,16 +579,13 @@ def detect_pipeline(f: str, op: str = 'model'):
                     shared.log.warning(f'Model detected as SD-XL refiner model, but attempting to load using backend=original: {f} size={size} GB')
                 if op == 'model':
                     shared.log.warning(f'Model detected as SD-XL refiner model, but attempting to load a base model: {f} size={size} GB')
-                else:
-                    guess = 'Stable Diffusion XL'
+                guess = 'Stable Diffusion XL'
             elif size < 7:
                 if shared.backend == shared.Backend.ORIGINAL:
                     shared.log.warning(f'Model detected as SD-XL base model, but attempting to load using backend=original: {f} size={size} GB')
-                if op == 'refiner':
-                    shared.log.warning(f'Model size matches SD-XL base model, but attempting to load a refiner model: {f} size={size} GB')
-                else:
-                    guess = 'Stable Diffusion XL'
+                guess = 'Stable Diffusion XL'
             else:
+                guess = 'Unknown'
                 shared.log.error(f'Model autodetect failed, set diffuser pipeline manually: {f}')
                 return None, None
             shared.log.debug(f'Model autodetect {op}: {f} pipeline={guess} size={size} GB')

modules/shared.py

@@ -348,8 +348,8 @@ elif devices.backend == "rocm":
 else: # cuda
     cross_attention_optimization_default ="Scaled-Dot-Product"
 
-options_templates.update(options_section(('sd', "Stable Diffusion"), {
-    "sd_backend": OptionInfo("diffusers" if cmd_opts.use_openvino else "original", "Stable Diffusion backend", gr.Radio, lambda: {"choices": ["original", "diffusers"] }),
+options_templates.update(options_section(('sd', "Execution & Models"), {
+    "sd_backend": OptionInfo("diffusers" if cmd_opts.use_openvino else "original", "Execution backend", gr.Radio, lambda: {"choices": ["original", "diffusers"] }),
     "sd_checkpoint_autoload": OptionInfo(True, "Model autoload on server start"),
     "sd_model_checkpoint": OptionInfo(default_checkpoint, "Base model", gr.Dropdown, lambda: {"choices": list_checkpoint_tiles()}, refresh=refresh_checkpoints),
     "sd_model_refiner": OptionInfo('None', "Refiner model", gr.Dropdown, lambda: {"choices": ['None'] + list_checkpoint_tiles()}, refresh=refresh_checkpoints),
@@ -520,7 +520,7 @@ options_templates.update(options_section(('ui', "User Interface"), {
     "keyedit_precision_attention": OptionInfo(0.1, "Ctrl+up/down precision when editing (attention:1.1)", gr.Slider, {"minimum": 0.01, "maximum": 0.2, "step": 0.001}),
     "keyedit_precision_extra": OptionInfo(0.05, "Ctrl+up/down precision when editing <extra networks:0.9>", gr.Slider, {"minimum": 0.01, "maximum": 0.2, "step": 0.001}),
     "keyedit_delimiters": OptionInfo(".,\/!?%^*;:{}=`~()", "Ctrl+up/down word delimiters"), # pylint: disable=anomalous-backslash-in-string
-    "quicksettings_list": OptionInfo(["sd_model_checkpoint"], "Quicksettings list", ui_components.DropdownMulti, lambda: {"choices": list(opts.data_labels.keys())}),
+    "quicksettings_list": OptionInfo(["sd_model_checkpoint"] if backend == Backend.ORIGINAL else ["sd_model_checkpoint", "sd_model_refiner"], "Quicksettings list", ui_components.DropdownMulti, lambda: {"choices": list(opts.data_labels.keys())}),
     "ui_scripts_reorder": OptionInfo("", "UI scripts order"),
 }))
 
@@ -574,8 +574,8 @@ options_templates.update(options_section(('sampler-params', "Sampler Settings"),
 options_templates.update(options_section(('postprocessing', "Postprocessing"), {
     'postprocessing_enable_in_main_ui': OptionInfo([], "Enable addtional postprocessing operations", ui_components.DropdownMulti, lambda: {"choices": [x.name for x in shared_items.postprocessing_scripts()]}),
     'postprocessing_operation_order': OptionInfo([], "Postprocessing operation order", ui_components.DropdownMulti, lambda: {"choices": [x.name for x in shared_items.postprocessing_scripts()]}),
-    "use_old_hires_fix_width_height": OptionInfo(False, "Hires fix uses width & height to set final resolution"),
-    "dont_fix_second_order_samplers_schedule": OptionInfo(False, "Do not fix prompt schedule for second order samplers"),
+    # "use_old_hires_fix_width_height": OptionInfo(False, "Hires fix uses width & height to set final resolution"),
+    # "dont_fix_second_order_samplers_schedule": OptionInfo(False, "Do not fix prompt schedule for second order samplers"),
 
     "postprocessing_sep_img2img": OptionInfo("<h2>Img2Img & Inpainting</h2>", "", gr.HTML),
     "img2img_color_correction": OptionInfo(False, "Apply color correction to match original colors"),

modules/ui.py

@@ -378,10 +378,10 @@ def create_ui(startup_timer = None):
 
                 with FormGroup(visible=show_advanced.value, elem_id="txt2img_advanced") as advanced_group:
                     with FormRow():
-                        cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.1, label='CFG Scale', value=6.0, elem_id="txt2img_cfg_scale")
+                        cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.1, label='CFG scale', value=6.0, elem_id="txt2img_cfg_scale")
                         clip_skip = gr.Slider(label='CLIP skip', value=1, minimum=1, maximum=14, step=1, elem_id='txt2img_clip_skip', interactive=True)
                     with FormRow(elem_id="guidence_scale_row", variant="compact"):
-                        image_cfg_scale = gr.Slider(minimum=1.1, maximum=30.0, step=0.1, label='Secondary CFG Scale', value=6.0, elem_id="txt2img_image_cfg_scale")
+                        image_cfg_scale = gr.Slider(minimum=1.1, maximum=30.0, step=0.1, label='Secondary CFG scale', value=6.0, elem_id="txt2img_image_cfg_scale")
                         diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Guidance rescale', value=0.7, elem_id="txt2img_image_cfg_rescale")
                     with FormRow(elem_classes="checkboxes-row", variant="compact"):
                         full_quality = gr.Checkbox(label='Full quality', value=True, elem_id="txt2img_full_quality")
@@ -680,11 +680,11 @@ def create_ui(startup_timer = None):
 
                 with FormGroup(visible=show_advanced.value, elem_id=f"{tab}_advanced_group") as advanced_group:
                     with FormRow():
-                        cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.5, label='CFG Scale', value=6.0, elem_id="img2img_cfg_scale")
-                        image_cfg_scale = gr.Slider(minimum=0, maximum=30.0, step=0.05, label='Image CFG Scale', value=1.5, elem_id="img2img_image_cfg_scale")
+                        cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.5, label='CFG scale', value=6.0, elem_id="img2img_cfg_scale")
+                        image_cfg_scale = gr.Slider(minimum=0, maximum=30.0, step=0.05, label='Image CFG scale', value=1.5, elem_id="img2img_image_cfg_scale")
                     with FormRow():
                         clip_skip = gr.Slider(label='CLIP skip', value=1, minimum=1, maximum=4, step=1, elem_id='img2img_clip_skip', interactive=True)
-                        diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Guidance Rescale', value=0.7, elem_id="txt2img_image_cfg_rescale")
+                        diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Guidance rescale', value=0.7, elem_id="txt2img_image_cfg_rescale")
                     with FormRow(elem_classes="img2img_checkboxes_row", variant="compact"):
                         full_quality = gr.Checkbox(label='Full quality', value=True, elem_id="img2img_full_quality")
                         restore_faces = gr.Checkbox(label='Restore faces', value=False, visible=len(modules.shared.face_restorers) > 1, elem_id="img2img_restore_faces")