From b10cdb9d3e1c3a84ad32d9291b28db9ad08a96ab Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 15 Nov 2023 12:08:51 -0500 Subject: [PATCH] multiple cleanups --- modules/images.py | 4 ++-- modules/processing.py | 17 +++++++------- modules/processing_diffusers.py | 36 +++++++++++++++++++++++++---- modules/ui.py | 8 +++---- requirements.txt | 4 ++-- scripts/xyz_grid.py | 40 ++++++++++++++++----------------- 6 files changed, 68 insertions(+), 41 deletions(-) diff --git a/modules/images.py b/modules/images.py index 1cbc2d622..0d5d74de9 100644 --- a/modules/images.py +++ b/modules/images.py @@ -255,7 +255,7 @@ def resize_image(resize_mode, im, width, height, upscaler_name=None, output_type src_w = width if ratio > src_ratio else im.width * height // im.height src_h = height if ratio <= src_ratio else im.height * width // im.width resized = resize(im, src_w, src_h) - res = Image.new("RGB", (width, height)) + res = Image.new(im.mode, (width, height)) res.paste(resized, box=(width // 2 - src_w // 2, height // 2 - src_h // 2)) else: ratio = width / height @@ -263,7 +263,7 @@ def resize_image(resize_mode, im, width, height, upscaler_name=None, output_type src_w = width if ratio < src_ratio else im.width * height // im.height src_h = height if ratio >= src_ratio else im.height * width // im.width resized = resize(im, src_w, src_h) - res = Image.new("RGB", (width, height)) + res = Image.new(im.mode, (width, height)) res.paste(resized, box=(width // 2 - src_w // 2, height // 2 - src_h // 2)) if ratio < src_ratio: fill_height = height // 2 - src_h // 2 diff --git a/modules/processing.py b/modules/processing.py index a68e113f8..3c3faada5 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -686,6 +686,7 @@ def process_images(p: StableDiffusionProcessing) -> Processed: p.override_settings.pop(k, None) for k in p.override_settings.keys(): stored_opts[k] = shared.opts.data.get(k, None) or shared.opts.data_labels[k].default + res = None try: # if no checkpoint override or the override checkpoint can't be found, remove override entry and load opts checkpoint if p.override_settings.get('sd_model_checkpoint', None) is not None and modules.sd_models.checkpoint_aliases.get(p.override_settings.get('sd_model_checkpoint')) is None: @@ -1210,7 +1211,6 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): def init(self, all_prompts, all_seeds, all_subseeds): if shared.backend == shared.Backend.DIFFUSERS and self.image_mask is not None: shared.sd_model = modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.INPAINTING) - # self.sd_model.dtype = self.sd_model.unet.dtype elif shared.backend == shared.Backend.DIFFUSERS and self.image_mask is None: shared.sd_model = modules.sd_models.set_diffuser_pipe(self.sd_model, modules.sd_models.DiffusersTaskType.IMAGE_2_IMAGE) @@ -1225,6 +1225,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): else: self.ops.append('img2img') crop_region = None + image_mask = self.image_mask if image_mask is not None: if type(image_mask) == list: @@ -1250,6 +1251,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): self.mask_for_overlay = Image.fromarray(np_mask) self.overlay_images = [] latent_mask = self.latent_mask if self.latent_mask is not None else image_mask + add_color_corrections = shared.opts.img2img_color_correction and self.color_corrections is None if add_color_corrections: self.color_corrections = [] @@ -1280,14 +1282,12 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): if crop_region is not None: image = image.crop(crop_region) image = images.resize_image(3, image, self.width, self.height) - if shared.backend == shared.Backend.DIFFUSERS: - unprocessed.append(image) - self.init_images = [image] # assign early for diffusers - if image_mask is not None: - if self.inpainting_fill != 1: - image = modules.masking.fill(image, latent_mask) + if image_mask is not None and self.inpainting_fill != 1: + image = modules.masking.fill(image, latent_mask) if add_color_corrections: self.color_corrections.append(setup_color_correction(image)) + if shared.backend == shared.Backend.DIFFUSERS: + unprocessed.append(image) # assign early for diffusers image = np.array(image).astype(np.float32) / 255.0 image = np.moveaxis(image, 2, 0) imgs.append(image) @@ -1304,8 +1304,7 @@ class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): else: raise RuntimeError(f"bad number of images passed: {len(imgs)}; expecting {self.batch_size} or less") if shared.backend == shared.Backend.DIFFUSERS: - # we've already set self.init_images and self.mask and we dont need any more processing - return + return # we've already set self.init_images and self.mask and we dont need any more processing image = torch.from_numpy(batch_images) image = 2. * image - 1. diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index bf3b6d430..e3f330189 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -31,10 +31,10 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro p.height = tgt_height p.width = tgt_width hypertile_set(p) - if getattr(p, 'mask', None) is not None: - p.mask = images.resize_image(1, p.mask, tgt_width, tgt_height, upscaler_name=None) - if getattr(p, 'mask_for_overlay', None) is not None: - p.mask_for_overlay = images.resize_image(1, p.mask_for_overlay, tgt_width, tgt_height, upscaler_name=None) + if getattr(p, 'mask', None) is not None and p.mask.size != (tgt_width, tgt_height): + p.mask = images.resize_image(1, p.mask, tgt_width, tgt_height, upscaler_name=None) + if getattr(p, 'mask_for_overlay', None) is not None and p.mask_for_overlay.size != (tgt_width, tgt_height): + p.mask_for_overlay = images.resize_image(1, p.mask_for_overlay, tgt_width, tgt_height, upscaler_name=None) def hires_resize(latents): # input=latents output=pil latent_upscaler = shared.latent_upscale_modes.get(p.hr_upscaler, None) @@ -215,7 +215,30 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro p.mask = TF.to_pil_image(torch.ones_like(TF.to_tensor(p.init_images[0]))).convert("L") width = 8 * math.ceil(p.init_images[0].width / 8) height = 8 * math.ceil(p.init_images[0].height / 8) + + # option-1: use images as inputs task_args = {"image": p.init_images, "mask_image": p.mask, "strength": p.denoising_strength, "height": height, "width": width} + + """ # option-2: preprocess images into latents using diffusers + vae_scale_factor = 2 ** (len(model.vae.config.block_out_channels) - 1) + image_processor = diffusers.image_processor.VaeImageProcessor(vae_scale_factor=vae_scale_factor) + mask_processor = diffusers.image_processor.VaeImageProcessor(vae_scale_factor=vae_scale_factor, do_normalize=False, do_binarize=True, do_convert_grayscale=True) + init_image = image_processor.preprocess(p.init_images[0], width=width, height=height) + mask_image = mask_processor.preprocess(p.mask, width=width, height=height) + task_args = {"image": p.init_images, "mask_image": p.mask, "strength": p.denoising_strength, "height": height, "width": width} + """ + + """ # option-2: manually assemble masked image latents + masked_image_latents = [] + mask_image = TF.to_tensor(p.mask) + for init_image in p.init_images: + init_image = TF.to_tensor(p.init_images[0]) + masked_image = init_image * (mask_image > 0.5) + masked_image_latents.append(torch.cat([masked_image, mask_image], dim=0)) + masked_image_latents = torch.stack(masked_image_latents, dim=0).to(shared.device) + task_args = {"image": p.init_images, "mask_image": mask_image, "masked_image_latents": masked_image_latents, "strength": p.denoising_strength, "height": height, "width": width} + """ + if model.__class__.__name__ == 'LatentConsistencyModelPipeline' and hasattr(p, 'init_images') and len(p.init_images) > 0: init_latents = [vae_encode(image, model=shared.sd_model, full_quality=p.full_quality).squeeze(dim=0) for image in p.init_images] init_latent = torch.stack(init_latents, dim=0).to(shared.device) @@ -307,6 +330,8 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro clean['image'] = type(clean['image']) if 'mask_image' in clean: clean['mask_image'] = type(clean['mask_image']) + if 'masked_image_latents' in clean: + clean['masked_image_latents'] = type(clean['masked_image_latents']) if 'prompt' in clean: clean['prompt'] = len(clean['prompt']) if 'negative_prompt' in clean: @@ -408,6 +433,9 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro shared.log.debug(f'Steps: type=refiner input={p.refiner_steps} output={steps} start={p.refiner_start} denoise={p.denoising_strength}') return max(2, int(steps)) + a = 1 + a = (a * 2) + # pipeline type is set earlier in processing, but check for sanity if sd_models.get_diffusers_task(shared.sd_model) != sd_models.DiffusersTaskType.TEXT_2_IMAGE and len(getattr(p, 'init_images' ,[])) == 0: shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) # reset pipeline diff --git a/modules/ui.py b/modules/ui.py index 507e628f6..0dcf9c25f 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -421,7 +421,7 @@ def create_ui(startup_timer = None): cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.1, label='CFG scale', value=6.0, elem_id="txt2img_cfg_scale") clip_skip = gr.Slider(label='CLIP skip', value=1, minimum=1, maximum=14, step=1, elem_id='txt2img_clip_skip', interactive=True) with FormRow(): - image_cfg_scale = gr.Slider(minimum=1.1, maximum=30.0, step=0.1, label='Secondary CFG scale', value=6.0, elem_id="txt2img_image_cfg_scale") + image_cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.1, label='Secondary CFG scale', value=6.0, elem_id="txt2img_image_cfg_scale") diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Guidance rescale', value=0.7, elem_id="txt2img_image_cfg_rescale") with FormRow(): full_quality = gr.Checkbox(label='Full quality', value=True, elem_id="txt2img_full_quality") @@ -711,8 +711,8 @@ def create_ui(startup_timer = None): with gr.Accordion(open=False, label="Advanced", elem_classes=["small-accordion"], elem_id="img2img_advanced_group"): with FormRow(): - cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.5, label='CFG scale', value=6.0, elem_id="img2img_cfg_scale") - image_cfg_scale = gr.Slider(minimum=0, maximum=30.0, step=0.05, label='Image CFG scale', value=1.5, elem_id="img2img_image_cfg_scale") + cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.1, label='CFG scale', value=6.0, elem_id="img2img_cfg_scale") + image_cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.15, label='Image CFG scale', value=1.5, elem_id="img2img_image_cfg_scale") with FormRow(): clip_skip = gr.Slider(label='CLIP skip', value=1, minimum=1, maximum=4, step=1, elem_id='img2img_clip_skip', interactive=True) diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Guidance rescale', value=0.7, elem_id="txt2img_image_cfg_rescale") @@ -729,7 +729,7 @@ def create_ui(startup_timer = None): with gr.Column(): inpainting_mask_invert = gr.Radio(label='Mask mode', choices=['Inpaint masked', 'Inpaint not masked'], value='Inpaint masked', type="index", elem_id="img2img_mask_mode") with gr.Column(): - inpainting_fill = gr.Radio(label='Masked content', choices=['fill', 'original', 'latent noise', 'latent nothing'], value='original', type="index", elem_id="img2img_inpainting_fill") + inpainting_fill = gr.Radio(label='Masked content', choices=['fill', 'original', 'noise', 'nothing'], value='original', type="index", elem_id="img2img_inpainting_fill") with FormRow(): with gr.Column(): inpaint_full_res = gr.Radio(label="Inpaint area", choices=["Whole picture", "Only masked"], type="index", value="Whole picture", elem_id="img2img_inpaint_full_res") diff --git a/requirements.txt b/requirements.txt index b62ebcb89..4207bf910 100644 --- a/requirements.txt +++ b/requirements.txt @@ -53,14 +53,14 @@ opencv-python-headless==4.7.0.72 diffusers==0.23.0 einops==0.4.1 gradio==3.43.2 -huggingface_hub==0.18.0 +huggingface_hub==0.19.2 numexpr==2.8.4 numpy==1.24.4 numba==0.57.1 pandas==1.5.3 protobuf==3.20.3 pytorch_lightning==1.9.4 -transformers==4.34.1 +transformers==4.35.1 tomesd==0.1.3 urllib3==1.26.15 Pillow==9.5.0 diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py index ed3a2f388..ad8cb65af 100644 --- a/scripts/xyz_grid.py +++ b/scripts/xyz_grid.py @@ -236,29 +236,29 @@ axis_options = [ AxisOption("Clip skip", int, apply_clip_skip), AxisOption("Denoising strength", float, apply_field("denoising_strength")), AxisOption("Prompt order", str_permutations, apply_order, fmt=format_value_join_list), + AxisOption("Model dictionary", str, apply_dict, fmt=format_value, cost=1.0, choices=lambda: ['None'] + list(sd_models.checkpoints_list)), + AxisOptionImg2Img("Image mask weight", float, apply_field("inpainting_mask_weight")), AxisOption("[Postprocess] Upscaler", str, apply_upscaler, choices=lambda: [x.name for x in shared.sd_upscalers][1:]), AxisOption("[Postprocess] Face restore", str, apply_face_restore, fmt=format_value), - AxisOptionImg2Img("Image mask weight", float, apply_field("inpainting_mask_weight")), - AxisOption("Model dictionary", str, apply_dict, fmt=format_value, cost=1.0, choices=lambda: ['None'] + list(sd_models.checkpoints_list)), - AxisOption("[Sampler] sigma min", float, apply_field("s_min")), - AxisOption("[Sampler] sigma max", float, apply_field("s_max")), - AxisOption("[Sampler] sigma tmin", float, apply_field("s_tmin")), - AxisOption("[Sampler] sigma tmax", float, apply_field("s_tmax")), - AxisOption("[Sampler] sigma Churn", float, apply_field("s_churn")), - AxisOption("[Sampler] sigma noise", float, apply_field("s_noise")), - AxisOption("[Sampler] eta", float, apply_field("eta")), - AxisOption("[Sampler] solver order", int, apply_setting("schedulers_solver_order")), - AxisOption("[Second pass] upscaler", str, apply_field("hr_upscaler"), choices=lambda: [*shared.latent_upscale_modes, *[x.name for x in shared.sd_upscalers]]), - AxisOption("[Second pass] sampler", str, apply_latent_sampler, fmt=format_value, confirm=confirm_samplers, choices=lambda: [x.name for x in sd_samplers.samplers]), - AxisOption("[Second pass] denoising Strength", float, apply_field("denoising_strength")), - AxisOption("[Second pass] hires steps", int, apply_field("hr_second_pass_steps")), + AxisOption("[Sampler] Sigma min", float, apply_field("s_min")), + AxisOption("[Sampler] Sigma max", float, apply_field("s_max")), + AxisOption("[Sampler] Sigma tmin", float, apply_field("s_tmin")), + AxisOption("[Sampler] Sigma tmax", float, apply_field("s_tmax")), + AxisOption("[Sampler] Sigma Churn", float, apply_field("s_churn")), + AxisOption("[Sampler] Sigma noise", float, apply_field("s_noise")), + AxisOption("[Sampler] ETA", float, apply_field("eta")), + AxisOption("[Sampler] Solver order", int, apply_setting("schedulers_solver_order")), + AxisOption("[Second pass] Upscaler", str, apply_field("hr_upscaler"), choices=lambda: [*shared.latent_upscale_modes, *[x.name for x in shared.sd_upscalers]]), + AxisOption("[Second pass] Sampler", str, apply_latent_sampler, fmt=format_value, confirm=confirm_samplers, choices=lambda: [x.name for x in sd_samplers.samplers]), + AxisOption("[Second pass] Denoising Strength", float, apply_field("denoising_strength")), + AxisOption("[Second pass] Hires steps", int, apply_field("hr_second_pass_steps")), AxisOption("[Second pass] CFG scale", float, apply_field("image_cfg_scale")), - AxisOption("[Second pass] guidance rescale", float, apply_field("diffusers_guidance_rescale")), - AxisOption("[Refiner] model", str, apply_refiner, fmt=format_value, cost=1.0, choices=lambda: ['None'] + sorted(sd_models.checkpoints_list)), - AxisOption("[Refiner] refiner start", float, apply_field("refiner_start")), - AxisOption("[Refiner] refiner steps", float, apply_field("refiner_steps")), - AxisOption("[TOME] Token merging ratio (txt2img)", float, apply_override('token_merging_ratio')), - AxisOption("[TOME] Token merging ratio (hires)", float, apply_override('token_merging_ratio_hr')), + AxisOption("[Second pass] Guidance rescale", float, apply_field("diffusers_guidance_rescale")), + AxisOption("[Refiner] Model", str, apply_refiner, fmt=format_value, cost=1.0, choices=lambda: ['None'] + sorted(sd_models.checkpoints_list)), + AxisOption("[Refiner] Refiner start", float, apply_field("refiner_start")), + AxisOption("[Refiner] Refiner steps", float, apply_field("refiner_steps")), + AxisOption("[ToMe] Token merging ratio (txt2img)", float, apply_override('token_merging_ratio')), + AxisOption("[ToMe] Token merging ratio (hires)", float, apply_override('token_merging_ratio_hr')), AxisOption("[FreeU] 1st stage backbone factor", float, apply_setting('freeu_b1')), AxisOption("[FreeU] 2nd stage backbone factor", float, apply_setting('freeu_b2')), AxisOption("[FreeU] 1st stage skip factor", float, apply_setting('freeu_s1')),