diff --git a/CHANGELOG.md b/CHANGELOG.md index 82aa9c037..bc1fa1e76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,11 +8,13 @@ - based on [VGen](https://huggingface.co/ali-vilab/i2vgen-xl) - **Improvements** - **FaceID** extend support for LoRA, HyperTile and FreeU, thanks @Trojaner + - new setting in image options *include mask in output* - default theme updates and additional built-in theme *black-gray* - **Internal** - remove obsolete textual inversion training code - remove obsolete hypernetworks training code - **Fixes** + - improve model offload compatibility - fix extra networks refresh - improve ZLUDA installer when using `--use-zluda` cli param, thanks @lshqqytiger diff --git a/TODO.md b/TODO.md index 03af7e25b..69c6c1311 100644 --- a/TODO.md +++ b/TODO.md @@ -7,11 +7,10 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - defork - stable cascade: - ipadapter masking: -- init latents: variations, tiling, img2img - x-adapter: -- diffusers public callbacks -- video2video - async lowvram: +- init latents: variations, tiling, img2img +- diffusers public callbacks - remove builtin: controlnet - remove builtin: image-browser diff --git a/modules/control/run.py b/modules/control/run.py index 7f359d240..94e38a8fa 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -482,6 +482,8 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ debug(f'Control exec pipeline: task={sd_models.get_diffusers_task(pipe)} class={pipe.__class__}') debug(f'Control exec pipeline: p={vars(p)}') debug(f'Control exec pipeline: args={p.task_args} image={p.task_args.get("image", None)} control={p.task_args.get("control_image", None)} mask={p.task_args.get("mask_image", None) or p.image_mask} ref={p.task_args.get("ref_image", None)}') + if sd_models.get_diffusers_task(pipe) != sd_models.DiffusersTaskType.TEXT_2_IMAGE: # force vae back to gpu if not in txt2img mode + sd_models.move_model(pipe.vae, devices.device) p.scripts = scripts.scripts_control p.script_args = input_script_args processed = p.scripts.run(p, *input_script_args) @@ -508,6 +510,10 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_ output_image = images.resize_image(resize_mode_after, output_image, width_after, height_after, resize_name_after) output_images.append(output_image) + if shared.opts.include_mask: + if processed_image is not None and isinstance(processed_image, Image.Image): + output_images.append(processed_image) + if is_generator: image_txt = f'{output_image.width}x{output_image.height}' if output_image is not None else 'None' if video is not None: diff --git a/modules/processing.py b/modules/processing.py index b11cbe60a..521d4502b 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -396,6 +396,10 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: if not p.disable_extra_networks: extra_networks.deactivate(p, extra_network_data) + if shared.opts.include_mask: + if getattr(p, 'image_mask', None) is not None and isinstance(p.image_mask, Image.Image): + output_images.append(p.image_mask) + processed = Processed( p, images_list=output_images, diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index e04513398..254f94d72 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -345,7 +345,6 @@ def process_diffusers(p: processing.StableDiffusionProcessing): p.task_args['sag_scale'] = p.sag_scale else: shared.log.warning(f'SAG incompatible scheduler: current={sd_model.scheduler.__class__.__name__} supported={supported}') - if shared.opts.cuda_compile_backend == "olive-ai": sd_model = olive_check_parameters_changed(p, is_refiner_enabled()) if sd_model.__class__.__name__ == "OnnxRawPipeline": @@ -362,12 +361,6 @@ def process_diffusers(p: processing.StableDiffusionProcessing): shared.sd_model = orig_pipeline return results - if shared.opts.diffusers_move_base: - sd_models.move_model(shared.sd_model, devices.device) - - # recompile if a parameter changes - sd_models_compile.openvino_recompile_model(p, hires=False, refiner=False) - # pipeline type is set earlier in processing, but check for sanity is_control = getattr(p, 'is_control', False) is True has_images = len(getattr(p, 'init_images' ,[])) > 0 @@ -378,6 +371,9 @@ def process_diffusers(p: processing.StableDiffusionProcessing): if len(getattr(p, 'init_images' ,[])) == 0: p.init_images = [TF.to_pil_image(torch.rand((3, getattr(p, 'height', 512), getattr(p, 'width', 512))))] + sd_models.move_model(shared.sd_model, devices.device) + sd_models_compile.openvino_recompile_model(p, hires=False, refiner=False) # recompile if a parameter changes + use_refiner_start = is_txt2img() and is_refiner_enabled() and not p.is_hr_pass and p.refiner_start > 0 and p.refiner_start < 1 use_denoise_start = not is_txt2img() and p.refiner_start > 0 and p.refiner_start < 1 diff --git a/modules/sd_models.py b/modules/sd_models.py index d2c5aaaf0..2b80c0fdf 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -742,6 +742,12 @@ def set_diffuser_options(sd_model, vae = None, op: str = 'model'): def move_model(model, device=None, force=False): if model is not None: + if getattr(model, 'vae', None) is not None and get_diffusers_task(model) != DiffusersTaskType.TEXT_2_IMAGE: + if device == devices.device: # force vae back to gpu if not in txt2img mode + model.vae.to(device) + if hasattr(model.vae, '_hf_hook'): + debug_move(f'Model move: to={device} class={model.vae.__class__} function={sys._getframe(1).f_code.co_name}') # pylint: disable=protected-access + model.vae._hf_hook.execution_device = device # pylint: disable=protected-access if getattr(model, 'has_accelerate', False) and not force: return debug_move(f'Model move: to={device} class={model.__class__} function={sys._getframe(1).f_code.co_name}') # pylint: disable=protected-access diff --git a/modules/shared.py b/modules/shared.py index f07c079b5..6a1e4039c 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -507,6 +507,7 @@ options_templates.update(options_section(('saving-images', "Image Options"), { "img_max_size_mp": OptionInfo(250, "Maximum image size (MP)", gr.Slider, {"minimum": 100, "maximum": 2000, "step": 1}), "webp_lossless": OptionInfo(False, "WebP lossless compression"), "save_selected_only": OptionInfo(True, "Save only saves selected image"), + "include_mask": OptionInfo(False, "Include mask in outputs"), "samples_save_zip": OptionInfo(True, "Create ZIP archive"), "image_sep_metadata": OptionInfo("

Metadata/Logging

", "", gr.HTML), diff --git a/scripts/differential_diffusion.py b/scripts/differential_diffusion.py index e876f05ed..f0cfbf2eb 100644 --- a/scripts/differential_diffusion.py +++ b/scripts/differential_diffusion.py @@ -1518,7 +1518,7 @@ class StableDiffusionDiffImg2ImgPipeline(DiffusionPipeline): negative_prompt=None, prompt_embeds: Optional[torch.FloatTensor] = None, negative_prompt_embeds: Optional[torch.FloatTensor] = None, - lora_scale: Optional[float] = None, + lora_scale: Optional[float] = None, # pylint: disable=unused-argument clip_skip: Optional[int] = None, ): if prompt is not None and isinstance(prompt, str): @@ -1892,11 +1892,11 @@ class Script(scripts.Script): image = gr.Image(label="Image map", show_label=False, type="pil", source="upload", interactive=True, tool="editor", visible=True, image_mode='RGB') return enabled, strength, invert, model, image - def depthmap(self, image_init: Image.Image, image_map: Image.Image, model: str, strength: float, invert: bool, output_type="tensor"): + def depthmap(self, image_init: Image.Image, image_map: Image.Image, model: str, strength: float, invert: bool): global detector # pylint: disable=global-statement from modules.control.proc.dpt import DPTDetector if image_init is None: - return None, None + return None, None, None image_map = None if image_map is not None: image_map = image_map.resize(image_init.size, Image.Resampling.LANCZOS) @@ -1916,14 +1916,14 @@ class Script(scripts.Script): init_img_hash = hashlib.sha256(image_map.tobytes()).hexdigest()[0:8] # pylint: disable=attribute-defined-outside-init images.save_image(image_map, path=shared.opts.outdir_init_images, basename=None, forced_filename=init_img_hash, suffix="-init-image") else: - return None, None - if output_type == "tensor": - image_map = transforms.ToTensor()(image_map) - image_map = image_map.to(devices.device) - image_init = 2 * transforms.ToTensor()(image_init) - 1 - image_init = image_init.unsqueeze(0) - image_init = image_init.to(devices.device) - return image_init, image_map + return None, None, None + image_mask = image_map.copy() + image_map = transforms.ToTensor()(image_map) + image_map = image_map.to(devices.device) + image_init = 2 * transforms.ToTensor()(image_init) - 1 + image_init = image_init.unsqueeze(0) + image_init = image_init.to(devices.device) + return image_init, image_map, image_mask def run(self, p: processing.StableDiffusionProcessingImg2Img, enabled, strength, invert, model, image): # pylint: disable=arguments-differ if not enabled: @@ -1935,7 +1935,7 @@ class Script(scripts.Script): shared.log.error('Differential-diffusion: no input images') return - image_init, image_map = self.depthmap(p.init_images[0], image, model, strength, invert, output_type="tensor") + image_init, image_map, image_mask = self.depthmap(p.init_images[0], image, model, strength, invert) if image_map is None: shared.log.error('Differential-diffusion: no image map') return @@ -1974,6 +1974,7 @@ class Script(scripts.Script): p.task_args['original_image'] = image_init shared.log.debug(f'Differential-diffusion: pipeline={pipe.__class__.__name__} strength={strength} model={model} auto={image is None}') shared.sd_model = pipe + sd_models.move_model(pipe.vae, devices.device, force=True) except Exception as e: shared.log.error(f'Differential-diffusion: pipeline creation failed: {e}') errors.display(e, 'Differential-diffusion: pipeline creation failed') @@ -1981,6 +1982,9 @@ class Script(scripts.Script): # run pipeline processed: processing.Processed = processing.process_images(p) # runs processing using main loop + if shared.opts.include_mask: + if image_mask is not None and isinstance(image_mask, Image.Image): + processed.images.append(image_mask) # restore pipeline and params pipe = None