Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-12 07:58:43 +02:00
Code Issues Packages Projects Releases Wiki Activity

add include mask in output, improve model offload compatibility

Browse Source
This commit is contained in:
Vladimir Mandic
2024-02-24 11:04:01 -05:00
parent 03ec491a69
commit 17f2f8d98f
8 changed files with 40 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+2
View File
@@ -8,11 +8,13 @@
- based on [VGen](https://huggingface.co/ali-vilab/i2vgen-xl)
- **Improvements**
- **FaceID** extend support for LoRA, HyperTile and FreeU, thanks @Trojaner
- new setting in image options *include mask in output*
- default theme updates and additional built-in theme *black-gray*
- **Internal**
- remove obsolete textual inversion training code
- remove obsolete hypernetworks training code
- **Fixes**
- improve model offload compatibility
- fix extra networks refresh
- improve ZLUDA installer when using `--use-zluda` cli param, thanks @lshqqytiger
TODO.md
+2 -3
View File
@@ -7,11 +7,10 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma
- defork
- stable cascade: <https://github.com/vladmandic/automatic/wiki/Stable-Cascade>
- ipadapter masking: <https://github.com/huggingface/diffusers/pull/6847>
- init latents: variations, tiling, img2img
- x-adapter: <https://github.com/showlab/X-Adapter>
- diffusers public callbacks
- video2video
- async lowvram: <https://github.com/AUTOMATIC1111/stable-diffusion-webui/pull/14855>
- init latents: variations, tiling, img2img
- diffusers public callbacks
- remove builtin: controlnet
- remove builtin: image-browser
modules/control/run.py
+6
View File
@@ -482,6 +482,8 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
debug(f'Control exec pipeline: task={sd_models.get_diffusers_task(pipe)} class={pipe.__class__}')
debug(f'Control exec pipeline: p={vars(p)}')
debug(f'Control exec pipeline: args={p.task_args} image={p.task_args.get("image", None)} control={p.task_args.get("control_image", None)} mask={p.task_args.get("mask_image", None) or p.image_mask} ref={p.task_args.get("ref_image", None)}')
if sd_models.get_diffusers_task(pipe) != sd_models.DiffusersTaskType.TEXT_2_IMAGE: # force vae back to gpu if not in txt2img mode
sd_models.move_model(pipe.vae, devices.device)
p.scripts = scripts.scripts_control
p.script_args = input_script_args
processed = p.scripts.run(p, *input_script_args)
@@ -508,6 +510,10 @@ def control_run(units: List[unit.Unit], inputs, inits, mask, unit_type: str, is_
output_image = images.resize_image(resize_mode_after, output_image, width_after, height_after, resize_name_after)
output_images.append(output_image)
if shared.opts.include_mask:
if processed_image is not None and isinstance(processed_image, Image.Image):
output_images.append(processed_image)
if is_generator:
image_txt = f'{output_image.width}x{output_image.height}' if output_image is not None else 'None'
if video is not None:
modules/processing.py
+4
View File
@@ -396,6 +396,10 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
if not p.disable_extra_networks:
extra_networks.deactivate(p, extra_network_data)
if shared.opts.include_mask:
if getattr(p, 'image_mask', None) is not None and isinstance(p.image_mask, Image.Image):
output_images.append(p.image_mask)
processed = Processed(
p,
images_list=output_images,
modules/processing_diffusers.py
+3 -7
View File
@@ -345,7 +345,6 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
p.task_args['sag_scale'] = p.sag_scale
else:
shared.log.warning(f'SAG incompatible scheduler: current={sd_model.scheduler.__class__.__name__} supported={supported}')
if shared.opts.cuda_compile_backend == "olive-ai":
sd_model = olive_check_parameters_changed(p, is_refiner_enabled())
if sd_model.__class__.__name__ == "OnnxRawPipeline":
@@ -362,12 +361,6 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
shared.sd_model = orig_pipeline
return results
if shared.opts.diffusers_move_base:
sd_models.move_model(shared.sd_model, devices.device)
# recompile if a parameter changes
sd_models_compile.openvino_recompile_model(p, hires=False, refiner=False)
# pipeline type is set earlier in processing, but check for sanity
is_control = getattr(p, 'is_control', False) is True
has_images = len(getattr(p, 'init_images' ,[])) > 0
@@ -378,6 +371,9 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
if len(getattr(p, 'init_images' ,[])) == 0:
p.init_images = [TF.to_pil_image(torch.rand((3, getattr(p, 'height', 512), getattr(p, 'width', 512))))]
sd_models.move_model(shared.sd_model, devices.device)
sd_models_compile.openvino_recompile_model(p, hires=False, refiner=False) # recompile if a parameter changes
use_refiner_start = is_txt2img() and is_refiner_enabled() and not p.is_hr_pass and p.refiner_start > 0 and p.refiner_start < 1
use_denoise_start = not is_txt2img() and p.refiner_start > 0 and p.refiner_start < 1
modules/sd_models.py
+6
View File
@@ -742,6 +742,12 @@ def set_diffuser_options(sd_model, vae = None, op: str = 'model'):
def move_model(model, device=None, force=False):
if model is not None:
if getattr(model, 'vae', None) is not None and get_diffusers_task(model) != DiffusersTaskType.TEXT_2_IMAGE:
if device == devices.device: # force vae back to gpu if not in txt2img mode
model.vae.to(device)
if hasattr(model.vae, '_hf_hook'):
debug_move(f'Model move: to={device} class={model.vae.__class__} function={sys._getframe(1).f_code.co_name}') # pylint: disable=protected-access
model.vae._hf_hook.execution_device = device # pylint: disable=protected-access
if getattr(model, 'has_accelerate', False) and not force:
return
debug_move(f'Model move: to={device} class={model.__class__} function={sys._getframe(1).f_code.co_name}') # pylint: disable=protected-access
modules/shared.py
+1
View File
@@ -507,6 +507,7 @@ options_templates.update(options_section(('saving-images', "Image Options"), {
"img_max_size_mp": OptionInfo(250, "Maximum image size (MP)", gr.Slider, {"minimum": 100, "maximum": 2000, "step": 1}),
"webp_lossless": OptionInfo(False, "WebP lossless compression"),
"save_selected_only": OptionInfo(True, "Save only saves selected image"),
"include_mask": OptionInfo(False, "Include mask in outputs"),
"samples_save_zip": OptionInfo(True, "Create ZIP archive"),
"image_sep_metadata": OptionInfo("<h2>Metadata/Logging</h2>", "", gr.HTML),
scripts/differential_diffusion.py
+16 -12
View File
@@ -1518,7 +1518,7 @@ class StableDiffusionDiffImg2ImgPipeline(DiffusionPipeline):
negative_prompt=None,
prompt_embeds: Optional[torch.FloatTensor] = None,
negative_prompt_embeds: Optional[torch.FloatTensor] = None,
lora_scale: Optional[float] = None,
lora_scale: Optional[float] = None, # pylint: disable=unused-argument
clip_skip: Optional[int] = None,
):
if prompt is not None and isinstance(prompt, str):
@@ -1892,11 +1892,11 @@ class Script(scripts.Script):
image = gr.Image(label="Image map", show_label=False, type="pil", source="upload", interactive=True, tool="editor", visible=True, image_mode='RGB')
return enabled, strength, invert, model, image
def depthmap(self, image_init: Image.Image, image_map: Image.Image, model: str, strength: float, invert: bool, output_type="tensor"):
def depthmap(self, image_init: Image.Image, image_map: Image.Image, model: str, strength: float, invert: bool):
global detector # pylint: disable=global-statement
from modules.control.proc.dpt import DPTDetector
if image_init is None:
return None, None
return None, None, None
image_map = None
if image_map is not None:
image_map = image_map.resize(image_init.size, Image.Resampling.LANCZOS)
@@ -1916,14 +1916,14 @@ class Script(scripts.Script):
init_img_hash = hashlib.sha256(image_map.tobytes()).hexdigest()[0:8] # pylint: disable=attribute-defined-outside-init
images.save_image(image_map, path=shared.opts.outdir_init_images, basename=None, forced_filename=init_img_hash, suffix="-init-image")
else:
return None, None
if output_type == "tensor":
image_map = transforms.ToTensor()(image_map)
image_map = image_map.to(devices.device)
image_init = 2 * transforms.ToTensor()(image_init) - 1
image_init = image_init.unsqueeze(0)
image_init = image_init.to(devices.device)
return image_init, image_map
return None, None, None
image_mask = image_map.copy()
image_map = transforms.ToTensor()(image_map)
image_map = image_map.to(devices.device)
image_init = 2 * transforms.ToTensor()(image_init) - 1
image_init = image_init.unsqueeze(0)
image_init = image_init.to(devices.device)
return image_init, image_map, image_mask
def run(self, p: processing.StableDiffusionProcessingImg2Img, enabled, strength, invert, model, image): # pylint: disable=arguments-differ
if not enabled:
@@ -1935,7 +1935,7 @@ class Script(scripts.Script):
shared.log.error('Differential-diffusion: no input images')
return
image_init, image_map = self.depthmap(p.init_images[0], image, model, strength, invert, output_type="tensor")
image_init, image_map, image_mask = self.depthmap(p.init_images[0], image, model, strength, invert)
if image_map is None:
shared.log.error('Differential-diffusion: no image map')
return
@@ -1974,6 +1974,7 @@ class Script(scripts.Script):
p.task_args['original_image'] = image_init
shared.log.debug(f'Differential-diffusion: pipeline={pipe.__class__.__name__} strength={strength} model={model} auto={image is None}')
shared.sd_model = pipe
sd_models.move_model(pipe.vae, devices.device, force=True)
except Exception as e:
shared.log.error(f'Differential-diffusion: pipeline creation failed: {e}')
errors.display(e, 'Differential-diffusion: pipeline creation failed')
@@ -1981,6 +1982,9 @@ class Script(scripts.Script):
# run pipeline
processed: processing.Processed = processing.process_images(p) # runs processing using main loop
if shared.opts.include_mask:
if image_mask is not None and isinstance(image_mask, Image.Image):
processed.images.append(image_mask)
# restore pipeline and params
pipe = None