mirror of
https://github.com/vladmandic/automatic
synced 2026-08-27 07:31:01 +02:00
CalamitousFelicitousness
abfb5ac3ed
Vae-class components never take group hooks, so group mode kept them resident on the gpu; a MiniMax-class video vae holds about 10GB that way while running only seconds per generation. Components above 1GB now rest in system memory: the apply_forward_hook bridge on encode and decode fires an on-demand hook that moves the whole module to the device, so tiled calls find every weight already loaded, and the processing seams return it to cpu once outputs are materialized. Small vaes stay resident since the transfer would cost more than it frees. - placement is decided per component by measured size and requires the entry bridge; components without it stay resident - move_model no longer forces on-demand vaes to the gpu for non-txt2img tasks, and full_vae_encode onloads before binding the input, which otherwise lands on the resting device - mode switches clear the stamp and hook in both directions
661 lines
35 KiB
Python
661 lines
35 KiB
Python
from types import SimpleNamespace
|
|
import os
|
|
import time
|
|
import numpy as np
|
|
import torch
|
|
from PIL import Image
|
|
from modules import shared, devices, processing, sd_models, errors, sd_hijack_hypertile, processing_vae, sd_models_compile, timer, modelstats, extra_networks, attention
|
|
from modules.logger import log
|
|
from modules.processing_helpers import resize_hires, calculate_base_steps, calculate_hires_steps, calculate_refiner_steps, save_intermediate, update_sampler, is_txt2img, is_refiner_enabled, get_job_name
|
|
from modules.processing_args import set_pipeline_args
|
|
from modules.onnx_impl import preprocess_pipeline as preprocess_onnx_pipeline, check_parameters_changed as olive_check_parameters_changed
|
|
from modules.lora import lora_common
|
|
from modules.image import convert
|
|
|
|
|
|
debug = os.environ.get('SD_DIFFUSERS_DEBUG', None) is not None
|
|
output_type = 'np' if os.environ.get('SD_VAE_DEFAULT', None) is not None else 'latent'
|
|
last_p = None
|
|
orig_pipeline = shared.sd_model
|
|
|
|
|
|
def restore_state(p: processing.StableDiffusionProcessing):
|
|
if p.state in ['reprocess_refine', 'reprocess_detail']:
|
|
# validate
|
|
if last_p is None:
|
|
log.warning(f'Restore state: op={p.state} last state missing')
|
|
return p
|
|
if p.__class__ != last_p.__class__:
|
|
log.warning(f'Restore state: op={p.state} last state is different type')
|
|
return p
|
|
if shared.history.count == 0:
|
|
log.warning(f'Restore state: op={p.state} last latents missing')
|
|
return p
|
|
state = p.state
|
|
|
|
# set ops
|
|
if state == 'reprocess_refine':
|
|
width, width_before, width_after, width_mask = p.width, p.width_before, p.width_after, p.width_mask
|
|
height, height_before, height_after, height_mask = p.height, p.height_before, p.height_after, p.height_mask
|
|
scale_by, scale_by_before, scale_by_after, scale_by_mask = p.scale_by, p.scale_by_before, p.scale_by_after, p.scale_by_mask
|
|
resize_name, resize_name_before, resize_name_after, resize_name_mask = p.resize_name, p.resize_name_before, p.resize_name_after, p.resize_name_mask
|
|
resize_mode, resize_mode_before, resize_mode_after, resize_mode_mask = p.resize_mode, p.resize_mode_before, p.resize_mode_after, p.resize_mode_mask
|
|
resize_context, resize_context_before, resize_context_after, resize_context_mask = p.resize_context, p.resize_context_before, p.resize_context_after, p.resize_context_mask
|
|
selected_scale_tab, selected_scale_tab_before, selected_scale_tab_after, selected_scale_tab_mask = p.selected_scale_tab, p.selected_scale_tab_before, p.selected_scale_tab_after, p.selected_scale_tab_mask
|
|
hr_scale, hr_resize_mode, hr_resize_context, hr_upscaler, hr_second_pass_steps = p.hr_scale, p.hr_resize_mode, p.hr_resize_context, p.hr_upscaler, p.hr_second_pass_steps
|
|
hr_resize_x, hr_resize_y, hr_upscale_to_x, hr_upscale_to_y, hr_denoising_strength = p.hr_resize_x, p.hr_resize_y, p.hr_upscale_to_x, p.hr_upscale_to_y, p.hr_denoising_strength
|
|
|
|
p = last_p
|
|
p.skip = ['encode', 'base']
|
|
p.state = state
|
|
p.enable_hr = True
|
|
p.hr_force = True
|
|
p.init_images = None
|
|
|
|
p.width, p.width_before, p.width_after, p.width_mask = width, width_before, width_after, width_mask
|
|
p.height, p.height_before, p.height_after, p.height_mask = height, height_before, height_after, height_mask
|
|
p.resize_name, p.resize_name_before, p.resize_name_after, p.resize_name_mask = resize_name, resize_name_before, resize_name_after, resize_name_mask
|
|
p.resize_mode, p.resize_mode_before, p.resize_mode_after, p.resize_mode_mask = resize_mode, resize_mode_before, resize_mode_after, resize_mode_mask
|
|
p.resize_context, p.resize_context_before, p.resize_context_after, p.resize_context_mask = resize_context, resize_context_before, resize_context_after, resize_context_mask
|
|
p.selected_scale_tab, p.selected_scale_tab_before, p.selected_scale_tab_after, p.selected_scale_tab_mask = selected_scale_tab, selected_scale_tab_before, selected_scale_tab_after, selected_scale_tab_mask
|
|
p.scale_by, p.scale_by_before, p.scale_by_after, p.scale_by_mask = scale_by, scale_by_before, scale_by_after, scale_by_mask
|
|
p.hr_scale, p.hr_resize_mode, p.hr_resize_context, p.hr_upscaler, p.hr_second_pass_steps = hr_scale, hr_resize_mode, hr_resize_context, hr_upscaler, hr_second_pass_steps
|
|
p.hr_resize_x, p.hr_resize_y, p.hr_upscale_to_x, p.hr_upscale_to_y, p.hr_denoising_strength = hr_resize_x, hr_resize_y, hr_upscale_to_x, hr_upscale_to_y, hr_denoising_strength
|
|
if state == 'reprocess_detail':
|
|
p.skip = ['encode', 'base', 'hires']
|
|
p.detailer_enabled = True
|
|
log.info(f'Restore state: op={p.state} skip={p.skip}')
|
|
return p
|
|
|
|
|
|
def process_pre(p: processing.StableDiffusionProcessing):
|
|
from modules import ipadapter, sd_hijack_freeu, para_attention, teacache, hidiffusion, ras, pag, cfgzero, transformer_cache, token_merge, linfusion, cachedit
|
|
if shared.sd_model is None:
|
|
log.warning('Processing modifiers: model not loaded')
|
|
return
|
|
log.info('Processing modifiers: apply')
|
|
try:
|
|
# apply-with-unapply
|
|
sd_models_compile.check_deepcache(enable=True)
|
|
ipadapter.apply(shared.sd_model, p)
|
|
token_merge.apply_token_merging(shared.sd_model)
|
|
hidiffusion.apply(p, shared.sd_model_type)
|
|
ras.apply(shared.sd_model, p)
|
|
pag.apply(p)
|
|
cfgzero.apply(p)
|
|
linfusion.apply(shared.sd_model)
|
|
cachedit.apply_cache_dit(shared.sd_model)
|
|
|
|
# apply-only
|
|
sd_hijack_freeu.apply_freeu(p)
|
|
transformer_cache.set_cache()
|
|
para_attention.apply_first_block_cache()
|
|
teacache.apply_teacache(p)
|
|
except Exception as e:
|
|
log.error(f'Processing apply: {e}')
|
|
errors.display(e, 'apply')
|
|
|
|
shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
|
|
# if hasattr(shared.sd_model, 'unet'):
|
|
# sd_models.move_model(shared.sd_model.unet, devices.device)
|
|
# if hasattr(shared.sd_model, 'transformer'):
|
|
# sd_models.move_model(shared.sd_model.transformer, devices.device)
|
|
|
|
from modules import modular
|
|
if modular.is_compatible(shared.sd_model):
|
|
modular_pipe = modular.convert_to_modular(shared.sd_model)
|
|
if modular_pipe is not None:
|
|
shared.sd_model = modular_pipe
|
|
if modular.is_guider(shared.sd_model):
|
|
from modules import modular_guiders
|
|
modular_guiders.set_guider(p)
|
|
|
|
timer.process.record('pre')
|
|
|
|
|
|
def process_post(p: processing.StableDiffusionProcessing):
|
|
from modules import ipadapter, hidiffusion, ras, pag, cfgzero, token_merge, linfusion, cachedit
|
|
log.info('Processing modifiers: unapply')
|
|
|
|
try:
|
|
sd_models_compile.check_deepcache(enable=False)
|
|
ipadapter.unapply(shared.sd_model, unload=getattr(p, 'ip_adapter_unload', False))
|
|
token_merge.remove_token_merging(shared.sd_model)
|
|
hidiffusion.unapply()
|
|
ras.unapply(shared.sd_model)
|
|
pag.unapply()
|
|
cfgzero.unapply()
|
|
linfusion.unapply(shared.sd_model)
|
|
cachedit.unapply_cache_dir(shared.sd_model)
|
|
except Exception as e:
|
|
log.error(f'Processing unapply: {e}')
|
|
errors.display(e, 'unapply')
|
|
timer.process.record('post')
|
|
|
|
|
|
def process_base(p: processing.StableDiffusionProcessing):
|
|
jobid = shared.state.begin('Base')
|
|
txt2img = is_txt2img()
|
|
use_refiner_start = is_refiner_enabled(p) and (not p.is_hr_pass)
|
|
use_denoise_start = not txt2img and p.refiner_start > 0 and p.refiner_start < 1
|
|
|
|
shared.sd_model = update_pipeline(shared.sd_model, p)
|
|
update_sampler(p, shared.sd_model)
|
|
timer.process.record('prepare')
|
|
process_pre(p)
|
|
sched_eta = p.scheduler_eta if p.scheduler_eta is not None else shared.opts.scheduler_eta
|
|
desc = 'Base'
|
|
if 'detailer' in p.ops:
|
|
desc = 'Detail'
|
|
p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts, p.network_data)
|
|
extra_networks.activate_filtered(p) # networks must patch weights before prompt encode so te loras affect embeds
|
|
base_args = set_pipeline_args(
|
|
p=p,
|
|
model=shared.sd_model,
|
|
prompts=p.prompts,
|
|
negative_prompts=p.negative_prompts,
|
|
prompts_2=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else p.prompts,
|
|
negative_prompts_2=[p.refiner_negative] if len(p.refiner_negative) > 0 else p.negative_prompts,
|
|
num_inference_steps=calculate_base_steps(p, use_refiner_start=use_refiner_start, use_denoise_start=use_denoise_start),
|
|
eta=sched_eta,
|
|
guidance_scale=p.cfg_scale if p.cfg_scale is not None and p.cfg_scale > -1 else None,
|
|
guidance_rescale=p.cfg_rescale if p.cfg_rescale is not None and p.cfg_rescale > -1 else None,
|
|
true_cfg_scale=p.cfg_true if p.cfg_true is not None and p.cfg_true > -1 else None,
|
|
denoising_start=0 if use_refiner_start else p.refiner_start if use_denoise_start else None,
|
|
denoising_end=p.refiner_start if use_refiner_start else 1 if use_denoise_start else None,
|
|
num_frames=getattr(p, 'frames', 1),
|
|
output_type=output_type,
|
|
clip_skip=p.clip_skip,
|
|
prompt_attention=getattr(p, 'prompt_attention', None),
|
|
desc=desc,
|
|
)
|
|
base_steps = base_args.get('prior_num_inference_steps', None) or p.steps or base_args.get('num_inference_steps', None)
|
|
shared.state.update(get_job_name(p, shared.sd_model), base_steps, 1)
|
|
if sched_eta is not None and sched_eta > 0 and sched_eta < 1:
|
|
p.extra_generation_params["Sampler Eta"] = sched_eta
|
|
output = None
|
|
if debug:
|
|
modelstats.analyze()
|
|
try:
|
|
t0 = time.time()
|
|
if hasattr(shared.sd_model, 'tgate') and getattr(p, 'gate_step', -1) > 0:
|
|
base_args['gate_step'] = p.gate_step
|
|
output = shared.sd_model.tgate(**base_args) # pylint: disable=not-callable
|
|
else:
|
|
taskid = shared.state.begin('Inference')
|
|
output = shared.sd_model(**base_args)
|
|
shared.state.end(taskid)
|
|
if isinstance(output, dict):
|
|
output = SimpleNamespace(**output)
|
|
if isinstance(output, list):
|
|
output = SimpleNamespace(images=output)
|
|
if isinstance(output, Image.Image):
|
|
output = SimpleNamespace(images=[output])
|
|
if hasattr(output, 'image'):
|
|
output.images = output.image
|
|
if hasattr(output, 'images'):
|
|
shared.history.add(output.images, info=processing.create_infotext(p), ops=p.ops)
|
|
timer.process.record('pipeline')
|
|
sd_models_compile.openvino_post_compile(op="base") # only executes on compiled vino models
|
|
if shared.cmd_opts.profile:
|
|
t1 = time.time()
|
|
log.debug(f'Profile: pipeline call: {t1-t0:.2f}')
|
|
if not hasattr(output, 'images') and hasattr(output, 'frames'):
|
|
if hasattr(output.frames[0], 'shape'):
|
|
log.debug(f'Generated: frames={output.frames[0].shape[1]}')
|
|
else:
|
|
log.debug(f'Generated: frames={len(output.frames[0])}')
|
|
output.images = output.frames[0]
|
|
if hasattr(output, 'images') and isinstance(output.images, np.ndarray):
|
|
output.images = torch.from_numpy(output.images)
|
|
except AssertionError as e:
|
|
log.info(e)
|
|
except ValueError as e:
|
|
shared.state.interrupted = True
|
|
err_args = base_args.copy()
|
|
for k, v in base_args.items():
|
|
if isinstance(v, torch.Tensor):
|
|
err_args[k] = f'{v.device}:{v.dtype}:{v.shape}'
|
|
log.error(f'Processing: args={err_args} {e}')
|
|
errors.display(e, 'Processing')
|
|
except RuntimeError as e:
|
|
shared.state.interrupted = True
|
|
err_args = base_args.copy()
|
|
for k, v in base_args.items():
|
|
if isinstance(v, torch.Tensor):
|
|
err_args[k] = f'{v.device}:{v.dtype}:{v.shape}'
|
|
log.error(f'Processing: step=base args={err_args} {e}')
|
|
errors.display(e, 'Processing')
|
|
modelstats.analyze()
|
|
finally:
|
|
process_post(p)
|
|
|
|
if hasattr(shared.sd_model, 'postprocess') and callable(shared.sd_model.postprocess):
|
|
output = shared.sd_model.postprocess(p, output)
|
|
|
|
shared.state.end(jobid)
|
|
shared.state.nextjob()
|
|
return output
|
|
|
|
|
|
def process_hires(p: processing.StableDiffusionProcessing, output):
|
|
# optional second pass
|
|
if (output is None) or not hasattr(output, 'images') or (output.images is None):
|
|
return output
|
|
if p.enable_hr:
|
|
jobid = shared.state.begin('Hires')
|
|
p.is_hr_pass = True
|
|
if hasattr(p, 'init_hr'):
|
|
p.init_hr(p.hr_scale, p.hr_upscaler, force=p.hr_force)
|
|
else:
|
|
if not p.is_hr_pass: # fake hires for img2img if not actual hr pass
|
|
p.hr_scale = p.scale_by
|
|
p.hr_upscaler = p.resize_name
|
|
p.hr_resize_mode = p.resize_mode
|
|
p.hr_resize_context = p.resize_context
|
|
p.hr_upscale_to_x = int(p.width * p.hr_scale) if p.hr_resize_x == 0 else p.hr_resize_x
|
|
p.hr_upscale_to_y = int(p.height * p.hr_scale) if p.hr_resize_y == 0 else p.hr_resize_y
|
|
|
|
# hires runs on original pipeline
|
|
if hasattr(shared.sd_model, 'restore_pipeline') and (shared.sd_model.restore_pipeline is not None) and (not shared.opts.control_hires):
|
|
shared.sd_model.restore_pipeline()
|
|
if (getattr(shared.sd_model, 'controlnet', None) is not None) and (((isinstance(shared.sd_model.controlnet, list) and len(shared.sd_model.controlnet) > 1)) or ('Multi' in type(shared.sd_model.controlnet).__name__)):
|
|
log.warning(f'Process: control={type(shared.sd_model.controlnet)} not supported in hires')
|
|
return output
|
|
|
|
# upscale
|
|
if hasattr(p, 'height') and hasattr(p, 'width') and p.hr_resize_mode > 0 and (p.hr_upscaler != 'None' or p.hr_resize_mode == 5):
|
|
log.info(f'Upscale: mode={p.hr_resize_mode} upscaler="{p.hr_upscaler}" context="{p.hr_resize_context}" resize={p.hr_resize_x}x{p.hr_resize_y} upscale={p.hr_upscale_to_x}x{p.hr_upscale_to_y}')
|
|
p.ops.append('upscale')
|
|
if shared.opts.samples_save and not p.do_not_save_samples and shared.opts.save_images_before_highres_fix and hasattr(shared.sd_model, 'vae'):
|
|
save_intermediate(p, latents=output.images, suffix="-before-hires")
|
|
output.images = resize_hires(p, latents=output.images)
|
|
sd_hijack_hypertile.hypertile_set(p, hr=True)
|
|
elif torch.is_tensor(output.images) and output.images.shape[-1] == 3: # nhwc
|
|
if output.images.dim() == 3:
|
|
output.images = convert.to_pil(output.images)
|
|
elif output.images.dim() == 4:
|
|
output.images = [convert.to_pil(output.images[i]) for i in range(output.images.shape[0])]
|
|
|
|
strength = p.hr_denoising_strength if p.hr_denoising_strength > 0 else p.denoising_strength
|
|
if (p.hr_upscaler is not None) and (p.hr_upscaler.lower().startswith('latent') or p.hr_force) and strength > 0:
|
|
p.ops.append('hires')
|
|
sd_models_compile.openvino_recompile_model(p, hires=True, refiner=False)
|
|
if shared.sd_model.__class__.__name__ == "OnnxRawPipeline":
|
|
shared.sd_model = preprocess_onnx_pipeline(p)
|
|
p.hr_force = True
|
|
|
|
# hires
|
|
if p.hr_force and strength == 0:
|
|
log.warning('Hires skip: denoising=0')
|
|
p.hr_force = False
|
|
if p.hr_force:
|
|
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE)
|
|
if 'Upscale' in shared.sd_model.__class__.__name__ or 'Flux' in shared.sd_model.__class__.__name__ or 'Kandinsky' in shared.sd_model.__class__.__name__:
|
|
output.images = processing_vae.vae_decode(latents=output.images, model=shared.sd_model, vae_type=p.vae_type, output_type='pil', width=p.width, height=p.height)
|
|
if p.is_control and hasattr(p, 'task_args') and p.task_args.get('image', None) is not None:
|
|
if hasattr(shared.sd_model, "vae") and output.images is not None and len(output.images) > 0:
|
|
output.images = processing_vae.vae_decode(latents=output.images, model=shared.sd_model, vae_type=p.vae_type, output_type='pil', width=p.hr_upscale_to_x, height=p.hr_upscale_to_y) # controlnet cannot deal with latent input
|
|
update_sampler(p, shared.sd_model, second_pass=True)
|
|
orig_denoise = p.denoising_strength
|
|
p.denoising_strength = strength
|
|
orig_image = p.task_args.pop('image', None) # remove image override from hires
|
|
process_pre(p)
|
|
|
|
prompts = p.prompts
|
|
reset_prompts = False
|
|
sched_eta = p.scheduler_eta if p.scheduler_eta is not None else shared.opts.scheduler_eta
|
|
if len(p.refiner_prompt) > 0:
|
|
prompts = len(output.images)* [p.refiner_prompt]
|
|
prompts, p.network_data = extra_networks.parse_prompts(prompts)
|
|
reset_prompts = True
|
|
if reset_prompts or ('base' in p.skip):
|
|
extra_networks.activate_filtered(p)
|
|
|
|
hires_args = set_pipeline_args(
|
|
p=p,
|
|
model=shared.sd_model,
|
|
prompts=prompts,
|
|
negative_prompts=len(output.images) * [p.refiner_negative] if len(p.refiner_negative) > 0 else p.negative_prompts,
|
|
prompts_2=len(output.images) * [p.refiner_prompt] if len(p.refiner_prompt) > 0 else p.prompts,
|
|
negative_prompts_2=len(output.images) * [p.refiner_negative] if len(p.refiner_negative) > 0 else p.negative_prompts,
|
|
num_inference_steps=calculate_hires_steps(p),
|
|
eta=sched_eta,
|
|
guidance_scale=p.cfg_image if p.cfg_image is not None and p.cfg_image > -1 else p.cfg_scale,
|
|
guidance_rescale=p.cfg_rescale if p.cfg_rescale is not None and p.cfg_rescale > -1 else None,
|
|
true_cfg_scale=p.cfg_true if p.cfg_true is not None and p.cfg_true > -1 else None,
|
|
output_type=output_type,
|
|
clip_skip=p.clip_skip,
|
|
image=output.images,
|
|
strength=strength,
|
|
prompt_attention=getattr(p, 'prompt_attention', None),
|
|
desc='Hires',
|
|
)
|
|
|
|
hires_steps = hires_args.get('prior_num_inference_steps', None) or p.hr_second_pass_steps or hires_args.get('num_inference_steps', None)
|
|
shared.state.update(get_job_name(p, shared.sd_model), hires_steps, 1)
|
|
try:
|
|
taskid = shared.state.begin('Inference')
|
|
output = shared.sd_model(**hires_args) # pylint: disable=not-callable
|
|
shared.state.end(taskid)
|
|
if isinstance(output, dict):
|
|
output = SimpleNamespace(**output)
|
|
if hasattr(output, 'images'):
|
|
shared.history.add(output.images, info=processing.create_infotext(p), ops=p.ops)
|
|
sd_models_compile.check_deepcache(enable=False)
|
|
sd_models_compile.openvino_post_compile(op="base")
|
|
except AssertionError as e:
|
|
log.info(e)
|
|
except RuntimeError as e:
|
|
shared.state.interrupted = True
|
|
log.error(f'Processing step=hires: args={hires_args} {e}')
|
|
errors.display(e, 'Processing')
|
|
modelstats.analyze()
|
|
finally:
|
|
process_post(p)
|
|
if hasattr(shared.sd_model, 'postprocess') and callable(shared.sd_model.postprocess):
|
|
output = shared.sd_model.postprocess(p, output)
|
|
if orig_image is not None:
|
|
p.task_args['image'] = orig_image
|
|
p.denoising_strength = orig_denoise
|
|
shared.state.end(jobid)
|
|
shared.state.nextjob()
|
|
p.is_hr_pass = False
|
|
timer.process.record('hires')
|
|
return output
|
|
|
|
|
|
def process_refine(p: processing.StableDiffusionProcessing, output):
|
|
# optional refiner pass or decode
|
|
if (output is None) or not hasattr(output, 'images') or (output.images is None):
|
|
return output
|
|
if is_refiner_enabled(p):
|
|
if shared.opts.samples_save and not p.do_not_save_samples and shared.opts.save_images_before_refiner and hasattr(shared.sd_model, 'vae'):
|
|
save_intermediate(p, latents=output.images, suffix="-before-refiner")
|
|
if shared.opts.diffusers_move_base:
|
|
log.debug('Moving to CPU: model=base')
|
|
sd_models.move_model(shared.sd_model, devices.cpu)
|
|
if shared.state.interrupted or shared.state.skipped:
|
|
shared.sd_model = orig_pipeline
|
|
return output
|
|
jobid = shared.state.begin('Refine')
|
|
shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
|
|
if shared.opts.diffusers_move_refiner:
|
|
sd_models.move_model(shared.sd_refiner, devices.device)
|
|
if hasattr(shared.sd_refiner, 'unet'):
|
|
sd_models.move_model(shared.sd_model.unet, devices.device)
|
|
if hasattr(shared.sd_refiner, 'transformer'):
|
|
sd_models.move_model(shared.sd_model.transformer, devices.device)
|
|
p.ops.append('refine')
|
|
p.is_refiner_pass = True
|
|
|
|
sd_models_compile.openvino_recompile_model(p, hires=False, refiner=True)
|
|
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
|
|
shared.sd_refiner = sd_models.set_diffuser_pipe(shared.sd_refiner, sd_models.DiffusersTaskType.IMAGE_2_IMAGE)
|
|
for i in range(len(output.images)):
|
|
image = output.images[i]
|
|
noise_level = round(350 * p.denoising_strength)
|
|
refiner_output_type = output_type
|
|
if 'Upscale' in shared.sd_refiner.__class__.__name__ or 'Flux' in shared.sd_refiner.__class__.__name__ or 'Kandinsky' in shared.sd_refiner.__class__.__name__:
|
|
image = processing_vae.vae_decode(latents=image, model=shared.sd_model, vae_type=p.vae_type, output_type='pil', width=p.width, height=p.height)
|
|
p.extra_generation_params['Noise level'] = noise_level
|
|
refiner_output_type = 'np'
|
|
update_sampler(p, shared.sd_refiner, second_pass=True)
|
|
sched_eta = p.scheduler_eta if p.scheduler_eta is not None else shared.opts.scheduler_eta
|
|
refiner_args = set_pipeline_args(
|
|
p=p,
|
|
model=shared.sd_refiner,
|
|
prompts=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else p.prompts[i],
|
|
negative_prompts=[p.refiner_negative] if len(p.refiner_negative) > 0 else p.negative_prompts[i],
|
|
num_inference_steps=calculate_refiner_steps(p),
|
|
eta=sched_eta,
|
|
noise_level=noise_level, # StableDiffusionUpscalePipeline only
|
|
guidance_scale=p.cfg_image if p.cfg_image is not None and p.cfg_image > -1 else p.cfg_scale,
|
|
guidance_rescale=p.cfg_rescale if p.cfg_rescale is not None and p.cfg_rescale > -1 else None,
|
|
true_cfg_scale=p.cfg_true if p.cfg_true is not None and p.cfg_true > -1 else None,
|
|
denoising_start=p.refiner_start if p.refiner_start > 0 and p.refiner_start < 1 else None,
|
|
denoising_end=1 if p.refiner_start > 0 and p.refiner_start < 1 else None,
|
|
image=image,
|
|
output_type=refiner_output_type,
|
|
clip_skip=p.clip_skip,
|
|
prompt_attention='fixed',
|
|
desc='Refiner',
|
|
)
|
|
refiner_steps = refiner_args.get('prior_num_inference_steps', None) or p.steps or refiner_args.get('num_inference_steps', None)
|
|
shared.state.update(get_job_name(p, shared.sd_refiner), refiner_steps, 1)
|
|
try:
|
|
if 'requires_aesthetics_score' in shared.sd_refiner.config: # sdxl-model needs false and sdxl-refiner needs true
|
|
shared.sd_refiner.register_to_config(requires_aesthetics_score = getattr(shared.sd_refiner, 'tokenizer', None) is None)
|
|
output = shared.sd_refiner(**refiner_args) # pylint: disable=not-callable
|
|
if isinstance(output, dict):
|
|
output = SimpleNamespace(**output)
|
|
if hasattr(output, 'images'):
|
|
shared.history.add(output.images, info=processing.create_infotext(p), ops=p.ops)
|
|
sd_models_compile.openvino_post_compile(op="refiner")
|
|
except AssertionError as e:
|
|
log.info(e)
|
|
except RuntimeError as e:
|
|
shared.state.interrupted = True
|
|
log.error(f'Processing step=refine: args={refiner_args} {e}')
|
|
errors.display(e, 'Processing')
|
|
modelstats.analyze()
|
|
|
|
if shared.opts.diffusers_offload_mode == "balanced":
|
|
shared.sd_refiner = sd_models.apply_balanced_offload(shared.sd_refiner)
|
|
elif shared.opts.diffusers_move_refiner:
|
|
log.debug('Moving to CPU: model=refiner')
|
|
sd_models.move_model(shared.sd_refiner, devices.cpu)
|
|
shared.state.end(jobid)
|
|
shared.state.nextjob()
|
|
p.is_refiner_pass = False
|
|
timer.process.record('refine')
|
|
return output
|
|
|
|
|
|
class AudioFrameList(list):
|
|
"""list subclass with an audio attribute. Plain lists can't hold attributes,
|
|
so process_decode uses this when the pipeline output includes audio."""
|
|
audio = None
|
|
|
|
|
|
def attach_audio(results, audio):
|
|
if audio is None:
|
|
return results
|
|
wrapped = AudioFrameList(results if isinstance(results, list) else list(results))
|
|
wrapped.audio = audio
|
|
return wrapped
|
|
|
|
|
|
def process_decode(p: processing.StableDiffusionProcessing, output):
|
|
shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae'])
|
|
if output is not None:
|
|
if hasattr(output, 'bytes') and output.bytes is not None:
|
|
log.debug(f'Generated: bytes={len(output.bytes)}')
|
|
return output
|
|
audio = getattr(output, 'audio', None)
|
|
if not hasattr(output, 'images') and hasattr(output, 'frames'):
|
|
log.debug(f'Generated: frames={len(output.frames[0])}')
|
|
output.images = output.frames[0]
|
|
if output.images is not None and len(output.images) > 0 and isinstance(output.images[0], Image.Image):
|
|
sd_models.offload_ondemand(shared.sd_model) # in-pipe decode paths return materialized frames; the vae seam in processing_vae never runs
|
|
return attach_audio(output.images, audio)
|
|
model = shared.sd_model if not is_refiner_enabled(p) else shared.sd_refiner
|
|
if not hasattr(model, 'vae'):
|
|
if hasattr(model, 'pipe') and hasattr(model.pipe, 'vae'):
|
|
model = model.pipe
|
|
if (hasattr(model, "vae") or hasattr(model, "vqgan")) and (output.images is not None) and (len(output.images) > 0):
|
|
if p.hr_resize_mode > 0 and (p.hr_upscaler != 'None' or p.hr_resize_mode == 5):
|
|
width = max(getattr(p, 'width', 0), getattr(p, 'hr_upscale_to_x', 0))
|
|
height = max(getattr(p, 'height', 0), getattr(p, 'hr_upscale_to_y', 0))
|
|
else:
|
|
width = getattr(p, 'width', 0)
|
|
height = getattr(p, 'height', 0)
|
|
frames = p.task_args.get('num_frames', None) or getattr(p, 'frames', None)
|
|
if isinstance(output.images, list):
|
|
results = []
|
|
for i in range(len(output.images)):
|
|
result_batch = processing_vae.vae_decode(
|
|
latents = output.images[i],
|
|
model = model,
|
|
vae_type = p.vae_type,
|
|
width = width,
|
|
height = height,
|
|
frames = frames,
|
|
)
|
|
for result in list(result_batch):
|
|
results.append(result)
|
|
else:
|
|
results = processing_vae.vae_decode(
|
|
latents = output.images,
|
|
model = model,
|
|
vae_type = p.vae_type,
|
|
width = width,
|
|
height = height,
|
|
frames = frames,
|
|
)
|
|
if not isinstance(results, list):
|
|
results = list(results)
|
|
elif hasattr(output, 'images'):
|
|
results = output.images
|
|
else:
|
|
log.warning('Processing: no results')
|
|
results = []
|
|
else:
|
|
log.warning('Processing: no results')
|
|
audio = None
|
|
results = []
|
|
return attach_audio(results, audio)
|
|
|
|
|
|
def update_pipeline(sd_model, p: processing.StableDiffusionProcessing):
|
|
if sd_model is None:
|
|
sd_model = shared.sd_model
|
|
if sd_model is None:
|
|
log.warning('Processing: op=update model not loaded')
|
|
return None
|
|
updated_model = sd_model
|
|
if sd_models.get_diffusers_task(sd_model) == sd_models.DiffusersTaskType.INPAINTING and getattr(p, 'image_mask', None) is None and p.task_args.get('image_mask', None) is None and getattr(p, 'mask', None) is None:
|
|
log.warning('Processing: mode=inpaint mask=None')
|
|
updated_model = sd_models.set_diffuser_pipe(sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE)
|
|
if shared.opts.cuda_compile_backend == "olive-ai":
|
|
updated_model = olive_check_parameters_changed(p, is_refiner_enabled(p))
|
|
if sd_model.__class__.__name__ == "OnnxRawPipeline":
|
|
updated_model = preprocess_onnx_pipeline(p)
|
|
global orig_pipeline # pylint: disable=global-statement
|
|
orig_pipeline = updated_model # processed ONNX pipeline should not be replaced with original pipeline.
|
|
current_attn = getattr(updated_model, "current_attn_name", None)
|
|
if (current_attn != shared.opts.cross_attention_optimization) and (current_attn != shared.opts.sdp_overrides):
|
|
log.info(f"Setting attention optimization: {shared.opts.cross_attention_optimization}")
|
|
attention.set_diffusers_attention(updated_model)
|
|
return updated_model
|
|
|
|
|
|
def validate_pipeline(p: processing.StableDiffusionProcessing):
|
|
from modules.video_models.models_def import models as video_models
|
|
models_cls = []
|
|
for family in video_models:
|
|
for m in video_models[family]:
|
|
if m.repo_cls is not None:
|
|
if isinstance(m.repo_cls, str):
|
|
models_cls.append(m.repo_cls)
|
|
else:
|
|
models_cls.append(m.repo_cls.__name__)
|
|
if m.custom is not None:
|
|
models_cls.append(m.custom)
|
|
is_video_model = shared.sd_model.__class__.__name__ in models_cls
|
|
override_video_pipelines = ['WanPipeline', 'WanImageToVideoPipeline', 'WanVACEPipeline']
|
|
is_video_pipeline = ('video' in p.__class__.__name__.lower()) or (shared.sd_model.__class__.__name__ in override_video_pipelines)
|
|
if is_video_model and not is_video_pipeline:
|
|
log.error(f'Mismatch: type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__} request={p.__class__.__name__} video model with non-video pipeline')
|
|
return False
|
|
elif not is_video_model and is_video_pipeline:
|
|
log.error(f'Mismatch: type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__} request={p.__class__.__name__} non-video model with video pipeline')
|
|
return False
|
|
return True
|
|
|
|
|
|
def process_diffusers(p: processing.StableDiffusionProcessing):
|
|
results = []
|
|
if debug:
|
|
log.trace(f'Process diffusers args: {vars(p)}')
|
|
if not validate_pipeline(p):
|
|
return results
|
|
|
|
p = restore_state(p)
|
|
global orig_pipeline # pylint: disable=global-statement
|
|
orig_pipeline = shared.sd_model
|
|
|
|
if shared.state.interrupted or shared.state.skipped:
|
|
shared.sd_model = orig_pipeline
|
|
return results
|
|
|
|
# sanitize init_images
|
|
if hasattr(p, 'init_images') and not isinstance(getattr(p, 'init_images', []), list):
|
|
p.init_images = [p.init_images]
|
|
if hasattr(p, 'init_images') and isinstance(getattr(p, 'init_images', []), list):
|
|
p.init_images = [i for i in p.init_images if i is not None]
|
|
if len(getattr(p, 'init_images', [])) > 0:
|
|
while len(p.init_images) < len(p.prompts):
|
|
p.init_images.append(p.init_images[-1])
|
|
|
|
# pipeline type is set earlier in processing, but check for sanity
|
|
is_control = getattr(p, 'is_control', False) is True
|
|
has_images = len(getattr(p, 'init_images', [])) > 0
|
|
if (sd_models.get_diffusers_task(shared.sd_model) != sd_models.DiffusersTaskType.TEXT_2_IMAGE) and (not has_images) and (not is_control):
|
|
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) # reset pipeline
|
|
if hasattr(shared.sd_model, 'unet') and hasattr(shared.sd_model.unet, 'config') and hasattr(shared.sd_model.unet.config, 'in_channels') and shared.sd_model.unet.config.in_channels == 9 and not is_control:
|
|
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.INPAINTING) # force pipeline
|
|
if len(getattr(p, 'init_images', [])) == 0:
|
|
p.init_images = [convert.to_pil(torch.rand((3, getattr(p, 'height', 512), getattr(p, 'width', 512))))]
|
|
if not p.prompts:
|
|
p.prompts = p.all_prompts[p.iteration * p.batch_size:(p.iteration+1) * p.batch_size]
|
|
if not p.negative_prompts:
|
|
p.negative_prompts = p.all_negative_prompts[p.iteration * p.batch_size:(p.iteration+1) * p.batch_size]
|
|
|
|
sd_models_compile.openvino_recompile_model(p, hires=False, refiner=False) # recompile if a parameter changes
|
|
|
|
if hasattr(p, 'dummy'):
|
|
images = [Image.new(mode='RGB', size=(p.width, p.height))]
|
|
return images
|
|
if 'base' not in p.skip:
|
|
output = process_base(p)
|
|
else:
|
|
# images, _index = shared.history.selected
|
|
images = shared.history.last_latent
|
|
output = SimpleNamespace(images=images) if images is not None else None
|
|
|
|
if (output is None or (hasattr(output, 'images') and len(output.images) == 0)) and has_images:
|
|
if output is not None:
|
|
log.debug('Processing: using input as base output')
|
|
output.images = p.init_images
|
|
|
|
if shared.state.interrupted or shared.state.skipped:
|
|
shared.sd_model = orig_pipeline
|
|
return results
|
|
|
|
if 'hires' not in p.skip:
|
|
output = process_hires(p, output)
|
|
if shared.state.interrupted or shared.state.skipped:
|
|
shared.sd_model = orig_pipeline
|
|
return results
|
|
|
|
if 'refine' not in p.skip:
|
|
output = process_refine(p, output)
|
|
if shared.state.interrupted or shared.state.skipped:
|
|
shared.sd_model = orig_pipeline
|
|
return results
|
|
|
|
extra_networks.deactivate(p)
|
|
timer.process.add('lora', lora_common.timer.total)
|
|
lora_common.timer.clear(complete=True)
|
|
|
|
results = process_decode(p, output)
|
|
timer.process.record('decode')
|
|
|
|
shared.sd_model = orig_pipeline
|
|
|
|
if p.state == '':
|
|
global last_p # pylint: disable=global-statement
|
|
last_p = p
|
|
return results
|