mirror of
https://github.com/vladmandic/automatic
synced 2026-08-25 22:20:46 +02:00
CalamitousFelicitousness
7792b1ab62
The shape branch read axis 1, which held the frame count back when animatediff returned (batch, channels, frames, height, width). Diffusers now returns (batch, frames, channels, height, width) and the modular video path puts channels last, so axis 1 reads channels or height and the line printed a plausible wrong number: a 124 frame 1024x576 generation logged frames=576. len() is the frame count under both layouts and is what the sibling call site in process_decode already uses.
659 lines
35 KiB
Python
659 lines
35 KiB
Python
from types import SimpleNamespace
|
|
import os
|
|
import time
|
|
import numpy as np
|
|
import torch
|
|
from PIL import Image
|
|
from modules import shared, processing, sd_models, errors, sd_hijack_hypertile, processing_vae, sd_models_compile, timer, modelstats, extra_networks, attention
|
|
from modules.logger import log
|
|
from modules.processing_helpers import resize_hires, calculate_base_steps, calculate_hires_steps, calculate_refiner_steps, save_intermediate, update_sampler, is_txt2img, is_refiner_enabled, get_job_name
|
|
from modules.processing_args import set_pipeline_args
|
|
from modules.onnx_impl import preprocess_pipeline as preprocess_onnx_pipeline, check_parameters_changed as olive_check_parameters_changed
|
|
from modules.lora import lora_common
|
|
from modules.image import convert
|
|
|
|
|
|
debug = os.environ.get('SD_DIFFUSERS_DEBUG', None) is not None
|
|
output_type = 'np' if os.environ.get('SD_VAE_DEFAULT', None) is not None else 'latent'
|
|
last_p = None
|
|
orig_pipeline = shared.sd_model
|
|
|
|
|
|
def restore_state(p: processing.StableDiffusionProcessing):
|
|
if p.state in ['reprocess_refine', 'reprocess_detail']:
|
|
# validate
|
|
if last_p is None:
|
|
log.warning(f'Restore state: op={p.state} last state missing')
|
|
return p
|
|
if p.__class__ != last_p.__class__:
|
|
log.warning(f'Restore state: op={p.state} last state is different type')
|
|
return p
|
|
if shared.history.count == 0:
|
|
log.warning(f'Restore state: op={p.state} last latents missing')
|
|
return p
|
|
state = p.state
|
|
|
|
# set ops
|
|
if state == 'reprocess_refine':
|
|
width, width_before, width_after, width_mask = p.width, p.width_before, p.width_after, p.width_mask
|
|
height, height_before, height_after, height_mask = p.height, p.height_before, p.height_after, p.height_mask
|
|
scale_by, scale_by_before, scale_by_after, scale_by_mask = p.scale_by, p.scale_by_before, p.scale_by_after, p.scale_by_mask
|
|
resize_name, resize_name_before, resize_name_after, resize_name_mask = p.resize_name, p.resize_name_before, p.resize_name_after, p.resize_name_mask
|
|
resize_mode, resize_mode_before, resize_mode_after, resize_mode_mask = p.resize_mode, p.resize_mode_before, p.resize_mode_after, p.resize_mode_mask
|
|
resize_context, resize_context_before, resize_context_after, resize_context_mask = p.resize_context, p.resize_context_before, p.resize_context_after, p.resize_context_mask
|
|
selected_scale_tab, selected_scale_tab_before, selected_scale_tab_after, selected_scale_tab_mask = p.selected_scale_tab, p.selected_scale_tab_before, p.selected_scale_tab_after, p.selected_scale_tab_mask
|
|
hr_scale, hr_resize_mode, hr_resize_context, hr_upscaler, hr_second_pass_steps = p.hr_scale, p.hr_resize_mode, p.hr_resize_context, p.hr_upscaler, p.hr_second_pass_steps
|
|
hr_resize_x, hr_resize_y, hr_upscale_to_x, hr_upscale_to_y, hr_denoising_strength = p.hr_resize_x, p.hr_resize_y, p.hr_upscale_to_x, p.hr_upscale_to_y, p.hr_denoising_strength
|
|
|
|
p = last_p
|
|
p.skip = ['encode', 'base']
|
|
p.state = state
|
|
p.enable_hr = True
|
|
p.hr_force = True
|
|
p.init_images = None
|
|
|
|
p.width, p.width_before, p.width_after, p.width_mask = width, width_before, width_after, width_mask
|
|
p.height, p.height_before, p.height_after, p.height_mask = height, height_before, height_after, height_mask
|
|
p.resize_name, p.resize_name_before, p.resize_name_after, p.resize_name_mask = resize_name, resize_name_before, resize_name_after, resize_name_mask
|
|
p.resize_mode, p.resize_mode_before, p.resize_mode_after, p.resize_mode_mask = resize_mode, resize_mode_before, resize_mode_after, resize_mode_mask
|
|
p.resize_context, p.resize_context_before, p.resize_context_after, p.resize_context_mask = resize_context, resize_context_before, resize_context_after, resize_context_mask
|
|
p.selected_scale_tab, p.selected_scale_tab_before, p.selected_scale_tab_after, p.selected_scale_tab_mask = selected_scale_tab, selected_scale_tab_before, selected_scale_tab_after, selected_scale_tab_mask
|
|
p.scale_by, p.scale_by_before, p.scale_by_after, p.scale_by_mask = scale_by, scale_by_before, scale_by_after, scale_by_mask
|
|
p.hr_scale, p.hr_resize_mode, p.hr_resize_context, p.hr_upscaler, p.hr_second_pass_steps = hr_scale, hr_resize_mode, hr_resize_context, hr_upscaler, hr_second_pass_steps
|
|
p.hr_resize_x, p.hr_resize_y, p.hr_upscale_to_x, p.hr_upscale_to_y, p.hr_denoising_strength = hr_resize_x, hr_resize_y, hr_upscale_to_x, hr_upscale_to_y, hr_denoising_strength
|
|
if state == 'reprocess_detail':
|
|
p.skip = ['encode', 'base', 'hires']
|
|
p.detailer_enabled = True
|
|
log.info(f'Restore state: op={p.state} skip={p.skip}')
|
|
return p
|
|
|
|
|
|
def process_pre(p: processing.StableDiffusionProcessing):
|
|
from modules import ipadapter, sd_hijack_freeu, para_attention, teacache, hidiffusion, ras, pag, cfgzero, transformer_cache, token_merge, linfusion, cachedit
|
|
if shared.sd_model is None:
|
|
log.warning('Processing modifiers: model not loaded')
|
|
return
|
|
log.info('Processing modifiers: apply')
|
|
try:
|
|
# apply-with-unapply
|
|
# sd_hijack_compile.install()
|
|
sd_models_compile.check_deepcache(enable=True)
|
|
ipadapter.apply(shared.sd_model, p)
|
|
token_merge.apply_token_merging(shared.sd_model)
|
|
hidiffusion.apply(p, shared.sd_model_type)
|
|
ras.apply(shared.sd_model, p)
|
|
pag.apply(p)
|
|
cfgzero.apply(p)
|
|
linfusion.apply(shared.sd_model)
|
|
cachedit.apply_cache_dit(shared.sd_model)
|
|
|
|
# apply-only
|
|
sd_hijack_freeu.apply_freeu(p)
|
|
transformer_cache.set_cache()
|
|
para_attention.apply_first_block_cache()
|
|
teacache.apply_teacache(p)
|
|
except Exception as e:
|
|
log.error(f'Processing apply: {e}')
|
|
errors.display(e, 'apply')
|
|
|
|
shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
|
|
# if hasattr(shared.sd_model, 'unet'):
|
|
# sd_models.move_model(shared.sd_model.unet, devices.device)
|
|
# if hasattr(shared.sd_model, 'transformer'):
|
|
# sd_models.move_model(shared.sd_model.transformer, devices.device)
|
|
|
|
from modules import modular
|
|
if modular.is_compatible(shared.sd_model):
|
|
modular_pipe = modular.convert_to_modular(shared.sd_model)
|
|
if modular_pipe is not None:
|
|
shared.sd_model = modular_pipe
|
|
if modular.is_guider(shared.sd_model):
|
|
from modules import modular_guiders
|
|
modular_guiders.set_guider(p)
|
|
|
|
timer.process.record('pre')
|
|
|
|
|
|
def process_post(p: processing.StableDiffusionProcessing):
|
|
from modules import ipadapter, hidiffusion, ras, pag, cfgzero, token_merge, linfusion, cachedit
|
|
log.info('Processing modifiers: unapply')
|
|
|
|
try:
|
|
sd_models_compile.check_deepcache(enable=False)
|
|
ipadapter.unapply(shared.sd_model, unload=getattr(p, 'ip_adapter_unload', False))
|
|
token_merge.remove_token_merging(shared.sd_model)
|
|
hidiffusion.unapply()
|
|
ras.unapply(shared.sd_model)
|
|
pag.unapply()
|
|
cfgzero.unapply()
|
|
linfusion.unapply(shared.sd_model)
|
|
cachedit.unapply_cache_dir(shared.sd_model)
|
|
except Exception as e:
|
|
log.error(f'Processing unapply: {e}')
|
|
errors.display(e, 'unapply')
|
|
timer.process.record('post')
|
|
|
|
|
|
def process_base(p: processing.StableDiffusionProcessing):
|
|
jobid = shared.state.begin('Base')
|
|
txt2img = is_txt2img()
|
|
use_refiner_start = is_refiner_enabled(p) and (not p.is_hr_pass)
|
|
use_denoise_start = not txt2img and p.refiner_start > 0 and p.refiner_start < 1
|
|
|
|
shared.sd_model = update_pipeline(shared.sd_model, p)
|
|
update_sampler(p, shared.sd_model)
|
|
timer.process.record('prepare')
|
|
process_pre(p)
|
|
sched_eta = p.scheduler_eta if p.scheduler_eta is not None else shared.opts.scheduler_eta
|
|
desc = 'Base'
|
|
if 'detailer' in p.ops:
|
|
desc = 'Detail'
|
|
p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts, p.network_data)
|
|
extra_networks.activate_filtered(p) # networks must patch weights before prompt encode so te loras affect embeds
|
|
base_args = set_pipeline_args(
|
|
p=p,
|
|
model=shared.sd_model,
|
|
prompts=p.prompts,
|
|
negative_prompts=p.negative_prompts,
|
|
prompts_2=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else p.prompts,
|
|
negative_prompts_2=[p.refiner_negative] if len(p.refiner_negative) > 0 else p.negative_prompts,
|
|
num_inference_steps=calculate_base_steps(p, use_refiner_start=use_refiner_start, use_denoise_start=use_denoise_start),
|
|
eta=sched_eta,
|
|
guidance_scale=p.cfg_scale if p.cfg_scale is not None and p.cfg_scale > -1 else None,
|
|
guidance_rescale=p.cfg_rescale if p.cfg_rescale is not None and p.cfg_rescale > -1 else None,
|
|
true_cfg_scale=p.cfg_true if p.cfg_true is not None and p.cfg_true > -1 else None,
|
|
denoising_start=0 if use_refiner_start else p.refiner_start if use_denoise_start else None,
|
|
denoising_end=p.refiner_start if use_refiner_start else 1 if use_denoise_start else None,
|
|
num_frames=getattr(p, 'frames', 1),
|
|
output_type=output_type,
|
|
clip_skip=p.clip_skip,
|
|
prompt_attention=getattr(p, 'prompt_attention', None),
|
|
desc=desc,
|
|
)
|
|
base_steps = base_args.get('prior_num_inference_steps', None) or p.steps or base_args.get('num_inference_steps', None)
|
|
shared.state.update(get_job_name(p, shared.sd_model), base_steps, 1)
|
|
if sched_eta is not None and sched_eta > 0 and sched_eta < 1:
|
|
p.extra_generation_params["Sampler Eta"] = sched_eta
|
|
output = None
|
|
if debug:
|
|
modelstats.analyze()
|
|
try:
|
|
t0 = time.time()
|
|
if hasattr(shared.sd_model, 'tgate') and getattr(p, 'gate_step', -1) > 0:
|
|
base_args['gate_step'] = p.gate_step
|
|
output = shared.sd_model.tgate(**base_args) # pylint: disable=not-callable
|
|
else:
|
|
taskid = shared.state.begin('Inference')
|
|
output = shared.sd_model(**base_args)
|
|
shared.state.end(taskid)
|
|
if isinstance(output, dict):
|
|
output = SimpleNamespace(**output)
|
|
if isinstance(output, list):
|
|
output = SimpleNamespace(images=output)
|
|
if isinstance(output, Image.Image):
|
|
output = SimpleNamespace(images=[output])
|
|
if not hasattr(output, 'frames') and hasattr(output, 'videos'):
|
|
output.frames = output.videos # modular video pipelines emit videos, not frames
|
|
if hasattr(output, 'image'):
|
|
output.images = output.image
|
|
if hasattr(output, 'images'):
|
|
shared.history.add(output.images, info=processing.create_infotext(p), ops=p.ops)
|
|
timer.process.record('pipeline')
|
|
sd_models_compile.openvino_post_compile(op="base") # only executes on compiled vino models
|
|
if shared.cmd_opts.profile:
|
|
t1 = time.time()
|
|
log.debug(f'Profile: pipeline call: {t1-t0:.2f}')
|
|
if not hasattr(output, 'images') and hasattr(output, 'frames'):
|
|
log.debug(f'Generated: frames={len(output.frames[0])}')
|
|
output.images = output.frames[0]
|
|
if hasattr(output, 'images') and isinstance(output.images, np.ndarray):
|
|
output.images = torch.from_numpy(output.images)
|
|
except AssertionError as e:
|
|
log.info(e)
|
|
except ValueError as e:
|
|
shared.state.interrupted = True
|
|
err_args = base_args.copy()
|
|
for k, v in base_args.items():
|
|
if isinstance(v, torch.Tensor):
|
|
err_args[k] = f'{v.device}:{v.dtype}:{v.shape}'
|
|
log.error(f'Processing: args={err_args} {e}')
|
|
errors.display(e, 'Processing')
|
|
except RuntimeError as e:
|
|
shared.state.interrupted = True
|
|
err_args = base_args.copy()
|
|
for k, v in base_args.items():
|
|
if isinstance(v, torch.Tensor):
|
|
err_args[k] = f'{v.device}:{v.dtype}:{v.shape}'
|
|
log.error(f'Processing: step=base args={err_args} {e}')
|
|
errors.display(e, 'Processing')
|
|
modelstats.analyze()
|
|
finally:
|
|
process_post(p)
|
|
|
|
if hasattr(shared.sd_model, 'postprocess') and callable(shared.sd_model.postprocess):
|
|
output = shared.sd_model.postprocess(p, output)
|
|
|
|
if hasattr(shared.sd_model, 'sdnext_phaseid'):
|
|
shared.state.end(shared.sd_model.sdnext_phaseid)
|
|
shared.sd_model.sdnext_phaseid = None
|
|
shared.state.end(jobid)
|
|
shared.state.nextjob()
|
|
return output
|
|
|
|
|
|
def process_hires(p: processing.StableDiffusionProcessing, output):
|
|
# optional second pass
|
|
if (output is None) or not hasattr(output, 'images') or (output.images is None):
|
|
return output
|
|
if p.enable_hr:
|
|
jobid = shared.state.begin('Hires')
|
|
p.is_hr_pass = True
|
|
if hasattr(p, 'init_hr'):
|
|
p.init_hr(p.hr_scale, p.hr_upscaler, force=p.hr_force)
|
|
else:
|
|
if not p.is_hr_pass: # fake hires for img2img if not actual hr pass
|
|
p.hr_scale = p.scale_by
|
|
p.hr_upscaler = p.resize_name
|
|
p.hr_resize_mode = p.resize_mode
|
|
p.hr_resize_context = p.resize_context
|
|
p.hr_upscale_to_x = int(p.width * p.hr_scale) if p.hr_resize_x == 0 else p.hr_resize_x
|
|
p.hr_upscale_to_y = int(p.height * p.hr_scale) if p.hr_resize_y == 0 else p.hr_resize_y
|
|
|
|
# hires runs on original pipeline
|
|
if hasattr(shared.sd_model, 'restore_pipeline') and (shared.sd_model.restore_pipeline is not None) and (not shared.opts.control_hires):
|
|
shared.sd_model.restore_pipeline()
|
|
if (getattr(shared.sd_model, 'controlnet', None) is not None) and (((isinstance(shared.sd_model.controlnet, list) and len(shared.sd_model.controlnet) > 1)) or ('Multi' in type(shared.sd_model.controlnet).__name__)):
|
|
log.warning(f'Process: control={type(shared.sd_model.controlnet)} not supported in hires')
|
|
return output
|
|
|
|
# upscale
|
|
if hasattr(p, 'height') and hasattr(p, 'width') and p.hr_resize_mode > 0 and (p.hr_upscaler != 'None' or p.hr_resize_mode == 5):
|
|
log.info(f'Upscale: mode={p.hr_resize_mode} upscaler="{p.hr_upscaler}" context="{p.hr_resize_context}" resize={p.hr_resize_x}x{p.hr_resize_y} upscale={p.hr_upscale_to_x}x{p.hr_upscale_to_y}')
|
|
p.ops.append('upscale')
|
|
if shared.opts.samples_save and not p.do_not_save_samples and shared.opts.save_images_before_highres_fix and hasattr(shared.sd_model, 'vae'):
|
|
save_intermediate(p, latents=output.images, suffix="-before-hires")
|
|
output.images = resize_hires(p, latents=output.images)
|
|
sd_hijack_hypertile.hypertile_set(p, hr=True)
|
|
elif torch.is_tensor(output.images) and output.images.shape[-1] == 3: # nhwc
|
|
if output.images.dim() == 3:
|
|
output.images = convert.to_pil(output.images)
|
|
elif output.images.dim() == 4:
|
|
output.images = [convert.to_pil(output.images[i]) for i in range(output.images.shape[0])]
|
|
|
|
strength = p.hr_denoising_strength if p.hr_denoising_strength > 0 else p.denoising_strength
|
|
if (p.hr_upscaler is not None) and (p.hr_upscaler.lower().startswith('latent') or p.hr_force) and strength > 0:
|
|
p.ops.append('hires')
|
|
sd_models_compile.openvino_recompile_model(p, hires=True, refiner=False)
|
|
if shared.sd_model.__class__.__name__ == "OnnxRawPipeline":
|
|
shared.sd_model = preprocess_onnx_pipeline(p)
|
|
p.hr_force = True
|
|
|
|
# hires
|
|
if p.hr_force and strength == 0:
|
|
log.warning('Hires skip: denoising=0')
|
|
p.hr_force = False
|
|
if p.hr_force:
|
|
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE)
|
|
if 'Upscale' in shared.sd_model.__class__.__name__ or 'Flux' in shared.sd_model.__class__.__name__ or 'Kandinsky' in shared.sd_model.__class__.__name__:
|
|
output.images = processing_vae.vae_decode(latents=output.images, model=shared.sd_model, vae_type=p.vae_type, output_type='pil', width=p.width, height=p.height)
|
|
if p.is_control and hasattr(p, 'task_args') and p.task_args.get('image', None) is not None:
|
|
if hasattr(shared.sd_model, "vae") and output.images is not None and len(output.images) > 0:
|
|
output.images = processing_vae.vae_decode(latents=output.images, model=shared.sd_model, vae_type=p.vae_type, output_type='pil', width=p.hr_upscale_to_x, height=p.hr_upscale_to_y) # controlnet cannot deal with latent input
|
|
update_sampler(p, shared.sd_model, second_pass=True)
|
|
orig_denoise = p.denoising_strength
|
|
p.denoising_strength = strength
|
|
orig_image = p.task_args.pop('image', None) # remove image override from hires
|
|
process_pre(p)
|
|
|
|
prompts = p.prompts
|
|
reset_prompts = False
|
|
sched_eta = p.scheduler_eta if p.scheduler_eta is not None else shared.opts.scheduler_eta
|
|
if len(p.refiner_prompt) > 0:
|
|
prompts = len(output.images)* [p.refiner_prompt]
|
|
prompts, p.network_data = extra_networks.parse_prompts(prompts)
|
|
reset_prompts = True
|
|
if reset_prompts or ('base' in p.skip):
|
|
extra_networks.activate_filtered(p)
|
|
|
|
hires_args = set_pipeline_args(
|
|
p=p,
|
|
model=shared.sd_model,
|
|
prompts=prompts,
|
|
negative_prompts=len(output.images) * [p.refiner_negative] if len(p.refiner_negative) > 0 else p.negative_prompts,
|
|
prompts_2=len(output.images) * [p.refiner_prompt] if len(p.refiner_prompt) > 0 else p.prompts,
|
|
negative_prompts_2=len(output.images) * [p.refiner_negative] if len(p.refiner_negative) > 0 else p.negative_prompts,
|
|
num_inference_steps=calculate_hires_steps(p),
|
|
eta=sched_eta,
|
|
guidance_scale=p.cfg_image if p.cfg_image is not None and p.cfg_image > -1 else p.cfg_scale,
|
|
guidance_rescale=p.cfg_rescale if p.cfg_rescale is not None and p.cfg_rescale > -1 else None,
|
|
true_cfg_scale=p.cfg_true if p.cfg_true is not None and p.cfg_true > -1 else None,
|
|
output_type=output_type,
|
|
clip_skip=p.clip_skip,
|
|
image=output.images,
|
|
strength=strength,
|
|
prompt_attention=getattr(p, 'prompt_attention', None),
|
|
desc='Hires',
|
|
)
|
|
|
|
hires_steps = hires_args.get('prior_num_inference_steps', None) or p.hr_second_pass_steps or hires_args.get('num_inference_steps', None)
|
|
shared.state.update(get_job_name(p, shared.sd_model), hires_steps, 1)
|
|
try:
|
|
taskid = shared.state.begin('Inference')
|
|
output = shared.sd_model(**hires_args) # pylint: disable=not-callable
|
|
shared.state.end(taskid)
|
|
if isinstance(output, dict):
|
|
output = SimpleNamespace(**output)
|
|
if hasattr(output, 'images'):
|
|
shared.history.add(output.images, info=processing.create_infotext(p), ops=p.ops)
|
|
sd_models_compile.check_deepcache(enable=False)
|
|
sd_models_compile.openvino_post_compile(op="base")
|
|
except AssertionError as e:
|
|
log.info(e)
|
|
except RuntimeError as e:
|
|
shared.state.interrupted = True
|
|
log.error(f'Processing step=hires: args={hires_args} {e}')
|
|
errors.display(e, 'Processing')
|
|
modelstats.analyze()
|
|
finally:
|
|
process_post(p)
|
|
if hasattr(shared.sd_model, 'postprocess') and callable(shared.sd_model.postprocess):
|
|
output = shared.sd_model.postprocess(p, output)
|
|
if orig_image is not None:
|
|
p.task_args['image'] = orig_image
|
|
p.denoising_strength = orig_denoise
|
|
shared.state.end(jobid)
|
|
shared.state.nextjob()
|
|
p.is_hr_pass = False
|
|
timer.process.record('hires')
|
|
return output
|
|
|
|
|
|
def process_refine(p: processing.StableDiffusionProcessing, output):
|
|
# optional refiner pass or decode
|
|
if (output is None) or not hasattr(output, 'images') or (output.images is None):
|
|
return output
|
|
if is_refiner_enabled(p):
|
|
if shared.opts.samples_save and not p.do_not_save_samples and shared.opts.save_images_before_refiner and hasattr(shared.sd_model, 'vae'):
|
|
save_intermediate(p, latents=output.images, suffix="-before-refiner")
|
|
if shared.state.interrupted or shared.state.skipped:
|
|
shared.sd_model = orig_pipeline
|
|
return output
|
|
jobid = shared.state.begin('Refine')
|
|
shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model)
|
|
p.ops.append('refine')
|
|
p.is_refiner_pass = True
|
|
|
|
sd_models_compile.openvino_recompile_model(p, hires=False, refiner=True)
|
|
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
|
|
shared.sd_refiner = sd_models.set_diffuser_pipe(shared.sd_refiner, sd_models.DiffusersTaskType.IMAGE_2_IMAGE)
|
|
for i in range(len(output.images)):
|
|
image = output.images[i]
|
|
noise_level = round(350 * p.denoising_strength)
|
|
refiner_output_type = output_type
|
|
if 'Upscale' in shared.sd_refiner.__class__.__name__ or 'Flux' in shared.sd_refiner.__class__.__name__ or 'Kandinsky' in shared.sd_refiner.__class__.__name__:
|
|
image = processing_vae.vae_decode(latents=image, model=shared.sd_model, vae_type=p.vae_type, output_type='pil', width=p.width, height=p.height)
|
|
p.extra_generation_params['Noise level'] = noise_level
|
|
refiner_output_type = 'np'
|
|
update_sampler(p, shared.sd_refiner, second_pass=True)
|
|
sched_eta = p.scheduler_eta if p.scheduler_eta is not None else shared.opts.scheduler_eta
|
|
refiner_args = set_pipeline_args(
|
|
p=p,
|
|
model=shared.sd_refiner,
|
|
prompts=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else p.prompts[i],
|
|
negative_prompts=[p.refiner_negative] if len(p.refiner_negative) > 0 else p.negative_prompts[i],
|
|
num_inference_steps=calculate_refiner_steps(p),
|
|
eta=sched_eta,
|
|
noise_level=noise_level, # StableDiffusionUpscalePipeline only
|
|
guidance_scale=p.cfg_image if p.cfg_image is not None and p.cfg_image > -1 else p.cfg_scale,
|
|
guidance_rescale=p.cfg_rescale if p.cfg_rescale is not None and p.cfg_rescale > -1 else None,
|
|
true_cfg_scale=p.cfg_true if p.cfg_true is not None and p.cfg_true > -1 else None,
|
|
denoising_start=p.refiner_start if p.refiner_start > 0 and p.refiner_start < 1 else None,
|
|
denoising_end=1 if p.refiner_start > 0 and p.refiner_start < 1 else None,
|
|
image=image,
|
|
output_type=refiner_output_type,
|
|
clip_skip=p.clip_skip,
|
|
prompt_attention='fixed',
|
|
desc='Refiner',
|
|
)
|
|
refiner_steps = refiner_args.get('prior_num_inference_steps', None) or p.steps or refiner_args.get('num_inference_steps', None)
|
|
shared.state.update(get_job_name(p, shared.sd_refiner), refiner_steps, 1)
|
|
try:
|
|
if 'requires_aesthetics_score' in shared.sd_refiner.config: # sdxl-model needs false and sdxl-refiner needs true
|
|
shared.sd_refiner.register_to_config(requires_aesthetics_score = getattr(shared.sd_refiner, 'tokenizer', None) is None)
|
|
output = shared.sd_refiner(**refiner_args) # pylint: disable=not-callable
|
|
if isinstance(output, dict):
|
|
output = SimpleNamespace(**output)
|
|
if hasattr(output, 'images'):
|
|
shared.history.add(output.images, info=processing.create_infotext(p), ops=p.ops)
|
|
sd_models_compile.openvino_post_compile(op="refiner")
|
|
except AssertionError as e:
|
|
log.info(e)
|
|
except RuntimeError as e:
|
|
shared.state.interrupted = True
|
|
log.error(f'Processing step=refine: args={refiner_args} {e}')
|
|
errors.display(e, 'Processing')
|
|
modelstats.analyze()
|
|
|
|
if shared.opts.diffusers_offload_mode == "balanced":
|
|
shared.sd_refiner = sd_models.apply_balanced_offload(shared.sd_refiner)
|
|
shared.state.end(jobid)
|
|
shared.state.nextjob()
|
|
p.is_refiner_pass = False
|
|
timer.process.record('refine')
|
|
return output
|
|
|
|
|
|
class AudioFrameList(list):
|
|
"""list subclass with an audio attribute. Plain lists can't hold attributes,
|
|
so process_decode uses this when the pipeline output includes audio."""
|
|
audio = None
|
|
|
|
|
|
def attach_audio(results, audio):
|
|
if audio is None:
|
|
return results
|
|
wrapped = AudioFrameList(results if isinstance(results, list) else list(results))
|
|
wrapped.audio = audio
|
|
return wrapped
|
|
|
|
|
|
def process_decode(p: processing.StableDiffusionProcessing, output):
|
|
shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae'])
|
|
if output is not None:
|
|
if hasattr(output, 'bytes') and output.bytes is not None:
|
|
log.debug(f'Generated: bytes={len(output.bytes)}')
|
|
return output
|
|
audio = getattr(output, 'audio', None)
|
|
if audio is not None:
|
|
p.audio_sampling_rate = getattr(output, 'sampling_rate', None)
|
|
if not hasattr(output, 'images') and hasattr(output, 'frames'):
|
|
log.debug(f'Generated: frames={len(output.frames[0])}')
|
|
output.images = output.frames[0]
|
|
if getattr(p, 'video_still', False) and hasattr(output, 'images') and output.images is not None:
|
|
output.images = output.images[:1] # only the first frame derives from real latents; the rest decode from padding
|
|
if output.images is not None and len(output.images) > 0 and isinstance(output.images[0], Image.Image):
|
|
sd_models.offload_ondemand(shared.sd_model) # in-pipe decode paths return materialized frames; the vae seam in processing_vae never runs
|
|
return attach_audio(output.images, audio)
|
|
model = shared.sd_model if not is_refiner_enabled(p) else shared.sd_refiner
|
|
if not hasattr(model, 'vae'):
|
|
if hasattr(model, 'pipe') and hasattr(model.pipe, 'vae'):
|
|
model = model.pipe
|
|
if (hasattr(model, "vae") or hasattr(model, "vqgan")) and (output.images is not None) and (len(output.images) > 0):
|
|
if p.hr_resize_mode > 0 and (p.hr_upscaler != 'None' or p.hr_resize_mode == 5):
|
|
width = max(getattr(p, 'width', 0), getattr(p, 'hr_upscale_to_x', 0))
|
|
height = max(getattr(p, 'height', 0), getattr(p, 'hr_upscale_to_y', 0))
|
|
else:
|
|
width = getattr(p, 'width', 0)
|
|
height = getattr(p, 'height', 0)
|
|
frames = p.task_args.get('num_frames', None) or getattr(p, 'frames', None)
|
|
if isinstance(output.images, list):
|
|
results = []
|
|
for i in range(len(output.images)):
|
|
result_batch = processing_vae.vae_decode(
|
|
latents = output.images[i],
|
|
model = model,
|
|
vae_type = p.vae_type,
|
|
width = width,
|
|
height = height,
|
|
frames = frames,
|
|
)
|
|
for result in list(result_batch):
|
|
results.append(result)
|
|
else:
|
|
results = processing_vae.vae_decode(
|
|
latents = output.images,
|
|
model = model,
|
|
vae_type = p.vae_type,
|
|
width = width,
|
|
height = height,
|
|
frames = frames,
|
|
)
|
|
if not isinstance(results, list):
|
|
results = list(results)
|
|
elif hasattr(output, 'images'):
|
|
results = output.images
|
|
else:
|
|
log.warning('Processing: no results')
|
|
results = []
|
|
else:
|
|
log.warning('Processing: no results')
|
|
audio = None
|
|
results = []
|
|
return attach_audio(results, audio)
|
|
|
|
|
|
def update_pipeline(sd_model, p: processing.StableDiffusionProcessing):
|
|
if sd_model is None:
|
|
sd_model = shared.sd_model
|
|
if sd_model is None:
|
|
log.warning('Processing: op=update model not loaded')
|
|
return None
|
|
updated_model = sd_model
|
|
if 'MiniMaxH3' in sd_model.__class__.__name__ and not isinstance(p, processing.StableDiffusionProcessingVideo):
|
|
# image tabs run the model in still mode; the video tab applies its own overrides
|
|
from modules.video_models import video_minimax
|
|
video_minimax.apply_overrides(p, sd_model, still=True, audio=False)
|
|
if getattr(p, 'detailer_enabled', False):
|
|
log.warning(f'Processing: cls={sd_model.__class__.__name__} detailer not supported')
|
|
p.detailer_enabled = False
|
|
if sd_models.get_diffusers_task(sd_model) == sd_models.DiffusersTaskType.INPAINTING and getattr(p, 'image_mask', None) is None and p.task_args.get('image_mask', None) is None and getattr(p, 'mask', None) is None:
|
|
log.warning('Processing: mode=inpaint mask=None')
|
|
updated_model = sd_models.set_diffuser_pipe(sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE)
|
|
if shared.opts.cuda_compile_backend == "olive-ai":
|
|
updated_model = olive_check_parameters_changed(p, is_refiner_enabled(p))
|
|
if sd_model.__class__.__name__ == "OnnxRawPipeline":
|
|
updated_model = preprocess_onnx_pipeline(p)
|
|
global orig_pipeline # pylint: disable=global-statement
|
|
orig_pipeline = updated_model # processed ONNX pipeline should not be replaced with original pipeline.
|
|
current_attn = getattr(updated_model, "current_attn_name", None)
|
|
if (current_attn != shared.opts.cross_attention_optimization) and (current_attn != shared.opts.sdp_overrides):
|
|
log.info(f"Setting attention optimization: {shared.opts.cross_attention_optimization}")
|
|
attention.set_diffusers_attention(updated_model)
|
|
return updated_model
|
|
|
|
|
|
def validate_pipeline(p: processing.StableDiffusionProcessing):
|
|
from modules.video_models import models_def
|
|
is_video_model = shared.sd_model.__class__.__name__ in models_def.pipeline_classes()
|
|
override_video_pipelines = ['WanPipeline', 'WanImageToVideoPipeline', 'WanVACEPipeline', 'MiniMaxH3ModularPipeline']
|
|
is_video_pipeline = ('video' in p.__class__.__name__.lower()) or (shared.sd_model.__class__.__name__ in override_video_pipelines)
|
|
if is_video_model and not is_video_pipeline:
|
|
log.error(f'Mismatch: type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__} request={p.__class__.__name__} video model with non-video pipeline')
|
|
return False
|
|
elif not is_video_model and is_video_pipeline:
|
|
log.error(f'Mismatch: type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__} request={p.__class__.__name__} non-video model with video pipeline')
|
|
return False
|
|
if getattr(shared.sd_model, 'sdnext_video_workflow', None) == 'ref2va' and p.task_args.get('references', None) is None:
|
|
# the reference workflow loads its own transformer partition alone: without references the pipeline
|
|
# dispatches to the keyframe path and reaches a transformer that was never loaded
|
|
log.error(f'Mismatch: type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__} request={p.__class__.__name__} reference workflow requires reference images: use the video tab or the video api')
|
|
return False
|
|
return True
|
|
|
|
|
|
def process_diffusers(p: processing.StableDiffusionProcessing):
|
|
results = []
|
|
if debug:
|
|
log.trace(f'Process diffusers args: {vars(p)}')
|
|
if not validate_pipeline(p):
|
|
return results
|
|
|
|
p = restore_state(p)
|
|
global orig_pipeline # pylint: disable=global-statement
|
|
orig_pipeline = shared.sd_model
|
|
|
|
if shared.state.interrupted or shared.state.skipped:
|
|
shared.sd_model = orig_pipeline
|
|
return results
|
|
|
|
# sanitize init_images
|
|
if hasattr(p, 'init_images') and not isinstance(getattr(p, 'init_images', []), list):
|
|
p.init_images = [p.init_images]
|
|
if hasattr(p, 'init_images') and isinstance(getattr(p, 'init_images', []), list):
|
|
p.init_images = [i for i in p.init_images if i is not None]
|
|
if len(getattr(p, 'init_images', [])) > 0:
|
|
while len(p.init_images) < len(p.prompts):
|
|
p.init_images.append(p.init_images[-1])
|
|
|
|
# pipeline type is set earlier in processing, but check for sanity
|
|
is_control = getattr(p, 'is_control', False) is True
|
|
has_images = len(getattr(p, 'init_images', [])) > 0
|
|
if (sd_models.get_diffusers_task(shared.sd_model) != sd_models.DiffusersTaskType.TEXT_2_IMAGE) and (not has_images) and (not is_control):
|
|
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) # reset pipeline
|
|
if hasattr(shared.sd_model, 'unet') and hasattr(shared.sd_model.unet, 'config') and hasattr(shared.sd_model.unet.config, 'in_channels') and shared.sd_model.unet.config.in_channels == 9 and not is_control:
|
|
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.INPAINTING) # force pipeline
|
|
if len(getattr(p, 'init_images', [])) == 0:
|
|
p.init_images = [convert.to_pil(torch.rand((3, getattr(p, 'height', 512), getattr(p, 'width', 512))))]
|
|
if not p.prompts:
|
|
p.prompts = p.all_prompts[p.iteration * p.batch_size:(p.iteration+1) * p.batch_size]
|
|
if not p.negative_prompts:
|
|
p.negative_prompts = p.all_negative_prompts[p.iteration * p.batch_size:(p.iteration+1) * p.batch_size]
|
|
|
|
sd_models_compile.openvino_recompile_model(p, hires=False, refiner=False) # recompile if a parameter changes
|
|
|
|
if hasattr(p, 'dummy'):
|
|
images = [Image.new(mode='RGB', size=(p.width, p.height))]
|
|
return images
|
|
|
|
if 'base' not in p.skip:
|
|
output = process_base(p)
|
|
else:
|
|
# images, _index = shared.history.selected
|
|
images = shared.history.last_latent
|
|
output = SimpleNamespace(images=images) if images is not None else None
|
|
|
|
if (output is None or (hasattr(output, 'images') and len(output.images) == 0)) and has_images:
|
|
if output is not None:
|
|
log.debug('Processing: using input as base output')
|
|
output.images = p.init_images
|
|
|
|
if shared.state.interrupted or shared.state.skipped:
|
|
shared.sd_model = orig_pipeline
|
|
return results
|
|
|
|
if 'hires' not in p.skip:
|
|
output = process_hires(p, output)
|
|
if shared.state.interrupted or shared.state.skipped:
|
|
shared.sd_model = orig_pipeline
|
|
return results
|
|
|
|
if 'refine' not in p.skip:
|
|
output = process_refine(p, output)
|
|
if shared.state.interrupted or shared.state.skipped:
|
|
shared.sd_model = orig_pipeline
|
|
return results
|
|
|
|
extra_networks.deactivate(p)
|
|
timer.process.add('lora', lora_common.timer.total)
|
|
lora_common.timer.clear(complete=True)
|
|
|
|
results = process_decode(p, output)
|
|
timer.process.record('decode')
|
|
|
|
shared.sd_model = orig_pipeline
|
|
|
|
if p.state == '':
|
|
global last_p # pylint: disable=global-statement
|
|
last_p = p
|
|
return results
|