Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-20 01:31:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

redesign diffuser vae handling

Browse Source
This commit is contained in:
Vladimir Mandic
2023-07-21 14:30:57 -04:00
parent 67e2258639
commit 4c2664dbc3
9 changed files with 60 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+14 -1
View File
@@ -1,10 +1,23 @@
# Change Log for SD.Next
## Update for 07/20/2023
## Update for 07/21/2023
- new loading screens and artwork
- extra networks: add add/remove tags to prompt (e.g. lora activation keywords)
- extensions: fix couple of compatibility items
- number of hires fixes
- diffusers: option to set vae upcast in settings
- sd-xl: enable fp16 vae decode when using optimized vae
this pretty much doubles performance of decode step (delay after generate is done)
- sd-xl: loading vae now applies to both base and refiner
- diffusers 0.19.dev
- sd-xl: denoising_start/denoising_end
- sd-xl: enable dual prompts
this is used regardless if refiner is enabled/loaded
if refiner is loaded & enabled, refiner prompt will also be used for refiner pass
- primary prompt goes to [OpenAI CLIP-ViT/L-14](https://huggingface.co/openai/clip-vit-large-patch14)
- refiner prompt goes to [OpenCLIP-ViT/bigG-14](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k)
## Update for 07/18/2023
javascript/style.css
+1 -1
View File
@@ -105,7 +105,7 @@ button.custom-button{
#txt2img_footer, #img2img_footer { height: fit-content; display: none; }
#txt2img_generate_box, #img2img_generate_box { gap: 0.5em; flex-wrap: wrap-reverse; }
#txt2img_actions_column, #img2img_actions_column { gap: 0.5em; }
#txt2img_generate_box > button, #img2img_generate_box > button { height: 2.2em; line-height: 0; }
#txt2img_generate_box > button, #img2img_generate_box > button { min-height: 36px; max-height: 42px; }
#txt2img_generate_line2, #img2img_generate_line2 { display: flex; }
#txt2img_generate_line2 > button, #img2img_generate_line2 > button, #extras_generate_box > button { height: 2.2em; line-height: 0; min-width: unset; display: block !important; }
#txt2img_tools > div, #img2img_tools > div { justify-content: space-around; margin-top: 0.5em; margin-bottom: 0em; }
modules/processing_diffusers.py
+20 -2
View File
@@ -16,6 +16,16 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
shared.state.sampling_steps = p.steps
shared.state.current_latent = latents
def vae_decode(latents, model):
if hasattr(model, 'vae'):
shared.log.debug(f'Diffusers VAE decode: name={model.vae.config.get("_name_or_path", "default")} upcast={model.vae.config.get("force_upcast", None)}')
decoded = model.vae.decode(latents / model.vae.config.scaling_factor, return_dict=False)[0]
images = model.image_processor.postprocess(decoded, output_type='np')
return images
else:
return latents
def set_pipeline_args(model, prompt, negative_prompt, **kwargs):
args = {}
pipeline = model.main if model.__class__.__name__ == 'PriorPipeline' else model
@@ -83,14 +93,21 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
model=shared.sd_model,
prompt=prompts,
negative_prompt=negative_prompts,
prompt_2=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts,
negative_prompt_2=[p.refiner_negative] if len(p.refiner_negative) > 0 else negative_prompts,
eta=shared.opts.eta_ddim,
guidance_rescale=p.diffusers_guidance_rescale,
denoising_start=p.refiner_denoise_start,
denoising_end=p.refiner_denoise_end,
# aesthetic_score=shared.opts.diffusers_aesthetics_score,
output_type='np' if (shared.sd_refiner is None or p.enable_hr is False or not shared.opts.diffusers_refiner_latents) else 'latent',
output_type='latent' if hasattr(shared.sd_model, 'vae') else 'np',
**task_specific_kwargs
)
output = shared.sd_model(**pipe_args) # pylint: disable=not-callable
if shared.sd_refiner is None or not p.enable_hr:
output.images = vae_decode(output.images, shared.sd_model)
if shared.sd_refiner is not None and p.enable_hr:
if shared.opts.diffusers_move_base:
shared.log.debug('Moving base model to CPU')
@@ -128,9 +145,10 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro
denoising_start=p.refiner_denoise_start,
denoising_end=p.refiner_denoise_end,
image=output.images[i],
output_type='np',
output_type='latent' if hasattr(shared.sd_model, 'vae') else 'np',
)
output = shared.sd_refiner(**pipe_args) # pylint: disable=not-callable
output.images = vae_decode(output.images, shared.sd_model)
results.append(output.images[0])
if shared.opts.diffusers_move_refiner:
modules/sd_models.py
+20 -13
View File
@@ -604,7 +604,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
if (model_data.sd_refiner is not None) and (checkpoint_info is not None) and (checkpoint_info.hash == model_data.sd_refiner.sd_checkpoint_info.hash): # trying to load the same model
return
shared.log.debug(f'Diffusers load config: {diffusers_load_config}')
shared.log.debug(f'Diffusers load {op} config: {diffusers_load_config}')
sd_model = None
try:
@@ -629,7 +629,8 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
return
shared.log.info(f'Loading diffuser {op}: {checkpoint_info.filename}')
if op == 'model':
vae = None
if op == 'model' or op == 'refiner':
vae_file, vae_source = sd_vae.resolve_vae(checkpoint_info.filename)
vae = sd_vae.load_vae_diffusers(None, vae_file, vae_source)
if vae is not None:
@@ -639,7 +640,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
try:
sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, **diffusers_load_config)
except Exception as e:
shared.log.error(f'Diffusers failed loading model: {checkpoint_info.path} {e}')
shared.log.error(f'Diffusers {op} failed loading model: {checkpoint_info.path} {e}')
else:
diffusers_load_config["local_files_only "] = True
diffusers_load_config["extract_ema"] = shared.opts.diffusers_extract_ema
@@ -669,9 +670,9 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
elif shared.opts.diffusers_pipeline == shared.pipelines[11]:
pipeline = diffusers.ShapEImg2ImgPipeline
else:
shared.log.error(f'Diffusers unknown pipeline: {shared.opts.diffusers_pipeline}')
shared.log.error(f'Diffusers {op} unknown pipeline: {shared.opts.diffusers_pipeline}')
except Exception as e:
shared.log.error(f'Diffusers failed initializing pipeline: {shared.opts.diffusers_pipeline} {e}')
shared.log.error(f'Diffusers {op} failed initializing pipeline: {shared.opts.diffusers_pipeline} {e}')
return
try:
if hasattr(pipeline, 'from_single_file'):
@@ -680,10 +681,10 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
elif hasattr(pipeline, 'from_ckpt'):
sd_model = pipeline.from_ckpt(checkpoint_info.path, **diffusers_load_config)
else:
shared.log.error(f'Diffusers cannot load safetensor model: {checkpoint_info.path} {shared.opts.diffusers_pipeline}')
shared.log.error(f'Diffusers {op} cannot load safetensor model: {checkpoint_info.path} {shared.opts.diffusers_pipeline}')
return
if sd_model is not None:
shared.log.debug(f'Diffusers pipeline: {sd_model.__class__.__name__}') # pylint: disable=protected-access
shared.log.debug(f'Diffusers {op}: pipeline={sd_model.__class__.__name__}') # pylint: disable=protected-access
except Exception as e:
shared.log.error(f'Diffusers failed loading model using pipeline: {checkpoint_info.path} {shared.opts.diffusers_pipeline} {e}')
return
@@ -705,34 +706,40 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
if hasattr(sd_model, "enable_model_cpu_offload"):
if shared.cmd_opts.medvram or shared.opts.diffusers_model_cpu_offload:
shared.log.debug('Diffusers: enable model CPU offload')
shared.log.debug(f'Diffusers {op}: enable model CPU offload')
sd_model.enable_model_cpu_offload()
if hasattr(sd_model, "enable_sequential_cpu_offload"):
if shared.opts.diffusers_seq_cpu_offload:
sd_model.enable_sequential_cpu_offload()
shared.log.debug('Diffusers: enable sequential CPU offload')
shared.log.debug(f'Diffusers {op}: enable sequential CPU offload')
if hasattr(sd_model, "enable_vae_slicing"):
if shared.cmd_opts.lowvram or shared.opts.diffusers_vae_slicing:
shared.log.debug('Diffusers: enable VAE slicing')
shared.log.debug(f'Diffusers {op}: enable VAE slicing')
sd_model.enable_vae_slicing()
else:
sd_model.disable_vae_slicing()
if hasattr(sd_model, "enable_vae_tiling"):
if shared.cmd_opts.lowvram or shared.opts.diffusers_vae_tiling:
shared.log.debug('Diffusers: enable VAE tiling')
shared.log.debug(f'Diffusers {op}: enable VAE tiling')
sd_model.enable_vae_tiling()
else:
sd_model.disable_vae_tiling()
if hasattr(sd_model, "enable_attention_slicing"):
if shared.cmd_opts.lowvram or shared.opts.diffusers_attention_slicing:
shared.log.debug('Diffusers: enable attention slicing')
shared.log.debug(f'Diffusers {op}: enable attention slicing')
sd_model.enable_attention_slicing()
else:
sd_model.disable_attention_slicing()
if hasattr(sd_model, "vae"):
if vae is not None:
shared.log.debug(f'Diffusers {op} VAE: name={vae.config.get("_name_or_path", "default")} upcast={vae.config.get("force_upcast", None)}')
sd_model.vae = vae # pylint: disable=attribute-defined-outside-init
if shared.opts.diffusers_vae_upcast != 'default':
sd_model.vae.config.force_upcast = True if shared.opts.upcast_sampling == 'true' else False
if shared.opts.cross_attention_optimization == "xFormers" and hasattr(sd_model, 'enable_xformers_memory_efficient_attention'):
sd_model.enable_xformers_memory_efficient_attention()
if shared.opts.opt_channelslast:
shared.log.debug('Diffusers: enable channels last')
shared.log.debug(f'Diffusers {op}: enable channels last')
sd_model.unet.to(memory_format=torch.channels_last)
base_sent_to_cpu=False
modules/sd_vae.py
+1
View File
@@ -177,6 +177,7 @@ def load_vae_diffusers(_model, vae_file=None, vae_source="from unknown source"):
try:
import diffusers
vae = diffusers.AutoencoderKL.from_pretrained(vae_file, **diffusers_load_config)
# shared.log.debug(f'Diffusers VAE config: {vae.config}')
return vae
except Exception as e:
shared.log.error(f"Loading diffusers VAE failed: {vae_file} {e}")
modules/shared.py
+1
View File
@@ -406,6 +406,7 @@ options_templates.update(options_section(('diffusers', "Diffusers Settings"), {
"diffusers_generator_device": OptionInfo("default", "Generator device", gr.Radio, lambda: {"choices": ["default", "cpu"]}),
"diffusers_seq_cpu_offload": OptionInfo(False, "Enable sequential CPU offload"),
"diffusers_model_cpu_offload": OptionInfo(False, "Enable model CPU offload"),
"diffusers_vae_upcast": OptionInfo("default", "VAE upcasting", gr.Radio, lambda: {"choices": ['default', 'true', 'false']}),
"diffusers_vae_slicing": OptionInfo(True, "Enable VAE slicing"),
"diffusers_vae_tiling": OptionInfo(False, "Enable VAE tiling"),
"diffusers_attention_slicing": OptionInfo(False, "Enable attention slicing"),
modules/ui.py
+1 -1
View File
@@ -399,7 +399,7 @@ def create_ui(startup_timer = None):
with FormRow(elem_id="txt2img_refiner_row1", variant="compact"):
image_cfg_scale = gr.Slider(minimum=1.1, maximum=30.0, step=0.1, label='Secondary CFG Scale', value=6.0, elem_id="txt2img_image_cfg_scale")
diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Guidance rescale', value=0.7, elem_id="txt2img_image_cfg_rescale")
refiner_denoise_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Denoise start', value=0.0, elem_id="txt2img_refiner_denoise_start")
refiner_denoise_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Denoise start', value=0.8, elem_id="txt2img_refiner_denoise_start")
refiner_denoise_end = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Denoise end', value=1.0, elem_id="txt2img_refiner_denoise_end")
with FormRow(elem_id="txt2img_refiner_row2", variant="compact"):
refiner_prompt = gr.Textbox(value='', label='Prompt')
modules/ui_common.py
+1 -1
View File
@@ -40,7 +40,7 @@ def infotext_to_html(text):
return res
def delete_files(js_data, images, _html_info, _do_make_zip, index):
def delete_files(js_data, images, _html_info, index):
try:
data = json.loads(js_data)
except Exception:
wiki
+1 -1
Submodule wiki updated: f6877509db...dafa622e44