diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f4b899c5..46c4670fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,23 @@ # Change Log for SD.Next -## Update for 07/20/2023 +## Update for 07/21/2023 - new loading screens and artwork - extra networks: add add/remove tags to prompt (e.g. lora activation keywords) - extensions: fix couple of compatibility items +- number of hires fixes +- diffusers: option to set vae upcast in settings + - sd-xl: enable fp16 vae decode when using optimized vae + this pretty much doubles performance of decode step (delay after generate is done) + - sd-xl: loading vae now applies to both base and refiner + +- diffusers 0.19.dev + - sd-xl: denoising_start/denoising_end + - sd-xl: enable dual prompts + this is used regardless if refiner is enabled/loaded + if refiner is loaded & enabled, refiner prompt will also be used for refiner pass + - primary prompt goes to [OpenAI CLIP-ViT/L-14](https://huggingface.co/openai/clip-vit-large-patch14) + - refiner prompt goes to [OpenCLIP-ViT/bigG-14](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k) ## Update for 07/18/2023 diff --git a/javascript/style.css b/javascript/style.css index 9f663af1b..93386ee1e 100644 --- a/javascript/style.css +++ b/javascript/style.css @@ -105,7 +105,7 @@ button.custom-button{ #txt2img_footer, #img2img_footer { height: fit-content; display: none; } #txt2img_generate_box, #img2img_generate_box { gap: 0.5em; flex-wrap: wrap-reverse; } #txt2img_actions_column, #img2img_actions_column { gap: 0.5em; } -#txt2img_generate_box > button, #img2img_generate_box > button { height: 2.2em; line-height: 0; } +#txt2img_generate_box > button, #img2img_generate_box > button { min-height: 36px; max-height: 42px; } #txt2img_generate_line2, #img2img_generate_line2 { display: flex; } #txt2img_generate_line2 > button, #img2img_generate_line2 > button, #extras_generate_box > button { height: 2.2em; line-height: 0; min-width: unset; display: block !important; } #txt2img_tools > div, #img2img_tools > div { justify-content: space-around; margin-top: 0.5em; margin-bottom: 0em; } diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 9e7ce49d8..75b930529 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -16,6 +16,16 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro shared.state.sampling_steps = p.steps shared.state.current_latent = latents + def vae_decode(latents, model): + if hasattr(model, 'vae'): + shared.log.debug(f'Diffusers VAE decode: name={model.vae.config.get("_name_or_path", "default")} upcast={model.vae.config.get("force_upcast", None)}') + decoded = model.vae.decode(latents / model.vae.config.scaling_factor, return_dict=False)[0] + images = model.image_processor.postprocess(decoded, output_type='np') + return images + else: + return latents + + def set_pipeline_args(model, prompt, negative_prompt, **kwargs): args = {} pipeline = model.main if model.__class__.__name__ == 'PriorPipeline' else model @@ -83,14 +93,21 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro model=shared.sd_model, prompt=prompts, negative_prompt=negative_prompts, + prompt_2=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else prompts, + negative_prompt_2=[p.refiner_negative] if len(p.refiner_negative) > 0 else negative_prompts, eta=shared.opts.eta_ddim, guidance_rescale=p.diffusers_guidance_rescale, + denoising_start=p.refiner_denoise_start, + denoising_end=p.refiner_denoise_end, # aesthetic_score=shared.opts.diffusers_aesthetics_score, - output_type='np' if (shared.sd_refiner is None or p.enable_hr is False or not shared.opts.diffusers_refiner_latents) else 'latent', + output_type='latent' if hasattr(shared.sd_model, 'vae') else 'np', **task_specific_kwargs ) output = shared.sd_model(**pipe_args) # pylint: disable=not-callable + if shared.sd_refiner is None or not p.enable_hr: + output.images = vae_decode(output.images, shared.sd_model) + if shared.sd_refiner is not None and p.enable_hr: if shared.opts.diffusers_move_base: shared.log.debug('Moving base model to CPU') @@ -128,9 +145,10 @@ def process_diffusers(p: StableDiffusionProcessing, seeds, prompts, negative_pro denoising_start=p.refiner_denoise_start, denoising_end=p.refiner_denoise_end, image=output.images[i], - output_type='np', + output_type='latent' if hasattr(shared.sd_model, 'vae') else 'np', ) output = shared.sd_refiner(**pipe_args) # pylint: disable=not-callable + output.images = vae_decode(output.images, shared.sd_model) results.append(output.images[0]) if shared.opts.diffusers_move_refiner: diff --git a/modules/sd_models.py b/modules/sd_models.py index 5dde9427a..dd208ad03 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -604,7 +604,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No if (model_data.sd_refiner is not None) and (checkpoint_info is not None) and (checkpoint_info.hash == model_data.sd_refiner.sd_checkpoint_info.hash): # trying to load the same model return - shared.log.debug(f'Diffusers load config: {diffusers_load_config}') + shared.log.debug(f'Diffusers load {op} config: {diffusers_load_config}') sd_model = None try: @@ -629,7 +629,8 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No return shared.log.info(f'Loading diffuser {op}: {checkpoint_info.filename}') - if op == 'model': + vae = None + if op == 'model' or op == 'refiner': vae_file, vae_source = sd_vae.resolve_vae(checkpoint_info.filename) vae = sd_vae.load_vae_diffusers(None, vae_file, vae_source) if vae is not None: @@ -639,7 +640,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No try: sd_model = diffusers.DiffusionPipeline.from_pretrained(checkpoint_info.path, **diffusers_load_config) except Exception as e: - shared.log.error(f'Diffusers failed loading model: {checkpoint_info.path} {e}') + shared.log.error(f'Diffusers {op} failed loading model: {checkpoint_info.path} {e}') else: diffusers_load_config["local_files_only "] = True diffusers_load_config["extract_ema"] = shared.opts.diffusers_extract_ema @@ -669,9 +670,9 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No elif shared.opts.diffusers_pipeline == shared.pipelines[11]: pipeline = diffusers.ShapEImg2ImgPipeline else: - shared.log.error(f'Diffusers unknown pipeline: {shared.opts.diffusers_pipeline}') + shared.log.error(f'Diffusers {op} unknown pipeline: {shared.opts.diffusers_pipeline}') except Exception as e: - shared.log.error(f'Diffusers failed initializing pipeline: {shared.opts.diffusers_pipeline} {e}') + shared.log.error(f'Diffusers {op} failed initializing pipeline: {shared.opts.diffusers_pipeline} {e}') return try: if hasattr(pipeline, 'from_single_file'): @@ -680,10 +681,10 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No elif hasattr(pipeline, 'from_ckpt'): sd_model = pipeline.from_ckpt(checkpoint_info.path, **diffusers_load_config) else: - shared.log.error(f'Diffusers cannot load safetensor model: {checkpoint_info.path} {shared.opts.diffusers_pipeline}') + shared.log.error(f'Diffusers {op} cannot load safetensor model: {checkpoint_info.path} {shared.opts.diffusers_pipeline}') return if sd_model is not None: - shared.log.debug(f'Diffusers pipeline: {sd_model.__class__.__name__}') # pylint: disable=protected-access + shared.log.debug(f'Diffusers {op}: pipeline={sd_model.__class__.__name__}') # pylint: disable=protected-access except Exception as e: shared.log.error(f'Diffusers failed loading model using pipeline: {checkpoint_info.path} {shared.opts.diffusers_pipeline} {e}') return @@ -705,34 +706,40 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No if hasattr(sd_model, "enable_model_cpu_offload"): if shared.cmd_opts.medvram or shared.opts.diffusers_model_cpu_offload: - shared.log.debug('Diffusers: enable model CPU offload') + shared.log.debug(f'Diffusers {op}: enable model CPU offload') sd_model.enable_model_cpu_offload() if hasattr(sd_model, "enable_sequential_cpu_offload"): if shared.opts.diffusers_seq_cpu_offload: sd_model.enable_sequential_cpu_offload() - shared.log.debug('Diffusers: enable sequential CPU offload') + shared.log.debug(f'Diffusers {op}: enable sequential CPU offload') if hasattr(sd_model, "enable_vae_slicing"): if shared.cmd_opts.lowvram or shared.opts.diffusers_vae_slicing: - shared.log.debug('Diffusers: enable VAE slicing') + shared.log.debug(f'Diffusers {op}: enable VAE slicing') sd_model.enable_vae_slicing() else: sd_model.disable_vae_slicing() if hasattr(sd_model, "enable_vae_tiling"): if shared.cmd_opts.lowvram or shared.opts.diffusers_vae_tiling: - shared.log.debug('Diffusers: enable VAE tiling') + shared.log.debug(f'Diffusers {op}: enable VAE tiling') sd_model.enable_vae_tiling() else: sd_model.disable_vae_tiling() if hasattr(sd_model, "enable_attention_slicing"): if shared.cmd_opts.lowvram or shared.opts.diffusers_attention_slicing: - shared.log.debug('Diffusers: enable attention slicing') + shared.log.debug(f'Diffusers {op}: enable attention slicing') sd_model.enable_attention_slicing() else: sd_model.disable_attention_slicing() + if hasattr(sd_model, "vae"): + if vae is not None: + shared.log.debug(f'Diffusers {op} VAE: name={vae.config.get("_name_or_path", "default")} upcast={vae.config.get("force_upcast", None)}') + sd_model.vae = vae # pylint: disable=attribute-defined-outside-init + if shared.opts.diffusers_vae_upcast != 'default': + sd_model.vae.config.force_upcast = True if shared.opts.upcast_sampling == 'true' else False if shared.opts.cross_attention_optimization == "xFormers" and hasattr(sd_model, 'enable_xformers_memory_efficient_attention'): sd_model.enable_xformers_memory_efficient_attention() if shared.opts.opt_channelslast: - shared.log.debug('Diffusers: enable channels last') + shared.log.debug(f'Diffusers {op}: enable channels last') sd_model.unet.to(memory_format=torch.channels_last) base_sent_to_cpu=False diff --git a/modules/sd_vae.py b/modules/sd_vae.py index b56a029ea..c6381b847 100644 --- a/modules/sd_vae.py +++ b/modules/sd_vae.py @@ -177,6 +177,7 @@ def load_vae_diffusers(_model, vae_file=None, vae_source="from unknown source"): try: import diffusers vae = diffusers.AutoencoderKL.from_pretrained(vae_file, **diffusers_load_config) + # shared.log.debug(f'Diffusers VAE config: {vae.config}') return vae except Exception as e: shared.log.error(f"Loading diffusers VAE failed: {vae_file} {e}") diff --git a/modules/shared.py b/modules/shared.py index 6a1d9b4ab..98b3a3a82 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -406,6 +406,7 @@ options_templates.update(options_section(('diffusers', "Diffusers Settings"), { "diffusers_generator_device": OptionInfo("default", "Generator device", gr.Radio, lambda: {"choices": ["default", "cpu"]}), "diffusers_seq_cpu_offload": OptionInfo(False, "Enable sequential CPU offload"), "diffusers_model_cpu_offload": OptionInfo(False, "Enable model CPU offload"), + "diffusers_vae_upcast": OptionInfo("default", "VAE upcasting", gr.Radio, lambda: {"choices": ['default', 'true', 'false']}), "diffusers_vae_slicing": OptionInfo(True, "Enable VAE slicing"), "diffusers_vae_tiling": OptionInfo(False, "Enable VAE tiling"), "diffusers_attention_slicing": OptionInfo(False, "Enable attention slicing"), diff --git a/modules/ui.py b/modules/ui.py index 876998147..bb2c4f300 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -399,7 +399,7 @@ def create_ui(startup_timer = None): with FormRow(elem_id="txt2img_refiner_row1", variant="compact"): image_cfg_scale = gr.Slider(minimum=1.1, maximum=30.0, step=0.1, label='Secondary CFG Scale', value=6.0, elem_id="txt2img_image_cfg_scale") diffusers_guidance_rescale = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Guidance rescale', value=0.7, elem_id="txt2img_image_cfg_rescale") - refiner_denoise_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Denoise start', value=0.0, elem_id="txt2img_refiner_denoise_start") + refiner_denoise_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Denoise start', value=0.8, elem_id="txt2img_refiner_denoise_start") refiner_denoise_end = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Denoise end', value=1.0, elem_id="txt2img_refiner_denoise_end") with FormRow(elem_id="txt2img_refiner_row2", variant="compact"): refiner_prompt = gr.Textbox(value='', label='Prompt') diff --git a/modules/ui_common.py b/modules/ui_common.py index 8e642d9f1..eeae5bfb3 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -40,7 +40,7 @@ def infotext_to_html(text): return res -def delete_files(js_data, images, _html_info, _do_make_zip, index): +def delete_files(js_data, images, _html_info, index): try: data = json.loads(js_data) except Exception: diff --git a/wiki b/wiki index f6877509d..dafa622e4 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit f6877509db1421db1991eccbc6af125ee7cecad3 +Subproject commit dafa622e449f67af3c561c50bf1a51220cd7bb43