From b216a35ddd79c02cddfebd80ab999d3a553225bb Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 4 Jul 2023 09:28:48 -0400 Subject: [PATCH] update diffusers and extra networks --- DIFFUSERS.md | 35 +++++++++++++++++++++++++++++------ html/locale_en.json | 6 +++--- javascript/extraNetworks.js | 10 ++++++---- javascript/style.css | 2 +- modules/devices.py | 1 + modules/paths.py | 1 + modules/processing.py | 6 +++--- modules/sd_models.py | 37 +++++++++++++++++++++++++++++++------ modules/shared.py | 9 +++++++++ modules/ui_models.py | 9 ++++----- scripts/xyz_grid.py | 6 +++--- 11 files changed, 91 insertions(+), 31 deletions(-) diff --git a/DIFFUSERS.md b/DIFFUSERS.md index a0ade1913..89b029b2a 100644 --- a/DIFFUSERS.md +++ b/DIFFUSERS.md @@ -26,6 +26,7 @@ so diffusers code can be merged into `master` and we can continue with developme whats implemented so far? +- new scheduler: deis - simple model downloader for huggingface models: tabs -> models -> hf hub - use huggingface models - extra networks ui @@ -58,6 +59,7 @@ whats implemented so far? ## Todo - sdxl model +- new schedulers ## Limitations @@ -73,25 +75,46 @@ will need to handle in the code before we get out of alpha ## Issues -- seed vs batch size? +- default model download ckpt vs hfhub? +- new dependency hell (not diffuser related)? +- extra networks ui auto-hide and transitions ## Notes for HF - removed `quicksettings` alternative completely -- added simple model downloader in ui: *tabs -> models -> huggingface* +- added simple model downloader in ui: *tabs -> models -> huggingface* +- attempting to download gated model without access token results in model/refs/commits not found instead of access denied + this is not very user friendly, it should be handled in the code - redone **textual inversion** support, core is now in `modules/textual_inversion/textual_inversion.py:load_diffusers_embedding()` the point is that sdnext pre-loads all compatible embeddings on model load so they are available in prompt context +- redone **lora** support, core is now in `modules/lora_diffusers.py` - added support for diffuser models in **safetensors/ckpt** format - btw, when i use: `diffusers.StableDiffusionPipeline.from_ckpt` +- when i use `diffusers.StableDiffusionPipeline.from_ckpt` first time it downloads something - what is that? > Downloading (…)lve/main/config.json: 4.55k > Downloading pytorch_model.bin: 1.22G - and in general, loading safetensors model is quite slow, is that expected? - for example, 2sec vs 18sec +- loading safetensors model is very slow + for example, 2sec without diffusers and 16sec with diffusers - in `modules/modelloader.py:download_diffusers_model()` i get unknown property for `hf.model_info(hub_id).cardData` can you double-check if this is linter issue or actual problem? -- redone **lora** support, core is now in `modules/lora_diffusers.py` - question on `pipe.load_lora_weights` does it support loading multiple loras? i don't see any notes on that in docs also, lora strength is specified using `cross_attention_kwargs={"scale": x}` during pipeline execution which means if there are multiple loras, they all have the same strength? +- **deepfloyd** failures: + > /home/disty/Apps/automatic/venv/lib/python3.10/site-packages/diffusers/configuration_utils.py:138 in __getattr__ + > AttributeError: 'DDPMScheduler' object has no attribute 'name +- question how do diffusers handle standard 75 token limit for sd? +- diffusers `convert_from_ckpt.py` uses fixed `print` statements so its not possible to control its output to console + it should use `logging` instead. in general, using `print` is bad idea + for example, it very annoyingly logs this every time `StableDiffusionPipeline.from_ckpt` is used: + > global_step key not found in model + > Checkpoint /home/vlado/dev/automatic/models/Stable-diffusion/best/absolutereality_v1.safetensors has both EMA and non-EMA weights. + > In this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag. + +## Update + +- sortable models table in downloader ui +- recommended scheduler: `deis` +- `channels_last` and `cudnn_benchmark` now apply to diffusers +- new settings section for diffusers fine-tuning diff --git a/html/locale_en.json b/html/locale_en.json index 3e69cd207..7f828998b 100644 --- a/html/locale_en.json +++ b/html/locale_en.json @@ -524,9 +524,9 @@ ], "scripts": [ {"id":"","label":"Script","localized":"","hint":""}, - {"id":"","label":"Swap X/Y axes","localized":"","hint":""}, - {"id":"","label":"Swap Y/Z axes","localized":"","hint":""}, - {"id":"","label":"Swap X/Z axes","localized":"","hint":""}, + {"id":"","label":"Swap X/Y","localized":"","hint":""}, + {"id":"","label":"Swap Y/Z","localized":"","hint":""}, + {"id":"","label":"Swap X/Z","localized":"","hint":""}, {"id":"","label":"Resize to","localized":"","hint":""}, {"id":"","label":"Resize by","localized":"","hint":""}, {"id":"","label":"Use via API","localized":"","hint":""}, diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index 39f75c91c..d4dfc243e 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -9,6 +9,7 @@ function setupExtraNetworksForTab(tabname) { const refresh = gradioApp().getElementById(`${tabname}_extra_refresh`); const description = gradioApp().getElementById(`${tabname}_description`); const close = gradioApp().getElementById(`${tabname}_extra_close`); + const en = gradioApp().getElementById(`${tabname}_extra_networks`); search.classList.add('search'); description.classList.add('description'); tabs.appendChild(refresh); @@ -23,16 +24,15 @@ function setupExtraNetworksForTab(tabname) { elem.style.display = text.indexOf(searchTerm) == -1 ? 'none' : ''; }); }); + intersectionObserver = new IntersectionObserver((entries) => { - // if (entries[0].intersectionRatio <= 0) onHidden(); - const en = gradioApp().getElementById(`${tabname}_extra_networks`); if (entries[0].intersectionRatio > 0) { for (el of Array.from(gradioApp().querySelectorAll('.extra-network-cards'))) { const rect = el.getBoundingClientRect(); - en.style.transition = 'width 0.2s ease'; if (rect.top > 0) { if (!en) return if (window.opts.extra_networks_card_cover == 'cover') { + en.style.transition = ''; en.style.zIndex = 9999; en.style.position = 'absolute'; en.style.right = 'unset'; @@ -40,6 +40,7 @@ function setupExtraNetworksForTab(tabname) { el.style.height = document.body.offsetHeight - el.getBoundingClientRect().top + 'px'; gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = 'unset' } if (window.opts.extra_networks_card_cover == 'sidebar') { + en.style.transition = 'width 0.2s ease'; en.style.zIndex = 0; en.style.position = 'absolute'; en.style.right = '0'; @@ -47,6 +48,7 @@ function setupExtraNetworksForTab(tabname) { el.style.height = gradioApp().getElementById(`${tabname}_settings`).offsetHeight - 90 + 'px'; gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = 100 - 2 - window.opts.extra_networks_sidebar_width + 'vw'; } else { + en.style.transition = ''; en.style.zIndex = 0; en.style.position = 'relative'; en.style.right = 'unset'; @@ -61,7 +63,7 @@ function setupExtraNetworksForTab(tabname) { gradioApp().getElementById(`${tabname}_settings`).parentNode.style.width = 'unset' } }); - intersectionObserver.observe(search); // monitor visibility of + intersectionObserver.observe(en); // monitor visibility of } function setupExtraNetworks() { diff --git a/javascript/style.css b/javascript/style.css index eef8d75ee..2c72ca68e 100644 --- a/javascript/style.css +++ b/javascript/style.css @@ -542,7 +542,7 @@ table.settings-value-table td{ .extra-networks-page { display: flex } .extra-networks .custom-button { min-width: 80px; max-width: 240px; width: 100%; background: none; justify-content: left; text-align: left; padding: 2px 8px 2px 8px; box-shadow: none; } .extra-networks .custom-button:hover { background: var(--button-primary-background-fill) } -.extra-network-cards { display: flex; flex-wrap: wrap; height: 50vh; overflow-y: scroll; overflow-x: hidden; scroll-snap-type: y mandatory; width: -webkit-fill-available; } +.extra-network-cards { display: flex; flex-wrap: wrap; height: 50vh; overflow-y: scroll; overflow-x: hidden; width: -webkit-fill-available; } .extra-network-cards .card { height: fit-content; margin: 0.5em; position: relative; scroll-snap-align: start; scroll-margin-top: 0; } .extra-network-cards .card .overlay { position: absolute; bottom: 0; padding: 0.2em; z-index: 10; width: 100%; background: none; } .extra-network-cards .card:hover .overlay { background: rgba(0, 0, 0, 0.40); } diff --git a/modules/devices.py b/modules/devices.py index 64852d109..eedbb432b 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -122,6 +122,7 @@ def set_cuda_params(): try: torch.backends.cudnn.benchmark = True if shared.opts.cudnn_benchmark: + shared.log.debug('Torch enable cuDNN benchmark') torch.backends.cudnn.benchmark_limit = 0 torch.backends.cudnn.allow_tf32 = shared.opts.cuda_allow_tf32 except Exception: diff --git a/modules/paths.py b/modules/paths.py index deb022c75..966349580 100644 --- a/modules/paths.py +++ b/modules/paths.py @@ -76,6 +76,7 @@ def create_paths(opts): create_path(fix_path('hypernetwork_dir')) create_path(fix_path('ckpt_dir')) create_path(fix_path('vae_dir')) + create_path(fix_path('diffusers_dir')) create_path(fix_path('embeddings_dir')) create_path(fix_path('outdir_samples')) create_path(fix_path('outdir_txt2img_samples')) diff --git a/modules/processing.py b/modules/processing.py index 68c18c977..f50e7482b 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -689,14 +689,14 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: p.scripts.postprocess_batch(p, x_samples_ddim, batch_number=n) elif backend == Backend.DIFFUSERS: - generator = [torch.Generator(device="cpu").manual_seed(s) for s in seeds] + generator_device = 'cpu' if shared.opts.diffusers_generator_device == "cpu" else shared.device + generator = [torch.Generator(generator_device).manual_seed(s) for s in seeds] if shared.sd_model.scheduler.name != p.sampler_name: sampler = sd_samplers.all_samplers_map.get(p.sampler_name, None) if sampler is None: sampler = sd_samplers.all_samplers_map.get("UniPC") scheduler = sampler.constructor(shared.sd_model.sd_checkpoint_info.filename) - # TODO(Patrick): For wrapped pipelines this is currently a no-op - shared.sd_model.scheduler = scheduler.sampler + shared.sd_model.scheduler = scheduler.sampler # TODO(Patrick): For wrapped pipelines this is currently a no-op cross_attention_kwargs={} if lora_state['active']: diff --git a/modules/sd_models.py b/modules/sd_models.py index 2181b09d7..186664c95 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -223,7 +223,8 @@ def select_checkpoint(model=True): checkpoint_info = next(iter(checkpoints_list.values())) if model_checkpoint is not None: shared.log.warning(f"Selected checkpoint not found: {model_checkpoint}") - shared.log.warning(f"Loading fallback checkpoint: {checkpoint_info.title}") + # shared.log.warning(f"Loading fallback checkpoint: {checkpoint_info.title}") + shared.opts.data['sd_checkpoint'] = checkpoint_info.title shared.log.debug(f'Select checkpoint: {checkpoint_info.title if checkpoint_info is not None else None}') return checkpoint_info @@ -579,15 +580,39 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No prior = diffusers.DiffusionPipeline.from_pretrained(prior_id, **diffusers_load_config) sd_model = PriorPipeline(prior=prior, main=sd_model) # wrap sd_model - if shared.cmd_opts.medvram: - sd_model.enable_model_cpu_offload() - if shared.cmd_opts.lowvram: - sd_model.enable_sequential_cpu_offload() + if hasattr(sd_model, "enable_sequential_cpu_offload"): + if shared.cmd_opts.lowvram or shared.opts.diffusers_seq_cpu_offload: + sd_model.enable_sequential_cpu_offload() + shared.log.debug('Diffusers: enable sequenctial CPU offload') + if hasattr(sd_model, "enable_model_cpu_offload"): + if shared.cmd_opts.medvram or shared.opts.diffusers_model_cpu_offload: + shared.log.debug('Diffusers: enable model CPU offload') + sd_model.enable_model_cpu_offload() + if hasattr(sd_model, "enable_vae_slicing"): + if shared.opts.diffusers_vae_slicing: + shared.log.debug('Diffusers: enable VAE slicing') + sd_model.enable_vae_slicing() + else: + sd_model.disable_vae_slicing() + if hasattr(sd_model, "enable_vae_tiling"): + if shared.opts.diffusers_vae_tiling: + shared.log.debug('Diffusers: enable VAE tiling') + sd_model.enable_vae_tiling() + else: + sd_model.disable_vae_tiling() + if hasattr(sd_model, "enable_attention_slicing"): + if shared.opts.diffusers_attention_slicing: + shared.log.debug('Diffusers: enable attention slicing') + sd_model.enable_attention_slicing() + else: + sd_model.disable_attention_slicing() if shared.opts.cross_attention_optimization == "xFormers": sd_model.enable_xformers_memory_efficient_attention() + if shared.opts.opt_channelslast: + shared.log.debug('Diffusers: enable channels last') + sd_model.unet.to(memory_format=torch.channels_last) if shared.opts.cuda_compile and torch.cuda.is_available(): sd_model.to(devices.device) - sd_model.unet.to(memory_format=torch.channels_last) import torch._dynamo as dynamo # pylint: disable=unused-import torch._dynamo.config.verbose = shared.opts.cuda_compile_verbose # pylint: disable=protected-access torch._dynamo.config.suppress_errors = shared.opts.cuda_compile_errors # pylint: disable=protected-access diff --git a/modules/shared.py b/modules/shared.py index e3a0a7704..301295723 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -342,6 +342,15 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "disable_gc": OptionInfo(False, "Disable Torch memory garbage collection"), })) +options_templates.update(options_section(('diffusers', "Diffusers Settings"), { + "diffusers_generator_device": OptionInfo("default", "Generator device", gr.Radio, lambda: {"choices": ["default", "cpu"]}), + "diffusers_seq_cpu_offload": OptionInfo(False, "Enable sequential CPU offload"), + "diffusers_model_cpu_offload": OptionInfo(False, "Enable model CPU offload"), + "diffusers_vae_slicing": OptionInfo(False, "Enable VAE slicing"), + "diffusers_vae_tiling": OptionInfo(False, "Enable VAE tiling"), + "diffusers_attention_slicing": OptionInfo(False, "Enable attention slicing"), +})) + options_templates.update(options_section(('system-paths', "System Paths"), { "temp_dir": OptionInfo("", "Directory for temporary images; leave empty for default"), "clean_temp_dir_at_start": OptionInfo(True, "Cleanup non-default temporary directory when starting webui"), diff --git a/modules/ui_models.py b/modules/ui_models.py index b7c078a96..325eca125 100644 --- a/modules/ui_models.py +++ b/modules/ui_models.py @@ -184,7 +184,7 @@ def create_ui(): data.append([model.modelId, model.pipeline_tag, tags, model.downloads, model.lastModified, f'https://huggingface.co/{model.modelId}']) return data - def hf_select(evt: gr.SelectData): + def hf_select(evt: gr.SelectData, data): return data[evt.index[0]][0] def hf_download_model(hub_id: str, token): @@ -212,13 +212,12 @@ def create_ui(): with gr.Row(): hf_headers = ['Name', 'Pipeline', 'Tags', 'Downloads', 'Updated', 'URL'] - hf_results = gr.DataFrame([], label = 'Search results', show_label = True, interactive = False, wrap = True, overflow_row_behaviour = 'paginate', max_rows = 10, headers = hf_headers, type='array') + hf_types = ['str', 'str', 'str', 'number', 'date', 'markdown'] + hf_results = gr.DataFrame([], label = 'Search results', show_label = True, interactive = False, wrap = True, overflow_row_behaviour = 'paginate', max_rows = 10, headers = hf_headers, datatype = hf_types, type='array') hf_search_text.submit(fn=hf_search, inputs=[hf_search_text], outputs=[hf_results]) - hf_results.select(hf_select, inputs=None, outputs=[hf_selected]) + hf_results.select(fn=hf_select, inputs=[hf_results], outputs=[hf_selected]) hf_download_model_btn.click(fn=hf_download_model, inputs=[hf_selected, hf_token], outputs=[models_outcome]) - # TODO load_diffusers_lora - with gr.Tab(label="CivitAI"): pass diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py index 5ecb24134..11f716d00 100644 --- a/scripts/xyz_grid.py +++ b/scripts/xyz_grid.py @@ -410,9 +410,9 @@ class Script(scripts.Script): with gr.Row(variant="compact", elem_id="axis_options"): margin_size = gr.Slider(label="Grid margins", minimum=0, maximum=500, value=0, step=2, elem_id=self.elem_id("margin_size")) with gr.Row(variant="compact", elem_id="swap_axes"): - swap_xy_axes_button = gr.Button(value="Swap X/Y axes", elem_id="xy_grid_swap_axes_button") - swap_yz_axes_button = gr.Button(value="Swap Y/Z axes", elem_id="yz_grid_swap_axes_button") - swap_xz_axes_button = gr.Button(value="Swap X/Z axes", elem_id="xz_grid_swap_axes_button") + swap_xy_axes_button = gr.Button(value="Swap X/Y", elem_id="xy_grid_swap_axes_button", variant="secondary") + swap_yz_axes_button = gr.Button(value="Swap Y/Z", elem_id="yz_grid_swap_axes_button", variant="secondary") + swap_xz_axes_button = gr.Button(value="Swap X/Z", elem_id="xz_grid_swap_axes_button", variant="secondary") def swap_axes(axis1_type, axis1_values, axis1_values_dropdown, axis2_type, axis2_values, axis2_values_dropdown): return self.current_axis_options[axis2_type].label, axis2_values, axis2_values_dropdown, self.current_axis_options[axis1_type].label, axis1_values, axis1_values_dropdown