diff --git a/DIFFUSERS.md b/DIFFUSERS.md index 199623c4d..52616aa2d 100644 --- a/DIFFUSERS.md +++ b/DIFFUSERS.md @@ -23,27 +23,31 @@ to update repo, do not use `--upgrade` flag, use manual `git pull` instead ### Diffusers - sd 1.5 and sd 2.1 model - models can be downloaded from huggingface hub - but focus on default model for now and i'll add downloader soon -- lora, textual inversion - only loras/textual-inversions downloaded from huggingface hub are supported for now - i'll add standard safetensors soon +- model downloader: tabs -> models -> hf hub - txt2img, img2img, inpaint, outpaint, process +- hires fix, restore faces, etc? -### Experimental +### Experimental - don't test yet -- cuda model compile using `reduce overhead` model with and without `fullgraph` +- cuda model compile using `reduce overhead` model with or without `fullgraph` - kandinsky model ## Todo -- enable loading of safetensors models +- lora +- embedding +- safetensors models - cleanup logging -- search&download models from hfhub - controlnet extension - multidiffusion extension - sdxl model +## Limitations + +- extra networks +- controlnet +- multi-diffusion + ## Issues - TBD diff --git a/cli/hf-search.py b/cli/hf-search.py index bbb4a4f0f..ac97b6c26 100755 --- a/cli/hf-search.py +++ b/cli/hf-search.py @@ -11,9 +11,8 @@ if __name__ == "__main__": model_filter = hf.ModelFilter( model_name=keyword, task='text-to-image', - tags='stable-diffusion', - library=['diffusers', 'stable-diffusion'], + library=['diffusers'], ) res = hf_api.list_models(filter=model_filter, full=True, limit=50, sort="downloads", direction=-1) - models = [{ 'name': m.modelId, 'downloads': m.downloads, 'mtime': m.lastModified, 'url': f'https://huggingface.co/{m.modelId}' } for m in res] - print('Online', models) + models = [{ 'name': m.modelId, 'downloads': m.downloads, 'mtime': m.lastModified, 'url': f'https://huggingface.co/{m.modelId}', 'pipeline': m.pipeline_tag, 'tags': m.tags } for m in res] + print(models) diff --git a/modules/cmd_args.py b/modules/cmd_args.py index 3c0c2ee6c..7d85e014c 100644 --- a/modules/cmd_args.py +++ b/modules/cmd_args.py @@ -90,6 +90,8 @@ def compatibility_args(opts, args): group.add_argument("--sub-quad-chunk-threshold", help=argparse.SUPPRESS, default=opts.sub_quad_chunk_threshold) group.add_argument("--lora-dir", help=argparse.SUPPRESS, default=opts.lora_dir) group.add_argument("--lyco-dir", help=argparse.SUPPRESS, default=opts.lyco_dir) + group.add_argument("--embeddings-dir", help=argparse.SUPPRESS, default=opts.embeddings_dir) + group.add_argument("--hypernetwork-dir", help=argparse.SUPPRESS, default=opts.hypernetwork_dir) group.add_argument("--lyco-patch-lora", help=argparse.SUPPRESS, default=opts.lyco_patch_lora) group.add_argument("--lyco-debug", help=argparse.SUPPRESS, action='store_true', default=False) group.add_argument("--enable-console-prompts", help=argparse.SUPPRESS, action='store_true', default=False) diff --git a/modules/modelloader.py b/modules/modelloader.py index 799537240..758dc7e34 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -11,6 +11,7 @@ from modules.paths import script_path, models_path diffuser_repos = [] + def download_diffusers_model(hub_id: str, cache_dir: str = None, download_config: Dict[str, str] = None): from diffusers import DiffusionPipeline import huggingface_hub as hf @@ -41,6 +42,7 @@ def download_diffusers_model(hub_id: str, cache_dir: str = None, download_config return pipeline_dir + def load_diffusers_models(model_path: str, command_path: str = None): import huggingface_hub as hf places = [] @@ -74,10 +76,9 @@ def find_diffuser(name: str): filt = hf.ModelFilter( model_name=name, task='text-to-image', - tags='stable-diffusion', - library=['diffusers', 'stable-diffusion'], + library=['diffusers'], ) - models = list(api.list_models(filter=filt, full=True, limit=50, sort="downloads", direction=-1)) + models = list(api.list_models(filter=filt, full=True, limit=5, sort="downloads", direction=-1)) shared.log.debug(f'Searching diffusers models: {name} {len(models) > 0}') if len(models) > 0: return models[0].modelId diff --git a/modules/sd_models.py b/modules/sd_models.py index d237f8728..b72f0d7dd 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -14,9 +14,9 @@ import safetensors.torch from omegaconf import OmegaConf import tomesd from transformers import logging as transformers_logging -import diffusers import ldm.modules.midas as midas from ldm.util import instantiate_from_config +import diffusers from modules import paths, shared, modelloader, devices, script_callbacks, sd_vae, sd_disable_initialization, errors, hashes, sd_models_config from modules.sd_hijack_inpainting import do_inpainting_hijack from modules.timer import Timer diff --git a/modules/shared.py b/modules/shared.py index f9ce0a981..3b31df4d2 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -234,31 +234,15 @@ def list_checkpoint_tiles(): default_checkpoint = list_checkpoint_tiles()[0] if len(list_checkpoint_tiles()) > 0 else "model.ckpt" -def load_diffusers_ckpt(model_repo: str): - cached_dir = modelloader.download_diffusers_model(model_repo) - print(f"Downloaded {cached_dir}") - return "" def load_diffusers_lora(lora_repo: str): pipe = sys.modules[__name__].sd_model - if lora_repo == "": pipe._remove_text_encoder_monkey_patch() # pylint: disable=W0212 proc_cls_name = next(iter(pipe.unet.attn_processors.values())).__class__.__name__ non_lora_proc_cls = getattr(diffusers.models.attention_processor, proc_cls_name[len("LORA"):]) pipe.unet.set_attn_processor(non_lora_proc_cls()) - print("Removed LoRA.") return "" - elif is_url(lora_repo): - with tempfile.TemporaryDirectory() as temp_dir: - os.system(f"wget -P {temp_dir} {lora_repo}") - temp_file_path = os.path.join(temp_dir, lora_repo.split('/')[-1]) - pipe.load_lora_weights(temp_file_path) - - lora_repo = '/'.join(lora_repo.split('/')[-2:]) - - print(f"Loaded Civit.ai LoRA: {lora_repo}") - return f"{lora_repo} is loaded. Pass empty text field to remove LoRA or pass new LoRA id." elif len(lora_repo.split('/')) == 2: lora_dir = os.path.dirname(opts.data["diffusers_dir"]) cache_dir = os.path.join(lora_dir, "Diffusers_LoRA") @@ -269,34 +253,30 @@ def load_diffusers_lora(lora_repo: str): print(f"{lora_repo} is not a valid LoRA identifier.") return "" + def load_diffusers_text_inv(text_inv_repo: str): pipe = sys.modules[__name__].sd_model - if text_inv_repo == "": pipe.tokenizer = pipe.tokenizer.__class__.from_pretrained(pipe.tokenizer.name_or_path) pipe.text_encoder.resize_token_embeddings(len(pipe.tokenizer)) - print("Removed all textual inversions.") return "" elif is_url(text_inv_repo): with tempfile.TemporaryDirectory() as temp_dir: os.system(f"wget -P {temp_dir} {text_inv_repo}") temp_file_path = os.path.join(temp_dir, text_inv_repo.split('/')[-1]) pipe.load_textual_inversion(temp_file_path) - text_inv_repo = '/'.join(text_inv_repo.split('/')[-2:]) - print(f"Loaded Civit.ai Textual Inv: {text_inv_repo}") elif len(text_inv_repo.split('/')) == 2: text_inv_dir = os.path.dirname(opts.data["diffusers_dir"]) cache_dir = os.path.join(text_inv_dir, "Diffusers_Text_Inv") pipe.load_textual_inversion(text_inv_repo, cache_dir=cache_dir) print(f"Loaded {text_inv_repo}") - text_inv_tokens = pipe.tokenizer.added_tokens_encoder.keys() text_inv_tokens = [t for t in text_inv_tokens if not (len(t.split("_")) > 1 and t.split("_")[-1].isdigit())] - return f"{', '.join(text_inv_tokens)} loaded. Pass empty text field to remove all or add new textual inversion id." + def refresh_checkpoints(): import modules.sd_models # pylint: disable=W0621 return modules.sd_models.list_models() diff --git a/modules/ui_models.py b/modules/ui_models.py index afa6dc702..369a3baec 100644 --- a/modules/ui_models.py +++ b/modules/ui_models.py @@ -162,14 +162,62 @@ def create_ui(): model_list_btn.click(fn=list_models, inputs=[], outputs=[model_table, models_outcome]) - with gr.Tab(label="HF Hub"): - """" - options_templates.update(options_section(('diffusers', "Diffusers"), { - "diffusers_ckpt_download": OptionInfo("", "HFHub Checkpoint download", gr.Textbox, {"placeholder": "e.g. runwayml/stable-diffusion-v1-5"}, submit=load_diffusers_ckpt), - "diffusers_lora_download": OptionInfo("", "HFHub LoRA download", gr.Textbox, {"placeholder": "e.g. pcuenq/pokemon-lora"}, submit=load_diffusers_lora), - "diffusers_text_inv_download": OptionInfo("", "HFHub Textual Inversion download", gr.Textbox, {"placeholder": "e.g. sd-concepts-library/midjourney-style"}, submit=load_diffusers_text_inv), - })) - """ + with gr.Tab(label="Huggingface"): + data = [] + os.environ.setdefault('HF_HUB_DISABLE_EXPERIMENTAL_WARNING', '1') + os.environ.setdefault('HF_HUB_DISABLE_SYMLINKS_WARNING', '1') + os.environ.setdefault('HF_HUB_DISABLE_IMPLICIT_TOKEN', '1') + os.environ.setdefault('HUGGINGFACE_HUB_VERBOSITY', 'warning') + + def hf_search(keyword): + import huggingface_hub as hf + hf_api = hf.HfApi() + model_filter = hf.ModelFilter( + model_name=keyword, + task='text-to-image', + library=['diffusers'], + ) + models = hf_api.list_models(filter=model_filter, full=True, limit=50, sort="downloads", direction=-1) + data.clear() + for model in models: + tags = [t for t in model.tags if not t.startswith('diffusers') and not t.startswith('license') and not t.startswith('arxiv') and len(t) > 2] + data.append([model.modelId, model.pipeline_tag, tags, model.downloads, model.lastModified, f'https://huggingface.co/{model.modelId}']) + return data + + def hf_select(evt: gr.SelectData): + return data[evt.index[0]][0] + + def hf_download_model(hub_id: str): + from modules.shared import log, opts + from modules.modelloader import download_diffusers_model + try: + download_diffusers_model(hub_id, cache_dir=opts.diffusers_dir) + except Exception as e: + log.error(f"Diffuser model downloaded error: model={hub_id} {e}") + return f"Diffuser model downloaded error: model={hub_id} {e}" + from modules.sd_models import list_models # pylint: disable=W0621 + list_models() + log.info(f"Diffuser model downloaded: model={hub_id}") + return f'Diffuser model downloaded: model={hub_id}' + + with gr.Row(): + hf_search_text = gr.Textbox('', label = 'Seach models', placeholder='search huggingface models') + + with gr.Row(): + hf_selected = gr.Textbox('', label = 'Select model', placeholder='select model from search results or enter model name manually') + with gr.Row(): + hf_download_model_btn = gr.Button(value="Download model", variant='primary') + + with gr.Row(): + hf_headers = ['Name', 'Pipeline', 'Tags', 'Downloads', 'Updated', 'URL'] + hf_results = gr.DataFrame([], label = 'Search results', show_label = True, interactive = False, wrap = True, overflow_row_behaviour = 'paginate', max_rows = 10, headers = hf_headers, type='array') + + hf_search_text.submit(fn=hf_search, inputs=[hf_search_text], outputs=[hf_results]) + hf_results.select(hf_select, inputs=None, outputs=[hf_selected]) + hf_download_model_btn.click(fn=hf_download_model, inputs=[hf_selected], outputs=[models_outcome]) + + # TODO load_diffusers_lora + # TODO load_diffusers_text_inv with gr.Tab(label="CivitAI"): pass