update torch load and external repos

TODO.md

@@ -1,49 +1,21 @@
-# Publish extensions
+# TODO
 
-- `sd-extension-aesthetic-scorer`
-- `sd-extension-steps-animation`
-- `sd-extension-system-info`
-
-# Investigating
-
-Things I'm actively investigating...
-
-## Embeddings
-
-Need to study more to determine best out-of-the-box settings:
-- Train object
-- Train style
-- Impact of Hires fix:
-  - e.g 25 steps and denoising strength 0.25-0.7
-- Impact of non-square target resolution
-
-## Hypernetworks
-
-## LORA
-
-
-# Ideas
-
-Things I'm looking into...
-
-## Automatic
+## Fixes
 
 Stuff to be fixed...
 
 - Reconnect WebUI  
-- Settings params on updates install  
+
+## Integration
 
 Tech that can be integrated as part of the core workflow...
 
 - [Merge without distortion](https://github.com/ogkalu2/Merge-Stable-Diffusion-models-without-distortion)
 - [Weighted merges](https://github.com/bbc-mc/sdweb-merge-block-weighted-gui/tree/master)
-- [Use scripts from API](https://github.com/AUTOMATIC1111/stable-diffusion-webui/pull/6469)
-- [Face swap](https://github.com/kex0/batch-face-swap)
 - [Null-text inversion](https://github.com/ouhenio/null-text-inversion-colab)
 - [Custom diffusion](https://github.com/guaneec/custom-diffusion-webui)
   - <https://www.cs.cmu.edu/~custom-diffusion/>
 - [Dream artist](https://github.com/7eu7d7/DreamArtist-sd-webui-extension)
-- [LoCon](https://github.com/AUTOMATIC1111/stable-diffusion-webui/discussions/8169)
 
 ## Video Generation
 
@@ -53,17 +25,13 @@ Tech that can be integrated as part of the core workflow...
 - [Tune-a-Video](https://github.com/showlab/Tune-A-Video)
 - [Animator extension](https://github.com/Animator-Anon/animator_extension)
 - [Prompt travel](https://github.com/Kahsolt/stable-diffusion-webui-prompt-travel)
-- [KLMC2 animation](https://colab.research.google.com/github/dmarx/notebooks/blob/main/Stable_Diffusion_KLMC2_Animation.ipynb) 
+- [KLMC2 animation](https://colab.research.google.com/github/dmarx/notebooks/blob/main/Stable_Diffusion_KLMC2_Animation.ipynb)
 - [BOAAB-limit animation](https://colab.research.google.com/drive/17kesyBVqubV_Zzchf2XoR-7MHk5jxTuo?usp=sharing) <https://www.ajayjain.net/journey>
 - [Disco diffusion](https://colab.research.google.com/github/alembics/disco-diffusion/blob/main/Disco_Diffusion.ipynb)
 - [Video killed the radio star](https://colab.research.google.com/github/dmarx/video-killed-the-radio-star/blob/main/Video_Killed_The_Radio_Star_Defusion.ipynb)
 - [Seed travel](https://github.com/yownas/seed_travel)
 - [Prompt fusion](https://github.com/ljleb/prompt-fusion-extension)
 
-## Experimental
+## Random
 
-Cool stuff that is not integrated anywhere...
-
-- [TensorRT](https://www.photoroom.com/tech/stable-diffusion-25-percent-faster-and-save-seconds/)
-- [GIT](https://huggingface.co/microsoft/git-large-textcaps)
 - Bunch of stuff:<https://pharmapsychotic.com/tools.html>

automatic.sh

@@ -11,6 +11,7 @@ export CUDA_AUTO_BOOST=1
 export CUDA_MODULE_LOADING="LAZY"
 export CUDA_DEVICE_DEFAULT_PERSISTING_L2_CACHE_PERCENTAGE_LIMIT=0
 export GRADIO_ANALYTICS_ENABLED="False"
+export SAFETENSORS_FAST_GPU=1
 
 if [ "$PYTHON" == "" ]; then
   PYTHON=$(which python)

launch.py

@@ -222,8 +222,8 @@ def prepare_environment():
     codeformer_repo = os.environ.get('CODEFORMER_REPO', 'https://github.com/sczhou/CodeFormer.git')
     blip_repo = os.environ.get('BLIP_REPO', 'https://github.com/salesforce/BLIP.git')
 
-    stable_diffusion_commit_hash = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "47b6b607fdd31875c9279cd2f4f16b92e4ea958e")
-    taming_transformers_commit_hash = os.environ.get('TAMING_TRANSFORMERS_COMMIT_HASH', "24268930bf1dce879235a7fddd0b2355b84d7ea6")
+    stable_diffusion_commit_hash = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "fc1488421a2761937b9d54784194157882cbc3b1")
+    taming_transformers_commit_hash = os.environ.get('TAMING_TRANSFORMERS_COMMIT_HASH', "3ba01b241669f5ade541ce990f7650a3b8f65318")
     # k_diffusion_commit_hash = os.environ.get('K_DIFFUSION_COMMIT_HASH', "5b3af030dd83e0297272d861c19477735d0317ec")
     k_diffusion_commit_hash = os.environ.get('K_DIFFUSION_COMMIT_HASH', "b43db16749d51055f813255eea2fdf1def801919")
     codeformer_commit_hash = os.environ.get('CODEFORMER_COMMIT_HASH', "c5b4593074ba6214284d6acd5f1719b6c5d739af")

modules/sd_hijack.py

@@ -7,6 +7,7 @@ from modules import devices, sd_hijack_optimizations, shared, sd_hijack_checkpoi
 from modules.hypernetworks import hypernetwork
 from modules.shared import cmd_opts
 from modules import sd_hijack_clip, sd_hijack_open_clip, sd_hijack_unet, sd_hijack_xlmr, xlmr
+from rich import print
 
 import ldm.modules.attention
 import ldm.modules.diffusionmodules.model
@@ -169,6 +170,17 @@ class StableDiffusionModelHijack:
         if m.cond_stage_key == "edit":
             sd_hijack_unet.hijack_ddpm_edit()
 
+        """
+        try:
+            import torch._dynamo as dynamo
+            torch._dynamo.config.verbose = True
+            torch.backends.cudnn.benchmark = True
+            m.model = torch.compile(m.model, mode="default", backend="inductor", fullgraph=False, dynamic=False)
+            print("Model compiled set")
+        except Exception as err:
+            print(f"Model compile not supported: {err}")
+        """
+
         self.optimization_method = apply_optimizations()
 
         self.clip = m.cond_stage_model

modules/sd_models.py

@@ -9,6 +9,7 @@ from omegaconf import OmegaConf
 from os import mkdir
 from urllib import request
 import ldm.modules.midas as midas
+import io
 
 from ldm.util import instantiate_from_config
 
@@ -17,6 +18,9 @@ from modules.paths import models_path
 from modules.sd_hijack_inpainting import do_inpainting_hijack
 from modules.timer import Timer
 
+import rich
+from rich import print
+
 model_dir = "Stable-diffusion"
 model_path = os.path.abspath(os.path.join(paths.models_path, model_dir))
 
@@ -236,11 +240,15 @@ def read_metadata_from_safetensors(filename):
 
 def read_state_dict(checkpoint_file, print_global_state=False, map_location=None):
     _, extension = os.path.splitext(checkpoint_file)
+    device = map_location or shared.weight_load_location or devices.get_optimal_device_name()
     if extension.lower() == ".safetensors":
-        device = map_location or shared.weight_load_location or devices.get_optimal_device_name()
         pl_sd = safetensors.torch.load_file(checkpoint_file, device=device)
+    elif extension.lower() == ".ckpt":
+        with rich.progress.open(checkpoint_file, 'rb') as f:
+            buffer = io.BytesIO(f.read())
+            pl_sd = torch.load(buffer, map_location=device)
     else:
-        pl_sd = torch.load(checkpoint_file, map_location=map_location or shared.weight_load_location)
+        raise Exception(f"Unknown model type: {extension}")
 
     if print_global_state and "global_step" in pl_sd:
         print(f"Global Step: {pl_sd['global_step']}")
@@ -255,12 +263,12 @@ def get_checkpoint_state_dict(checkpoint_info: CheckpointInfo, timer):
 
     if checkpoint_info in checkpoints_loaded:
         # use checkpoint cache
-        print(f"Loading weights [{sd_model_hash}] from cache")
+        print(f"Loading weights from cache")
         return checkpoints_loaded[checkpoint_info]
 
-    print(f"Loading weights [{sd_model_hash}] from {checkpoint_info.filename}")
+    print(f"Loading weights from {checkpoint_info.filename}")
     res = read_state_dict(checkpoint_info.filename)
-    timer.record("load weights from disk")
+    timer.record("load weights")
 
     return res
 

modules/sd_vae.py

@@ -6,7 +6,7 @@ from collections import namedtuple
 from modules import paths, shared, devices, script_callbacks, sd_models
 import glob
 from copy import deepcopy
-
+from rich import print
 
 vae_path = os.path.abspath(os.path.join(paths.models_path, "VAE"))
 vae_ignore_keys = {"model_ema.decay", "model_ema.num_updates"}

modules/textual_inversion/textual_inversion.py

@@ -20,7 +20,7 @@ from modules.textual_inversion.learn_schedule import LearnRateScheduler
 
 from modules.textual_inversion.image_embedding import embedding_to_b64, embedding_from_b64, insert_image_data_embed, extract_image_data_embed, caption_image_overlay
 from modules.textual_inversion.logging import save_settings_to_file
-
+from rich import print
 
 TextualInversionTemplate = namedtuple("TextualInversionTemplate", ["name", "path"])
 textual_inversion_templates = {}

modules/ui.py

@@ -1443,7 +1443,7 @@ def create_ui():
             with gr.Column(scale=6):
                 settings_submit = gr.Button(value="Apply settings", variant='primary', elem_id="settings_submit")
             with gr.Column():
-                restart_gradio = gr.Button(value='Reload UI', variant='primary', elem_id="settings_restart_gradio", disabled=True)
+                restart_gradio = gr.Button(value='Reload UI', variant='primary', elem_id="settings_restart_gradio", visible=False)
 
         result = gr.HTML(elem_id="settings_result")
 

requirements_versions.txt

@@ -1,4 +1,5 @@
 accelerate==0.17.1
+clip_interrogator=0.6.0
 diffusers==0.14.0
 einops==0.4.1
 fastapi==0.94.0
@@ -13,4 +14,4 @@ protobuf==3.20.3
 pytorch_lightning==1.9.4
 realesrgan==0.3.0
 safetensors==0.3.0
-transformers==4.26.1
+transformers==4.27.1

webui.py

@@ -9,6 +9,7 @@ from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.middleware.gzip import GZipMiddleware
 from packaging import version
+from rich import print
 
 import logging
 logging.getLogger("xformers").addFilter(lambda record: 'A matching Triton is not available' not in record.getMessage())