remove duplicate diffusers and move aux models to hfcache

Signed-off-by: Vladimir Mandic <mandic00@live.com>

.gitignore

@@ -44,6 +44,11 @@ tunableop_results*.csv
 !webui.sh
 !package.json
 
+# pyinstaller
+*.spec
+build/
+dist/
+
 # dynamically generated
 /repositories/ip-instruct/
 
@@ -69,3 +74,4 @@ tunableop_results*.csv
 !/models/VAE-approx/model.pt
 !/models/Reference
 !/models/Reference/**/*
+

CHANGELOG.md

@@ -26,6 +26,7 @@
   - detailer min/max size as fractions of image size  
   - ipadapter load on-demand  
   - ipadapter face use correct yolo model  
+  - list diffusers remove duplicates  
   - fix legacy extensions access to shared objects  
   - fix diffusers load from folder  
 

modules/face/faceid.py

@@ -79,7 +79,7 @@ def face_id(
             ip_ckpt = FACEID_MODELS[model]
             folder, filename = os.path.split(ip_ckpt)
             basename, _ext = os.path.splitext(filename)
-            model_path = hf.hf_hub_download(repo_id=folder, filename=filename, cache_dir=shared.opts.diffusers_dir)
+            model_path = hf.hf_hub_download(repo_id=folder, filename=filename, cache_dir=shared.opts.hfcache_dir)
             if model_path is None:
                 shared.log.error(f'FaceID download failed: model={model} file="{ip_ckpt}"')
                 return None

modules/face/insightface.py

@@ -32,7 +32,7 @@ def get_app(mp_name):
                 repo_id='vladmandic/insightface-faceanalysis',
                 filename=f'{mp_name}.zip',
                 local_dir_use_symlinks=False,
-                cache_dir=opts.diffusers_dir,
+                cache_dir=opts.hfcache_dir,
                 local_dir=local_dir
             )
         if not os.path.exists(extract_dir):

modules/model_sd3.py

@@ -142,7 +142,7 @@ def load_sd3(checkpoint_info, cache_dir=None, config=None):
     else:
         kwargs['variant'] = 'fp16'
 
-    shared.log.debug(f'Load model: type=SD3 kwargs={list(kwargs)}')
+    shared.log.debug(f'Load model: type=SD3 kwargs={list(kwargs)} repo="{repo_id}"')
 
     kwargs = model_quant.create_bnb_config(kwargs)
     pipe = loader(

modules/modelloader.py

@@ -273,6 +273,7 @@ def load_diffusers_models(clear=True):
         place = os.path.join(models_path, 'Diffusers')
     if clear:
         diffuser_repos.clear()
+    already_found = []
     try:
         for folder in os.listdir(place):
             try:
@@ -303,7 +304,11 @@ def load_diffusers_models(clear=True):
                     if (not os.path.exists(index)) and (not os.path.exists(info)) and (not os.path.exists(config)):
                         debug(f'Diffusers skip model no info: {name}')
                         continue
+                    if name in already_found:
+                        debug(f'Diffusers skip model already found: {name}')
+                        continue
                     repo = { 'name': name, 'filename': name, 'friendly': friendly, 'folder': folder, 'path': commit, 'hash': snapshot, 'mtime': mtime, 'model_info': info, 'model_index': index, 'model_config': config }
+                    already_found.append(name)
                     diffuser_repos.append(repo)
                     if os.path.exists(os.path.join(folder, 'hidden')):
                         continue

scripts/prompt_enhance.py

@@ -32,10 +32,10 @@ class Script(scripts.Script):
 
     def load(self):
         if self.tokenizer is None:
-            self.tokenizer = AutoTokenizer.from_pretrained('gokaygokay/Flux-Prompt-Enhance', cache_dir=shared.opts.diffusers_dir)
+            self.tokenizer = AutoTokenizer.from_pretrained('gokaygokay/Flux-Prompt-Enhance', cache_dir=shared.opts.hfcache_dir)
         if self.model is None:
             shared.log.info(f'Prompt enhance: model="{repo_id}"')
-            self.model = AutoModelForSeq2SeqLM.from_pretrained('gokaygokay/Flux-Prompt-Enhance', cache_dir=shared.opts.diffusers_dir).to(device=devices.cpu, dtype=devices.dtype)
+            self.model = AutoModelForSeq2SeqLM.from_pretrained('gokaygokay/Flux-Prompt-Enhance', cache_dir=shared.opts.hfcache_dir).to(device=devices.cpu, dtype=devices.dtype)
 
     def enhance(self, prompt, auto_apply: bool = False, temperature: float = 0.7, repetition_penalty: float = 1.2, max_length: int = 128):
         self.load()