add clip blip and vit interrogate

cli/README.md

@@ -39,66 +39,100 @@ Combined pipeline:
 
 ### Benchmark
 
-Benchmark your **Automatic WebUI**
+Benchmark your **Automatic WebUI**  
+Note: Requires SD API  
+
 > python modules/bench.py
 
 ### Embedding Previews
 
-Create previews of embeddings using preview templates
+Create previews of embeddings using preview templates  
+Note: Requires SD API  
+
 > python modules/embedding-preview.py
 
 ## Grid
 
-Create flexible image grids from any number of images
+Create flexible image grids from any number of images  
+Note: Offline tool  
+
 > python modiles/grid.py
 
 ### Image Watermark
 
-Create invisible image watermark and remove existing EXIF tags
+Create invisible image watermark and remove existing EXIF tags  
+Note: Offline tool  
+
 > python modules/image-watermark.py
+
 ### Interrogate
 
-Runs CLiP and Booru image interrogation
+Runs CLiP and Booru image interrogation  
+Note: Requires SD API  
+
+> python modules/interrogate.py
+
+### Multi-Interrogate
+
+Standalone implementation of GiT, CLiP and ViT image interrogation  
+Note: Offline tool  
+
 > python modules/interrogate.py
 
 ### Models Previews
 
-Create previews of models using built-in templates
+Create previews of models using built-in templates  
+Note: Requires SD API  
+
 > python modules/models-preview.py
 
 ### Palette Extract
 
-Extract color palette from image(s)
+Extract color palette from image(s)  
+Note: Offline tool  
+
 > python modules/palette-extract.py
 
 ### Image Process
 
-Run image processing to extract face/body segments and run resolution/blur/dynamic-range checks
+Run image processing to extract face/body segments and run resolution/blur/dynamic-range checks  
+Note: Offline except for interrogate to generate caption files which requires SD API  
+
 > python modules/process.py
 
 ### Prompt Ideas
 
 Generate complex prompt ideas
+Note: Offline tool  
+
 > python modules/prompt-ideas.py
 
 ### Prompt Promptist
 
 Attempts to beautify the provided prompt  
+Note: Offline tool  
+
 > python modules/promptist.py
 
 ### Training Loss-Chart
 
 Create loss-chart from training log
+Note: Offline tool, may require adjustment to train paths if used with other repos  
+
 > python modules/train-losschart.py
 
 ### Training Loss-Rate
 
 Create customizable loss rate to be used in training
+Note: Offline tool  
+
 > python modules/train-lossrate.py
 
 ### Video Extract
 
-Extract frames from video files
+Extract frames from video files  
+Note: Offline tool  
+
 > python modules/video-extract.py
 
 <br>
@@ -107,6 +141,8 @@ Extract frames from video files
 ### SDAPI
 
 Utility module that handles async communication to Automatic API endpoints  
+Note: Requires SD API  
+
 Can be used to manually execute specific commands:
 > python sdapi.py progress  
 > python sdapi.py interrupt

cli/modules/interrogate_git.py

@@ -1,102 +0,0 @@
-#!/bin/env python
-
-import os
-import json
-import argparse
-import torch
-import filetype
-from PIL import Image
-from transformers import AutoProcessor, AutoModelForCausalLM
-from util import log, Map
-
-
-git_processor = None
-git_model = None
-device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-
-
-options = Map({
-    'input': '',
-    'model': 'microsoft/git-large-textcaps',
-    'length': 256,
-    'json': '',
-    'txt': False,
-    'tag': '',
-})
-
-def cleanup(s: str):
-    s = s.split('"')[0].split('.')[0].split(' that')[0]
-    s = s.split(' with a letter')[0].split(' with the number')[0].split(' with the word')[0]
-    return s.replace('a ', '')
-
-
-def load_model(args):
-    global git_processor
-    global git_model
-    if git_processor is None:
-        git_processor = AutoProcessor.from_pretrained(args.model)
-    if git_model is None:
-        git_model = AutoModelForCausalLM.from_pretrained(args.model)
-    git_model.to(device)
-    log.info( { 'interrogate loaded model': args.model })
-
-
-def interrogate_files(params, files):
-    args = Map({**options, **params})
-    data = [f for f in files if filetype.is_image(f)]
-    log.info({ 'interrogate files': len(files), 'images': len(data), 'args': args })
-    load_model(args)
-    metadata = {}
-    for image_path in data:
-        image = Image.open(image_path)
-        inputs = git_processor(images=[image], return_tensors="pt").to(device)
-        generated_ids = git_model.generate(pixel_values=inputs.pixel_values, max_length=args.length)
-        caption = git_processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
-        caption = cleanup(caption)
-        tags = ''
-        if args.tag != '':
-            tags += args.tag + ','
-        tags += caption.split(' ')[0]
-        if args.txt:
-            with open(os.path.splitext(image_path)[0] + '.txt', "wt", encoding='utf-8') as f:
-                f.write(caption + "\n")
-        metadata[image_path] = { 'caption': caption, 'tags': tags }
-        log.info({ 'interrogate image': image_path, 'caption': caption, 'tags': tags })
-
-    git_model.to('cpu')
-    if args.json != '':
-        with open(args.json, "wt", encoding='utf-8') as f:
-            f.write(json.dumps(metadata, indent=2) + "\n")
-    return metadata
-
-
-def unload_git():
-    global git_processor
-    global git_model
-    del git_processor
-    del git_model
-    git_processor = None
-    git_model = None
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description = 'image interrogate')
-    parser.add_argument('input', type=str, nargs='*', help='input file or directory')
-    parser.add_argument("--model", type=str, default="microsoft/git-large-textcaps", help="model id for GIT in HuggingFace")
-    parser.add_argument("--length", type=int, default=256, help="max length of caption")
-    parser.add_argument("--json", type=str, default='', help="output json file")
-    parser.add_argument("--tag", type=str, default='', help="append tag")
-    parser.add_argument('--txt', default = False, action='store_true', help = "write captions to text files")
-    params = parser.parse_args()
-    log.info({ 'interrogate args': vars(params) })
-    if len(params.input) == 0:
-        parser.print_help()
-        exit(1)
-    files = []
-    for loc in params.input:
-        if os.path.isfile(loc):
-            files.append(loc)
-        elif os.path.isdir(loc):
-            for root, _sub_dirs, dir in os.walk(loc):
-                files = [os.path.join(root, f) for f in dir]
-    metadata = interrogate_files(vars(params), files)

cli/modules/multiinterrogate.py

@@ -0,0 +1,165 @@
+#!/bin/env python
+
+import os
+import gc
+import json
+import time
+import argparse
+import torch
+import filetype
+from PIL import Image
+import transformers
+from transformers import AutoProcessor, AutoModelForCausalLM
+from transformers import BlipProcessor, BlipForConditionalGeneration
+from transformers import VisionEncoderDecoderModel, ViTFeatureExtractor, AutoTokenizer
+from util import log, Map
+
+
+model = None
+processor = None
+extractor = None
+dtype = torch.float32
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+options = Map({
+    'input': '',
+    'min': 8,
+    'max': 256,
+    'beams': 1,
+    'json': '',
+    'txt': False,
+    'tag': '',
+    'git': True,
+    'blip': True,
+    'precision': 'fp16'
+})
+
+
+def cleanup(s: str):
+    s = s.split('"')[0].split('.')[0].split(' that')[0]
+    s = s.split(' with a letter')[0].split(' with the number')[0].split(' with the word')[0]
+    s = s.replace('arafed image of ', '')
+    return s.replace('a ', '')
+
+
+def load_model(args):
+    global model
+    global processor
+    global extractor
+    transformers.logging.set_verbosity_error()
+    if args.model == 'git':
+        model_name = "microsoft/git-large-textcaps"
+        if model is None:
+            model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=dtype)
+            model.to(device)
+            processor = AutoProcessor.from_pretrained(model_name, torch_dtype=dtype)
+            log.info( { 'interrogate loaded model': model_name })
+    elif args.model == 'blip':
+        model_name = "Salesforce/blip-image-captioning-large"
+        if model is None:
+            model = BlipForConditionalGeneration.from_pretrained(model_name, torch_dtype=dtype)
+            model.to(device)
+            processor = BlipProcessor.from_pretrained(model_name, torch_dtype=dtype)
+            log.info( { 'interrogate loaded model': model_name })
+    elif args.model == 'vit':      
+        model_name = "nlpconnect/vit-gpt2-image-captioning"
+        if model is None:
+            model = VisionEncoderDecoderModel.from_pretrained(model_name, torch_dtype=dtype)
+            model.to(device)
+            extractor = ViTFeatureExtractor.from_pretrained(model_name, torch_dtype=dtype)
+            processor = AutoTokenizer.from_pretrained(model_name, torch_dtype=dtype)
+            log.info( { 'interrogate loaded model': model_name })
+    else:
+        log.info( { 'interrogate unknown model': args.model })
+
+
+def interrogate_files(params, files):
+    args = Map({**options, **params})
+    data = [f for f in files if filetype.is_image(f)]
+    log.info({ 'interrogate files': len(files), 'images': len(data), 'args': args })
+    load_model(args)
+    metadata = {}
+    for image_path in data:
+        image = Image.open(image_path).convert('RGB')
+        caption = ''
+        if args.model == 'git':
+            inputs = processor(images=[image], return_tensors="pt").to(device)
+            ids = model.generate(pixel_values=inputs.pixel_values, num_beams=args.beams, min_length=args.min, max_length=args.max)
+            caption = processor.batch_decode(ids, skip_special_tokens=True)[0]
+        elif args.model == 'blip':
+            inputs = processor(image, return_tensors="pt").to(device, dtype)
+            ids = model.generate(**inputs, num_beams=args.beams, min_length=args.min, max_length=args.max)
+            caption = processor.decode(ids[0], skip_special_tokens=True)
+        elif args.model == 'vit':
+            inputs = extractor(images=[image], return_tensors="pt").pixel_values.to(device)
+            ids = model.generate(inputs, num_beams=args.beams, min_length=args.min, max_length=args.max)
+            caption = processor.batch_decode(ids, skip_special_tokens=True)[0]
+        else:
+            log.error({ 'interrogate unknown model': args.model })
+
+        caption = cleanup(caption)
+        tags = ''
+        if args.tag != '':
+            tags += args.tag + ','
+        tags += caption.split(' ')[0]
+        if args.txt:
+            with open(os.path.splitext(image_path)[0] + '.txt', "wt", encoding='utf-8') as f:
+                f.write(caption + "\n")
+        metadata[image_path] = { 'caption': caption, 'tags': tags }
+        log.info({ 'interrogate image': image_path, 'moodel': args.model, 'caption': caption, 'tags': tags })
+
+    if args.json != '':
+        with open(args.json, "wt", encoding='utf-8') as f:
+            f.write(json.dumps(metadata, indent=2) + "\n")
+    return metadata
+
+
+def unload_model():
+    global processor
+    global model
+    global extractor
+    if model is not None:
+        del model
+        model = None
+    if processor is not None:
+        del processor
+        processor = None
+    if extractor is not None:
+        del extractor
+        extractor = None
+    gc.collect()
+    if torch.cuda.is_available():
+        with torch.no_grad():
+            torch.cuda.empty_cache()
+        with torch.cuda.device('cuda'):
+            torch.cuda.empty_cache()
+            torch.cuda.ipc_collect()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description = 'image interrogate')
+    parser.add_argument('input', type=str, nargs='*', help='input file or directory')
+    parser.add_argument('--model', default = 'git', choices = ['git', 'blip', 'vit'], help = "which model to use")
+    parser.add_argument("--min", type=int, default=8, help="min length of caption")
+    parser.add_argument("--max", type=int, default=256, help="max length of caption")
+    parser.add_argument("--beams", type=int, default=1, help="number of beams to use")
+    parser.add_argument("--json", type=str, default='', help="output json file")
+    parser.add_argument("--tag", type=str, default='', help="append tag")
+    parser.add_argument('--txt', default = False, action='store_true', help = "write captions to text files")
+    params = parser.parse_args()
+    log.info({ 'interrogate args': vars(params) })
+    if len(params.input) == 0:
+        parser.print_help()
+        exit(1)
+    files = []
+    for loc in params.input:
+        if os.path.isfile(loc):
+            files.append(loc)
+        elif os.path.isdir(loc):
+            for root, _sub_dirs, dir in os.walk(loc):
+                files = [os.path.join(root, f) for f in dir]
+    t0 = time.time()
+    metadata = interrogate_files(vars(params), files)
+    t1 = time.time()
+    log.info({ 'interrogate files': len(files), 'time': round(t1 - t0, 2) })
+    unload_model()

cli/modules/train-lora.py

@@ -11,13 +11,6 @@ Disabled/broken:
 - `xformers` due to *faketensors* requirement
 - `mem_eff_attn` due to *forwardfunc* mismatch
 - 'use_8bit_adam` due to *bitsandbyttes* CUDA errors
-
-Example:
-train-lora.py --name=ryan-faf-v0 --model=/mnt/d/Models/stable-diffusion/sd-v15-runwayml.ckpt --dir=/mnt/d/Models/lora --input=~/generative/Input/ryanreid/fuckafan --dim 4 --steps 4000
-train-lora.py --name=ryan-palmsprings-v0 --model=/mnt/d/Models/stable-diffusion/sd-v15-runwayml.ckpt --dir=/mnt/d/Models/lora --input=~/generative/Input/ryanreid/palmsprings --dim 16 --steps 6000
-train-lora.py --name=ryan-random-v0 --model=/mnt/d/Models/stable-diffusion/sd-v15-runwayml.ckpt --dir=/mnt/d/Models/lora --input=~/generative/Input/ryanreid/random --dim 16 --steps 6000
-train-lora.py --name=ryan-miami-v0 --model=/mnt/d/Models/stable-diffusion/sd-v15-runwayml.ckpt --dir=/mnt/d/Models/lora --input=~/generative/Input/ryanreid/miami --dim 64 --steps 8000
-train-lora.py --name=ryan-all-v0 --model=/mnt/d/Models/stable-diffusion/sd-v15-runwayml.ckpt --dir=/mnt/d/Models/lora --input=~/generative/Input/ryanreid/all --dim 128 --steps 10000
 """
 
 import os
@@ -30,9 +23,9 @@ import torch
 import transformers
 from pathlib import Path
 from util import log, Map, get_memory
-from process import process_file, unload_models
-from interrogate_git import interrogate_files, unload_git
-from lora_latents import create_vae_latents, unload_vae
+import process
+import multiinterrogate
+import lora_latents
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'modules', 'lora'))
 from train_network import train
@@ -177,8 +170,8 @@ if __name__ == '__main__':
         for root, _sub_dirs, folder in os.walk(args.input):
             files = [os.path.join(root, f) for f in folder]
         for f in files:
-            res, metadata = process_file(f = f, dst = dir, preview = False, offline = True)
-        unload_models()
+            res, metadata = process.process_file(f = f, dst = dir, preview = False, offline = True)
+        process.unload_models()
         options.train_data_dir = os.path.join(tempfile.gettempdir(), args.name)
         mem_stats()
 
@@ -186,11 +179,11 @@ if __name__ == '__main__':
         # interrogate
         for root, _sub_dirs, folder in os.walk(dir):
             files = [os.path.join(root, f) for f in folder]
-        metadata = interrogate_files(Map({ 'input': dir, 'json': '', 'tag': args.name }), files)
+        metadata = multiinterrogate.interrogate_files(Map({ 'input': dir, 'json': '', 'tag': args.name }), files)
         json_file = os.path.join(dir, args.name + '.json')
         with open(json_file, "w") as outfile:
             outfile.write(json.dumps(metadata, indent=2))
-        unload_git()
+        multiinterrogate.unload_model()
         mem_stats()
         options.in_json = json_file
 
@@ -198,7 +191,8 @@ if __name__ == '__main__':
 
     if not args.nolatents:
         # create latents
-        create_vae_latents(Map({ 'input': dir, 'json': json_file }))
+        lora_latents.create_vae_latents(Map({ 'input': dir, 'json': json_file }))
+        lora_latents.unload_vae()
         mem_stats()
 
     train(options)

config.json

@@ -45,14 +45,6 @@
     "grid_prevent_empty_spots": true,
     "grid_save_to_dirs": false,
     "grid_save": true,
-    "images_copy_image": false,
-    "images_delete_message": false,
-    "images_delete_recycle": false,
-    "images_history_page_columns": 6.0,
-    "images_history_page_rows": 20.0,
-    "images_history_pages_perload": 20.0,
-    "images_history_preload": false,
-    "images_history_with_subdirs": false,
     "images_record_paths": true,
     "img2img_background_color": "#ffffff",
     "img2img_color_correction": false,
@@ -128,7 +120,7 @@
     "save_training_settings_to_txt": true,
     "save_txt": false,
     "sd_checkpoint_cache": 0,
-    "sd_checkpoint_hash": "9dc4131ec86df35efc444e0d0f128800f787a9547eedb75afb1fc8fa20df1cbb",
+    "sd_checkpoint_hash": "cc6cb27103417325ff94f52b7a5d2dde45a7515b25c255d8e396c90014281516",
     "sd_hypernetwork_strength": 1.0,
     "sd_hypernetwork": "None",
     "sd_lora": "",
@@ -180,13 +172,21 @@
         "DPM2 a Karras",
         "LMS Karras"
     ],
-    "images_logger_warning": false,
-    "images_logger_debug": false,
-    "images_scan_exif": false,
     "additional_networks_extra_lora_path": "",
     "additional_networks_sort_models_by": "name",
     "additional_networks_model_name_filter": "",
     "additional_networks_xy_grid_model_metadata": "",
     "additional_networks_hash_thread_count": 1.0,
-    "images_txt_files": false
+    "image_browser_with_subdirs": false,
+    "image_browser_preload": false,
+    "image_browser_copy_image": false,
+    "image_browser_delete_message": false,
+    "image_browser_txt_files": false,
+    "image_browser_logger_warning": false,
+    "image_browser_logger_debug": false,
+    "image_browser_delete_recycle": false,
+    "image_browser_scan_exif": false,
+    "image_browser_page_columns": 6.0,
+    "image_browser_page_rows": 20.0,
+    "image_browser_pages_perload": 20.0
 }

requirements_versions.txt

@@ -8,7 +8,7 @@ GitPython==3.1.27
 gradio==3.16.2
 inflection==0.5.1
 jsonmerge==1.9.0
-kornia==0.6.7
+kornia==0.6.9
 lark==1.1.5
 numpy==1.23.5
 omegaconf==2.3.0

ui-config.json

@@ -1120,5 +1120,15 @@
     "customscript/xyz_grid.py/txt2img/Z type/value": "Nothing",
     "customscript/xyz_grid.py/txt2img/Z type/visible": true,
     "customscript/xyz_grid.py/txt2img/Z values/value": "",
-    "customscript/xyz_grid.py/txt2img/Z values/visible": true
+    "customscript/xyz_grid.py/txt2img/Z values/visible": true,
+    "customscript/dynamic_thresholding.py/txt2img/Power Scheduler Value/visible": true,
+    "customscript/dynamic_thresholding.py/txt2img/Power Scheduler Value/value": 4.0,
+    "customscript/dynamic_thresholding.py/txt2img/Power Scheduler Value/minimum": 0.0,
+    "customscript/dynamic_thresholding.py/txt2img/Power Scheduler Value/maximum": 15.0,
+    "customscript/dynamic_thresholding.py/txt2img/Power Scheduler Value/step": 0.5,
+    "customscript/dynamic_thresholding.py/img2img/Power Scheduler Value/visible": true,
+    "customscript/dynamic_thresholding.py/img2img/Power Scheduler Value/value": 4.0,
+    "customscript/dynamic_thresholding.py/img2img/Power Scheduler Value/minimum": 0.0,
+    "customscript/dynamic_thresholding.py/img2img/Power Scheduler Value/maximum": 15.0,
+    "customscript/dynamic_thresholding.py/img2img/Power Scheduler Value/step": 0.5
 }