diff --git a/cli/README.md b/cli/README.md index a09176230..e4c612e92 100644 --- a/cli/README.md +++ b/cli/README.md @@ -39,66 +39,100 @@ Combined pipeline: ### Benchmark -Benchmark your **Automatic WebUI** +Benchmark your **Automatic WebUI** +Note: Requires SD API + > python modules/bench.py ### Embedding Previews -Create previews of embeddings using preview templates +Create previews of embeddings using preview templates +Note: Requires SD API + > python modules/embedding-preview.py ## Grid -Create flexible image grids from any number of images +Create flexible image grids from any number of images +Note: Offline tool + > python modiles/grid.py ### Image Watermark -Create invisible image watermark and remove existing EXIF tags +Create invisible image watermark and remove existing EXIF tags +Note: Offline tool + > python modules/image-watermark.py + ### Interrogate -Runs CLiP and Booru image interrogation +Runs CLiP and Booru image interrogation +Note: Requires SD API + +> python modules/interrogate.py + +### Multi-Interrogate + +Standalone implementation of GiT, CLiP and ViT image interrogation +Note: Offline tool + > python modules/interrogate.py ### Models Previews -Create previews of models using built-in templates +Create previews of models using built-in templates +Note: Requires SD API + > python modules/models-preview.py ### Palette Extract -Extract color palette from image(s) +Extract color palette from image(s) +Note: Offline tool + > python modules/palette-extract.py ### Image Process -Run image processing to extract face/body segments and run resolution/blur/dynamic-range checks +Run image processing to extract face/body segments and run resolution/blur/dynamic-range checks +Note: Offline except for interrogate to generate caption files which requires SD API + > python modules/process.py ### Prompt Ideas Generate complex prompt ideas +Note: Offline tool + > python modules/prompt-ideas.py ### Prompt Promptist Attempts to beautify the provided prompt +Note: Offline tool + > python modules/promptist.py ### Training Loss-Chart Create loss-chart from training log +Note: Offline tool, may require adjustment to train paths if used with other repos + > python modules/train-losschart.py ### Training Loss-Rate Create customizable loss rate to be used in training +Note: Offline tool + > python modules/train-lossrate.py ### Video Extract -Extract frames from video files +Extract frames from video files +Note: Offline tool + > python modules/video-extract.py
@@ -107,6 +141,8 @@ Extract frames from video files ### SDAPI Utility module that handles async communication to Automatic API endpoints +Note: Requires SD API + Can be used to manually execute specific commands: > python sdapi.py progress > python sdapi.py interrupt diff --git a/cli/modules/interrogate_git.py b/cli/modules/interrogate_git.py deleted file mode 100755 index e81e50baf..000000000 --- a/cli/modules/interrogate_git.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/env python - -import os -import json -import argparse -import torch -import filetype -from PIL import Image -from transformers import AutoProcessor, AutoModelForCausalLM -from util import log, Map - - -git_processor = None -git_model = None -device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') - - -options = Map({ - 'input': '', - 'model': 'microsoft/git-large-textcaps', - 'length': 256, - 'json': '', - 'txt': False, - 'tag': '', -}) - -def cleanup(s: str): - s = s.split('"')[0].split('.')[0].split(' that')[0] - s = s.split(' with a letter')[0].split(' with the number')[0].split(' with the word')[0] - return s.replace('a ', '') - - -def load_model(args): - global git_processor - global git_model - if git_processor is None: - git_processor = AutoProcessor.from_pretrained(args.model) - if git_model is None: - git_model = AutoModelForCausalLM.from_pretrained(args.model) - git_model.to(device) - log.info( { 'interrogate loaded model': args.model }) - - -def interrogate_files(params, files): - args = Map({**options, **params}) - data = [f for f in files if filetype.is_image(f)] - log.info({ 'interrogate files': len(files), 'images': len(data), 'args': args }) - load_model(args) - metadata = {} - for image_path in data: - image = Image.open(image_path) - inputs = git_processor(images=[image], return_tensors="pt").to(device) - generated_ids = git_model.generate(pixel_values=inputs.pixel_values, max_length=args.length) - caption = git_processor.batch_decode(generated_ids, skip_special_tokens=True)[0] - caption = cleanup(caption) - tags = '' - if args.tag != '': - tags += args.tag + ',' - tags += caption.split(' ')[0] - if args.txt: - with open(os.path.splitext(image_path)[0] + '.txt', "wt", encoding='utf-8') as f: - f.write(caption + "\n") - metadata[image_path] = { 'caption': caption, 'tags': tags } - log.info({ 'interrogate image': image_path, 'caption': caption, 'tags': tags }) - - git_model.to('cpu') - if args.json != '': - with open(args.json, "wt", encoding='utf-8') as f: - f.write(json.dumps(metadata, indent=2) + "\n") - return metadata - - -def unload_git(): - global git_processor - global git_model - del git_processor - del git_model - git_processor = None - git_model = None - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description = 'image interrogate') - parser.add_argument('input', type=str, nargs='*', help='input file or directory') - parser.add_argument("--model", type=str, default="microsoft/git-large-textcaps", help="model id for GIT in HuggingFace") - parser.add_argument("--length", type=int, default=256, help="max length of caption") - parser.add_argument("--json", type=str, default='', help="output json file") - parser.add_argument("--tag", type=str, default='', help="append tag") - parser.add_argument('--txt', default = False, action='store_true', help = "write captions to text files") - params = parser.parse_args() - log.info({ 'interrogate args': vars(params) }) - if len(params.input) == 0: - parser.print_help() - exit(1) - files = [] - for loc in params.input: - if os.path.isfile(loc): - files.append(loc) - elif os.path.isdir(loc): - for root, _sub_dirs, dir in os.walk(loc): - files = [os.path.join(root, f) for f in dir] - metadata = interrogate_files(vars(params), files) diff --git a/cli/modules/multiinterrogate.py b/cli/modules/multiinterrogate.py new file mode 100755 index 000000000..ad6178255 --- /dev/null +++ b/cli/modules/multiinterrogate.py @@ -0,0 +1,165 @@ +#!/bin/env python + +import os +import gc +import json +import time +import argparse +import torch +import filetype +from PIL import Image +import transformers +from transformers import AutoProcessor, AutoModelForCausalLM +from transformers import BlipProcessor, BlipForConditionalGeneration +from transformers import VisionEncoderDecoderModel, ViTFeatureExtractor, AutoTokenizer +from util import log, Map + + +model = None +processor = None +extractor = None +dtype = torch.float32 +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + +options = Map({ + 'input': '', + 'min': 8, + 'max': 256, + 'beams': 1, + 'json': '', + 'txt': False, + 'tag': '', + 'git': True, + 'blip': True, + 'precision': 'fp16' +}) + + +def cleanup(s: str): + s = s.split('"')[0].split('.')[0].split(' that')[0] + s = s.split(' with a letter')[0].split(' with the number')[0].split(' with the word')[0] + s = s.replace('arafed image of ', '') + return s.replace('a ', '') + + +def load_model(args): + global model + global processor + global extractor + transformers.logging.set_verbosity_error() + if args.model == 'git': + model_name = "microsoft/git-large-textcaps" + if model is None: + model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=dtype) + model.to(device) + processor = AutoProcessor.from_pretrained(model_name, torch_dtype=dtype) + log.info( { 'interrogate loaded model': model_name }) + elif args.model == 'blip': + model_name = "Salesforce/blip-image-captioning-large" + if model is None: + model = BlipForConditionalGeneration.from_pretrained(model_name, torch_dtype=dtype) + model.to(device) + processor = BlipProcessor.from_pretrained(model_name, torch_dtype=dtype) + log.info( { 'interrogate loaded model': model_name }) + elif args.model == 'vit': + model_name = "nlpconnect/vit-gpt2-image-captioning" + if model is None: + model = VisionEncoderDecoderModel.from_pretrained(model_name, torch_dtype=dtype) + model.to(device) + extractor = ViTFeatureExtractor.from_pretrained(model_name, torch_dtype=dtype) + processor = AutoTokenizer.from_pretrained(model_name, torch_dtype=dtype) + log.info( { 'interrogate loaded model': model_name }) + else: + log.info( { 'interrogate unknown model': args.model }) + + +def interrogate_files(params, files): + args = Map({**options, **params}) + data = [f for f in files if filetype.is_image(f)] + log.info({ 'interrogate files': len(files), 'images': len(data), 'args': args }) + load_model(args) + metadata = {} + for image_path in data: + image = Image.open(image_path).convert('RGB') + caption = '' + if args.model == 'git': + inputs = processor(images=[image], return_tensors="pt").to(device) + ids = model.generate(pixel_values=inputs.pixel_values, num_beams=args.beams, min_length=args.min, max_length=args.max) + caption = processor.batch_decode(ids, skip_special_tokens=True)[0] + elif args.model == 'blip': + inputs = processor(image, return_tensors="pt").to(device, dtype) + ids = model.generate(**inputs, num_beams=args.beams, min_length=args.min, max_length=args.max) + caption = processor.decode(ids[0], skip_special_tokens=True) + elif args.model == 'vit': + inputs = extractor(images=[image], return_tensors="pt").pixel_values.to(device) + ids = model.generate(inputs, num_beams=args.beams, min_length=args.min, max_length=args.max) + caption = processor.batch_decode(ids, skip_special_tokens=True)[0] + else: + log.error({ 'interrogate unknown model': args.model }) + + caption = cleanup(caption) + tags = '' + if args.tag != '': + tags += args.tag + ',' + tags += caption.split(' ')[0] + if args.txt: + with open(os.path.splitext(image_path)[0] + '.txt', "wt", encoding='utf-8') as f: + f.write(caption + "\n") + metadata[image_path] = { 'caption': caption, 'tags': tags } + log.info({ 'interrogate image': image_path, 'moodel': args.model, 'caption': caption, 'tags': tags }) + + if args.json != '': + with open(args.json, "wt", encoding='utf-8') as f: + f.write(json.dumps(metadata, indent=2) + "\n") + return metadata + + +def unload_model(): + global processor + global model + global extractor + if model is not None: + del model + model = None + if processor is not None: + del processor + processor = None + if extractor is not None: + del extractor + extractor = None + gc.collect() + if torch.cuda.is_available(): + with torch.no_grad(): + torch.cuda.empty_cache() + with torch.cuda.device('cuda'): + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description = 'image interrogate') + parser.add_argument('input', type=str, nargs='*', help='input file or directory') + parser.add_argument('--model', default = 'git', choices = ['git', 'blip', 'vit'], help = "which model to use") + parser.add_argument("--min", type=int, default=8, help="min length of caption") + parser.add_argument("--max", type=int, default=256, help="max length of caption") + parser.add_argument("--beams", type=int, default=1, help="number of beams to use") + parser.add_argument("--json", type=str, default='', help="output json file") + parser.add_argument("--tag", type=str, default='', help="append tag") + parser.add_argument('--txt', default = False, action='store_true', help = "write captions to text files") + params = parser.parse_args() + log.info({ 'interrogate args': vars(params) }) + if len(params.input) == 0: + parser.print_help() + exit(1) + files = [] + for loc in params.input: + if os.path.isfile(loc): + files.append(loc) + elif os.path.isdir(loc): + for root, _sub_dirs, dir in os.walk(loc): + files = [os.path.join(root, f) for f in dir] + t0 = time.time() + metadata = interrogate_files(vars(params), files) + t1 = time.time() + log.info({ 'interrogate files': len(files), 'time': round(t1 - t0, 2) }) + unload_model() diff --git a/cli/modules/train-lora.py b/cli/modules/train-lora.py index 7ad7ef660..8709c0e94 100755 --- a/cli/modules/train-lora.py +++ b/cli/modules/train-lora.py @@ -11,13 +11,6 @@ Disabled/broken: - `xformers` due to *faketensors* requirement - `mem_eff_attn` due to *forwardfunc* mismatch - 'use_8bit_adam` due to *bitsandbyttes* CUDA errors - -Example: -train-lora.py --name=ryan-faf-v0 --model=/mnt/d/Models/stable-diffusion/sd-v15-runwayml.ckpt --dir=/mnt/d/Models/lora --input=~/generative/Input/ryanreid/fuckafan --dim 4 --steps 4000 -train-lora.py --name=ryan-palmsprings-v0 --model=/mnt/d/Models/stable-diffusion/sd-v15-runwayml.ckpt --dir=/mnt/d/Models/lora --input=~/generative/Input/ryanreid/palmsprings --dim 16 --steps 6000 -train-lora.py --name=ryan-random-v0 --model=/mnt/d/Models/stable-diffusion/sd-v15-runwayml.ckpt --dir=/mnt/d/Models/lora --input=~/generative/Input/ryanreid/random --dim 16 --steps 6000 -train-lora.py --name=ryan-miami-v0 --model=/mnt/d/Models/stable-diffusion/sd-v15-runwayml.ckpt --dir=/mnt/d/Models/lora --input=~/generative/Input/ryanreid/miami --dim 64 --steps 8000 -train-lora.py --name=ryan-all-v0 --model=/mnt/d/Models/stable-diffusion/sd-v15-runwayml.ckpt --dir=/mnt/d/Models/lora --input=~/generative/Input/ryanreid/all --dim 128 --steps 10000 """ import os @@ -30,9 +23,9 @@ import torch import transformers from pathlib import Path from util import log, Map, get_memory -from process import process_file, unload_models -from interrogate_git import interrogate_files, unload_git -from lora_latents import create_vae_latents, unload_vae +import process +import multiinterrogate +import lora_latents sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'modules', 'lora')) from train_network import train @@ -177,8 +170,8 @@ if __name__ == '__main__': for root, _sub_dirs, folder in os.walk(args.input): files = [os.path.join(root, f) for f in folder] for f in files: - res, metadata = process_file(f = f, dst = dir, preview = False, offline = True) - unload_models() + res, metadata = process.process_file(f = f, dst = dir, preview = False, offline = True) + process.unload_models() options.train_data_dir = os.path.join(tempfile.gettempdir(), args.name) mem_stats() @@ -186,11 +179,11 @@ if __name__ == '__main__': # interrogate for root, _sub_dirs, folder in os.walk(dir): files = [os.path.join(root, f) for f in folder] - metadata = interrogate_files(Map({ 'input': dir, 'json': '', 'tag': args.name }), files) + metadata = multiinterrogate.interrogate_files(Map({ 'input': dir, 'json': '', 'tag': args.name }), files) json_file = os.path.join(dir, args.name + '.json') with open(json_file, "w") as outfile: outfile.write(json.dumps(metadata, indent=2)) - unload_git() + multiinterrogate.unload_model() mem_stats() options.in_json = json_file @@ -198,7 +191,8 @@ if __name__ == '__main__': if not args.nolatents: # create latents - create_vae_latents(Map({ 'input': dir, 'json': json_file })) + lora_latents.create_vae_latents(Map({ 'input': dir, 'json': json_file })) + lora_latents.unload_vae() mem_stats() train(options) diff --git a/config.json b/config.json index 2241a15eb..fc4a8fb8d 100644 --- a/config.json +++ b/config.json @@ -45,14 +45,6 @@ "grid_prevent_empty_spots": true, "grid_save_to_dirs": false, "grid_save": true, - "images_copy_image": false, - "images_delete_message": false, - "images_delete_recycle": false, - "images_history_page_columns": 6.0, - "images_history_page_rows": 20.0, - "images_history_pages_perload": 20.0, - "images_history_preload": false, - "images_history_with_subdirs": false, "images_record_paths": true, "img2img_background_color": "#ffffff", "img2img_color_correction": false, @@ -128,7 +120,7 @@ "save_training_settings_to_txt": true, "save_txt": false, "sd_checkpoint_cache": 0, - "sd_checkpoint_hash": "9dc4131ec86df35efc444e0d0f128800f787a9547eedb75afb1fc8fa20df1cbb", + "sd_checkpoint_hash": "cc6cb27103417325ff94f52b7a5d2dde45a7515b25c255d8e396c90014281516", "sd_hypernetwork_strength": 1.0, "sd_hypernetwork": "None", "sd_lora": "", @@ -180,13 +172,21 @@ "DPM2 a Karras", "LMS Karras" ], - "images_logger_warning": false, - "images_logger_debug": false, - "images_scan_exif": false, "additional_networks_extra_lora_path": "", "additional_networks_sort_models_by": "name", "additional_networks_model_name_filter": "", "additional_networks_xy_grid_model_metadata": "", "additional_networks_hash_thread_count": 1.0, - "images_txt_files": false + "image_browser_with_subdirs": false, + "image_browser_preload": false, + "image_browser_copy_image": false, + "image_browser_delete_message": false, + "image_browser_txt_files": false, + "image_browser_logger_warning": false, + "image_browser_logger_debug": false, + "image_browser_delete_recycle": false, + "image_browser_scan_exif": false, + "image_browser_page_columns": 6.0, + "image_browser_page_rows": 20.0, + "image_browser_pages_perload": 20.0 } \ No newline at end of file diff --git a/extensions-builtin/sd-dynamic-thresholding b/extensions-builtin/sd-dynamic-thresholding index 0e324249e..61260cad8 160000 --- a/extensions-builtin/sd-dynamic-thresholding +++ b/extensions-builtin/sd-dynamic-thresholding @@ -1 +1 @@ -Subproject commit 0e324249e0c867b42bcb9c0897e26814ad1c7512 +Subproject commit 61260cad8f14842aad463938a74fb9d30b6b3bb5 diff --git a/extensions-builtin/stable-diffusion-webui-images-browser b/extensions-builtin/stable-diffusion-webui-images-browser index aafc81fef..04dc47f19 160000 --- a/extensions-builtin/stable-diffusion-webui-images-browser +++ b/extensions-builtin/stable-diffusion-webui-images-browser @@ -1 +1 @@ -Subproject commit aafc81fef837e53f7b89db4281415cc63f567ca7 +Subproject commit 04dc47f1954f22c8837bedb0698a7d49f27b1fdc diff --git a/modules/lora b/modules/lora index 6b790bace..53d60543e 160000 --- a/modules/lora +++ b/modules/lora @@ -1 +1 @@ -Subproject commit 6b790bace69c1deb7ed110c2ea119765de30090d +Subproject commit 53d60543e59d6fdf3a6b5d0d15019487d2e415db diff --git a/requirements_versions.txt b/requirements_versions.txt index 88f4fcadd..112d8bfe1 100644 --- a/requirements_versions.txt +++ b/requirements_versions.txt @@ -8,7 +8,7 @@ GitPython==3.1.27 gradio==3.16.2 inflection==0.5.1 jsonmerge==1.9.0 -kornia==0.6.7 +kornia==0.6.9 lark==1.1.5 numpy==1.23.5 omegaconf==2.3.0 diff --git a/ui-config.json b/ui-config.json index 2a524464e..e48226aad 100644 --- a/ui-config.json +++ b/ui-config.json @@ -1120,5 +1120,15 @@ "customscript/xyz_grid.py/txt2img/Z type/value": "Nothing", "customscript/xyz_grid.py/txt2img/Z type/visible": true, "customscript/xyz_grid.py/txt2img/Z values/value": "", - "customscript/xyz_grid.py/txt2img/Z values/visible": true + "customscript/xyz_grid.py/txt2img/Z values/visible": true, + "customscript/dynamic_thresholding.py/txt2img/Power Scheduler Value/visible": true, + "customscript/dynamic_thresholding.py/txt2img/Power Scheduler Value/value": 4.0, + "customscript/dynamic_thresholding.py/txt2img/Power Scheduler Value/minimum": 0.0, + "customscript/dynamic_thresholding.py/txt2img/Power Scheduler Value/maximum": 15.0, + "customscript/dynamic_thresholding.py/txt2img/Power Scheduler Value/step": 0.5, + "customscript/dynamic_thresholding.py/img2img/Power Scheduler Value/visible": true, + "customscript/dynamic_thresholding.py/img2img/Power Scheduler Value/value": 4.0, + "customscript/dynamic_thresholding.py/img2img/Power Scheduler Value/minimum": 0.0, + "customscript/dynamic_thresholding.py/img2img/Power Scheduler Value/maximum": 15.0, + "customscript/dynamic_thresholding.py/img2img/Power Scheduler Value/step": 0.5 } \ No newline at end of file