From f756da1596cf311c199d9dc82b925ebfcfe26f76 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 7 Feb 2023 09:53:19 -0500 Subject: [PATCH] update preview generation --- TODO.md | 11 ++-- cli/README.md | 114 ++++++++++++++++++++++++---------- cli/modules/models-preview.py | 58 +++++++++++++++-- cli/train.md | 28 --------- 4 files changed, 140 insertions(+), 71 deletions(-) delete mode 100644 cli/train.md diff --git a/TODO.md b/TODO.md index bbc109645..c2c575045 100644 --- a/TODO.md +++ b/TODO.md @@ -90,16 +90,18 @@ Cool stuff that is not integrated anywhere... - non-trivial ui updates - renamed scripts in `cli/modules` to be more descriptive if you're using old script names, update them - for example, `ffmpeg.py` is now `video-extract.py` + for example, `ffmpeg.py` is now `video-extract.py` + also possible that there are some bugs due to broken import paths, so testing is welcome - updated script `process.py` - new **brightness dynamic range** check - new **preview** mode to run all checks but without saving images plus print a summary at the end +- updated scripts `models-preview.py` + can generate **lora** previews as well, note that trigger keywords are inferred from model name so name models carefully - new script: `image-watermark.py` - optionally strip exif from images - add invisible watermark to images which persists even if user modifies image so we can always track it - new script: `palette-extract.py` - - creates color palette wheel from image - - not finished + - creates color palette wheel from image(s) - updated `embedding-preview.py` so it can skip existing previews or overwrite them - expose variation seed in main ui - integrated seed travel functionality into core @@ -110,6 +112,7 @@ Cool stuff that is not integrated anywhere... - updated `image browser` was broken for a while and maintainer is gone - initial work on **queue management** allowing to submit multiple requests to server -- initial work on `lora` integration (hidden) +- initial work on `lora` integration + can render loras without extensions, training tbd - initial work on `custom diffusion` integration (hidden) - spent quite some time making stable-diffusion compatible with upcomming `pytorch` 2.0 release diff --git a/cli/README.md b/cli/README.md index 159013851..a09176230 100644 --- a/cli/README.md +++ b/cli/README.md @@ -1,23 +1,31 @@ -# Scripts using Stable-Diffusion/Automatic API +# Stable-Diffusion Productivity Scripts -*Note*: Start **SD/Automatic** using `python launch.py --api` -## Generate +*Notes*: +- Offline scripts can be used with or without **Automatic WebUI** +- Online scripts rely on **Automatic WebUI** API which should be started with `--api` parameter +- All scripts have built-in `--help` parameter that can be used to get more information + +
+ +## Main Scripts + +### Generate Text-to-image with all of the possible parameters Supports upsampling, face restoration and grid creation -> python generate.py --help +> python generate.py By default uses parameters from `generate.json` -Parameters that are not specified will be randomized to some extent: +Parameters that are not specified will be randomized: - Prompt will be dynamically created from template of random samples: `random.json` - Sampler/Scheduler will be randomly picked from available ones - CFG Scale set to 5-10 -## Train +### Train End-to-end embedding training -> python train.py --help +> python train.py Combined pipeline: 1. Creates embedding @@ -25,42 +33,80 @@ Combined pipeline: 3. Preprocesses images 4. Runs training -## Interrogate +
-Runs CLiP and Booru image interrogation on any provided parameters -*(image, list of images, wildcards, folder, etc.)* -> python interrogate.py +## Auxiliary Scripts -## Promptist +### Benchmark -Attempts to beautify the provided prompt -> python promptist.py +Benchmark your **Automatic WebUI** +> python modules/bench.py -## Ideas +### Embedding Previews + +Create previews of embeddings using preview templates +> python modules/embedding-preview.py + +## Grid + +Create flexible image grids from any number of images +> python modiles/grid.py + +### Image Watermark + +Create invisible image watermark and remove existing EXIF tags +> python modules/image-watermark.py +### Interrogate + +Runs CLiP and Booru image interrogation +> python modules/interrogate.py + +### Models Previews + +Create previews of models using built-in templates +> python modules/models-preview.py + +### Palette Extract + +Extract color palette from image(s) +> python modules/palette-extract.py + +### Image Process + +Run image processing to extract face/body segments and run resolution/blur/dynamic-range checks +> python modules/process.py + +### Prompt Ideas Generate complex prompt ideas -> python ideas.py --help +> python modules/prompt-ideas.py -## SDAPI +### Prompt Promptist + +Attempts to beautify the provided prompt +> python modules/promptist.py + +### Training Loss-Chart + +Create loss-chart from training log +> python modules/train-losschart.py + +### Training Loss-Rate + +Create customizable loss rate to be used in training +> python modules/train-lossrate.py + +### Video Extract + +Extract frames from video files +> python modules/video-extract.py + +
+ +## Utility Scripts +### SDAPI Utility module that handles async communication to Automatic API endpoints Can be used to manually execute specific commands: > python sdapi.py progress > python sdapi.py interrupt - -## FFMPEG - -Utility module that handles video files -Can be used to manually execute specific commands: -> ffmpeg extract --help -> python ffmpeg.py extract --input ~/downloads/vlado.mp4 --output ./vlado --fps 2 --skipstart 3 --skipend 1 - -## Grid - -Utility module to create image grids -> python grid.py --help - -## Bench - -Benchmark your Automatic -> python bench.py diff --git a/cli/modules/models-preview.py b/cli/modules/models-preview.py index 289defb22..99671fc31 100755 --- a/cli/modules/models-preview.py +++ b/cli/modules/models-preview.py @@ -5,7 +5,9 @@ import json import time import asyncio import argparse +from pathlib import Path +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) sys.path.append(os.path.join(os.path.dirname(__file__), 'modules')) from generate import sd, generate from modules.util import Map, log @@ -16,7 +18,7 @@ from modules.grid import grid default = 'sd-v15-runwayml.ckpt [cc6cb27103]' embeddings = ['blonde', 'bruntette', 'sexy', 'naked', 'mia', 'lin', 'kelly', 'hanna', 'rreid-random-v0'] exclude = ['sd-v20', 'sd-v21', 'inpainting', 'pix2pix'] -prompt = "photo of beautiful woman , photograph, posing, pose, high detailed, intricate, elegant, sharp focus, skin texture, looking forward, facing camera, 135mm, shot on dslr, canon 5d, 4k, modelshoot style, cinematic lighting" +prompt = "photo of , photograph, posing, pose, high detailed, intricate, elegant, sharp focus, skin texture, looking forward, facing camera, 135mm, shot on dslr, canon 5d, 4k, modelshoot style, cinematic lighting" options = Map({ 'generate': { 'restore_faces': True, @@ -29,17 +31,20 @@ options = Map({ 'sampler_name': 'DPM2 Karras', 'cfg_scale': 7, 'width': 512, - 'height': 512 + 'height': 512, }, 'paths': { "root": "/mnt/c/Users/mandi/OneDrive/Generative/Generate", "generate": "image", "upscale": "upscale", - "grid": "grid" + "grid": "grid", }, 'options': { "sd_model_checkpoint": "sd-v15-runwayml", - "sd_vae": "vae-ft-mse-840000-ema-pruned.ckpt" + "sd_vae": "vae-ft-mse-840000-ema-pruned.ckpt", + }, + 'lora': { + 'strength': 0.8, } }) @@ -97,6 +102,7 @@ async def models(params): t0 = time.time() for embedding in embeddings: options.generate.prompt = prompt.replace('', f'\"{embedding}\"') + options.generate.prompt = options.generate.prompt.replace('', 'beautiful woman') log.info({ 'model generating': model, 'embedding': embedding, 'prompt': options.generate.prompt }) data = await generate(options = options, quiet=True) if 'image' in data: @@ -118,6 +124,48 @@ async def models(params): opt['sd_model_checkpoint'] = default await post('/sdapi/v1/options', opt) + +async def lora(params): + cmdflags = await get('/sdapi/v1/cmd-flags') + dir = cmdflags['lora_dir'] + if not os.path.exists(dir): + log.error({ 'lora directory not found': dir }) + return + models1 = [f for f in Path(dir).glob('*.safetensors')] + models2 = [f for f in Path(dir).glob('*.ckpt')] + models = [f.stem for f in models1 + models2] + log.info({ 'loras': len(models) }) + for model in models: + fn = os.path.join(dir, model + '.png') + if os.path.exists(fn) and len(params.input) == 0: # if model preview exists and not manually included + log.info({ 'lora preview exists': model }) + continue + images = [] + labels = [] + t0 = time.time() + keyword = model.replace('-', ' ') + options.generate.prompt = prompt.replace('', f'\"{keyword}\"') + options.generate.prompt = options.generate.prompt.replace('', '') + options.generate.prompt += f' ' + log.info({ 'lora generating': model, 'keyword': keyword, 'prompt': options.generate.prompt }) + data = await generate(options = options, quiet=True) + if 'image' in data: + for img in data['image']: + images.append(img) + labels.append(keyword) + else: + log.error({ 'lora': model, 'embedding': keyword, 'error': data }) + t1 = time.time() + image = grid(images = images, labels = labels, border = 8) + image.save(fn) + t = t1 - t0 + its = 1.0 * options.generate.steps * len(images) / t + log.info({ 'lora preview created': model, 'image': fn, 'images': len(images), 'grid': [image.width, image.height], 'time': round(t, 2), 'its': round(its, 2) }) + + +async def create_previews(params): + await models(params) + await lora(params) await close() @@ -126,4 +174,4 @@ if __name__ == '__main__': parser.add_argument('--output', type = str, default = '', required = False, help = 'output directory') parser.add_argument('input', type = str, nargs = '*') params = parser.parse_args() - asyncio.run(models(params)) + asyncio.run(create_previews(params)) diff --git a/cli/train.md b/cli/train.md deleted file mode 100644 index 076fe2098..000000000 --- a/cli/train.md +++ /dev/null @@ -1,28 +0,0 @@ -# learning notes - -## Using accumulation: 1 - -- steps: 4500 -- batch': 2 -- accumulation: 1 -- rate: "0.005000:100, 0.002500:300, 0.001000:600, 0.000500:1000, 0.000250:1500, 0.000100:2100, 0.000050:2800, 0.000025:3600, 0.000010:4500", - -## Using accumulation: 10 - -- steps: 200 -- batch': 2 -- accumulation: 10 -- learning-rate: '0.010:10, 0.008:20, 0.006:40, 0.004:80, 0.002:120, 0.001:160, 0.0005:200' - -## Train - - sdapi.py interrupt - rm -rf /tmp/train/ ~/dev/automatic/embeddings/rebeccagivens-v6.pt ~/dev/automatic/train/log/rebeccagivens-v6* - train.py --name rebeccagivens-v6 --src ~/generative/Input/rebeccagivens/ --init person,woman,girl,model --overwrite - -## Prompt - - a medium shot photo of "kelly", extremely detailed 8k wallpaper, intricate, high detail, dramatic, modelshoot style - -## Gen -