diff --git a/config.json b/config.json index 19443ad3e..946f17acc 100644 --- a/config.json +++ b/config.json @@ -56,7 +56,7 @@ "training_image_repeats_per_epoch": 1, "training_write_csv_every": 10.0, "training_xattention_optimizations": false, - "sd_model_checkpoint": "sd-v15-runwayml.ckpt [81761151]", + "sd_model_checkpoint": "mix-protogen-x58.safetensors [13a6777c]", "sd_checkpoint_cache": 0, "sd_vae": "vae-ft-mse-840000-ema-pruned", "sd_vae_as_default": false, @@ -126,7 +126,9 @@ "s_tmin": 0.0, "s_noise": 1.0, "eta_noise_seed_delta": 0, - "disabled_extensions": [], + "disabled_extensions": [ + "roll-artist" + ], "ldsr_steps": 100, "ldsr_cached": false, "SWIN_tile": 192, diff --git a/extensions-builtin/aesthetic-scorer/README.md b/extensions-builtin/aesthetic-scorer/README.md new file mode 100644 index 000000000..4077903d4 --- /dev/null +++ b/extensions-builtin/aesthetic-scorer/README.md @@ -0,0 +1,25 @@ +# Aesthetic Scorer extension for SD Automatic WebUI + +Uses existing CLiP model with an additional small pretrained to calculate perceived aesthetic score of an image + +This is an *"invisible"* extension, it runs in the background before any image save and +appends **`score`** as *PNG info section* and/or *EXIF comments* field + +## Notes + +- Configuration via **Settings** → **Aesthetic scorer** + ![screenshot](aesthetic-scorer.jpg) +- Extension obeys existing **Move VAE and CLiP to RAM** settings +- Models will be auto-downloaded upon first usage (small) +- Score values are `0..10` +- Supports both `CLiP-ViT-L/14` and `CLiP-ViT-B/16` + +This extension uses different method than [Aesthetic Image Scorer](https://github.com/tsngo/stable-diffusion-webui-aesthetic-image-scorer) extension which: +- Uses modified [SD Chad scorer](https://github.com/AUTOMATIC1111/stable-diffusion-webui/discussions/1831) implementation +- Windows-only! +- Executes as to replace `image.save` so limited compatibity with other *non-txt2img* use-cases + +## Credits + +- Based on: [simulacra-aesthetic-models](https://github.com/crowsonkb/simulacra-aesthetic-models) +- Training data set: [simulacra-aesthetic-captions](https://github.com/JD-P/simulacra-aesthetic-captions) diff --git a/extensions-builtin/aesthetic-scorer/aesthetic-scorer.jpg b/extensions-builtin/aesthetic-scorer/aesthetic-scorer.jpg new file mode 100644 index 000000000..dc09169a3 Binary files /dev/null and b/extensions-builtin/aesthetic-scorer/aesthetic-scorer.jpg differ diff --git a/extensions-builtin/aesthetic-scorer/scripts/aesthetic-scorer.py b/extensions-builtin/aesthetic-scorer/scripts/aesthetic-scorer.py new file mode 100644 index 000000000..39f30935d --- /dev/null +++ b/extensions-builtin/aesthetic-scorer/scripts/aesthetic-scorer.py @@ -0,0 +1,125 @@ +import os + +import gradio as gr +import requests +import torch +from clip import clip +from modules import devices, script_callbacks, shared +from modules.script_callbacks import ImageSaveParams +from torch import nn +from torch.nn import functional as f +from torchvision import transforms +from torchvision.transforms import functional as tf + +extension_path = 'extensions/aesthetic-scorer' +git_home = 'https://github.com/vladmandic/sd-extensions/blob/main/extensions/aesthetic-scorer/models' +error = False +clip_model = None +aesthetic_model = None +normalize = transforms.Normalize(mean = [0.48145466, 0.4578275, 0.40821073], std=[0.26862954, 0.26130258, 0.27577711]) + + +class AestheticMeanPredictionLinearModel(nn.Module): + def __init__(self, feats_in): + super().__init__() + self.linear = nn.Linear(feats_in, 1) + + def forward(self, tensor): + x = f.normalize(tensor, dim=-1) * tensor.shape[-1] ** 0.5 + return self.linear(x) + + +def find_model(): + global error + if shared.opts.aesthetic_scorer_clip_model == 'ViT-L/14': + model_name = 'sac_public_2022_06_29_vit_l_14_linear.pth' + elif shared.opts.aesthetic_scorer_clip_model == 'ViT-B/16': + model_name = 'sac_public_2022_06_29_vit_b_16_linear.pth' + else: + model_name = shared.opts.aesthetic_scorer_clip_model + print(f'Aesthetic scorer: cannot match model for CLiP model {shared.opts.aesthetic_scorer_clip_model}') + error = True + model_path = os.path.join(extension_path, 'models', model_name) + + if not error and not os.path.exists(model_path): + try: + print(f'Aesthetic scorer downloading model: {model_name}') + url = f"{git_home}/{model_name}?raw=true" + r = requests.get(url, timeout=60) + with open(model_path, "wb") as f: + f.write(r.content) + except Exception as e: + print(f'Aesthetic scorer downloading model failed: {model_name}:', e) + + return model_path + + +def load_models(): + global clip_model + global aesthetic_model + if clip_model is None: + print(f'Loading CLiP model {shared.opts.aesthetic_scorer_clip_model} ') + clip_model, _clip_preprocess = clip.load(shared.opts.aesthetic_scorer_clip_model, jit = False, device = shared.device, download_root = shared.cmd_opts.clip_models_path) + clip_model.eval().requires_grad_(False) + idx = torch.tensor(0).to(shared.device) + first_embedding = clip_model.token_embedding(idx) + expected_shape = first_embedding.shape[0] + if aesthetic_model is None: + aesthetic_model = AestheticMeanPredictionLinearModel(expected_shape) + aesthetic_model.load_state_dict(torch.load(find_model())) + # move to gpu + clip_model = clip_model.to(shared.device) + aesthetic_model = aesthetic_model.to(shared.device) + return + + +def cleanup_models(): + if not shared.opts.interrogate_keep_models_in_memory: + clip_model = clip_model.to(devices.cpu) + aesthetic_model = aesthetic_model.to(devices.cpu) + devices.torch_gc() + return + + +def on_before_image_saved(params: ImageSaveParams): + global error + if not shared.opts.aesthetic_scorer_enabled or error or params.image is None: # dont try again if previously errored out or no image + return params + try: + load_models() + img = params.image.convert('RGB') + img = tf.resize(img, 224, transforms.InterpolationMode.LANCZOS) # resizes smaller edge + img = tf.center_crop(img, (224,224)) # center crop non-squared images + img = tf.to_tensor(img).to(shared.device) + img = normalize(img) + clip_image_embed = f.normalize(clip_model.encode_image(img[None, ...]).float(), dim = -1) + score = aesthetic_model(clip_image_embed) + score = round(score.item(), 2) + params.pnginfo['score'] = score + cleanup_models() + except Exception as e: + print('Aesthetic scorer error:', e) + error = True + return params + + +def on_ui_settings(): + section = ('aesthetic_scorer', "Aesthetic scorer") + shared.opts.add_option("aesthetic_scorer_enabled", shared.OptionInfo( + default = True, + label = "Enabled", + component = gr.Checkbox, + component_args = { 'interactive': True }, + section = section + )) + shared.opts.add_option("aesthetic_scorer_clip_model", shared.OptionInfo( + default = 'ViT-L/14', + label = "CLiP model", + component = gr.Radio, + component_args = { 'choices': ['ViT-L/14', 'ViT-B/16'] }, + section = section + )) + + +script_callbacks.on_before_image_saved(on_before_image_saved) +script_callbacks.on_ui_settings(on_ui_settings) diff --git a/extensions-builtin/info-tab/README.md b/extensions-builtin/info-tab/README.md new file mode 100644 index 000000000..2a4a724b1 --- /dev/null +++ b/extensions-builtin/info-tab/README.md @@ -0,0 +1,33 @@ +# Info Tab extensions for SD Automatic WebUI + +Creates a top-level **Info** tab in Automatic WebUI with + +State & memory info are auto-updated every second if tab is visible (no updates are performed when tab is not visible) +All other information is updated once upon WebUI load and can be force refreshed if required + +## Current information: +- Version +- Current Model & VAE +- Current State +- Current Memory statistics + +## System data: +- Platform details +- Torch & CUDA details +- Active CMD flags such as `low-vram` or `med-vram` +- Versions of critical libraries +- Versions of dependent repositories + + ![screenshot](info-tab.jpg) + +## Models +- Models +- Hypernetworks +- Embeddings + + ![screenshot](info-tab-models.jpg) + +## Info Object +- System object is available as JSON for quick passing of information + + ![screenshot](info-tab-json.jpg) diff --git a/extensions-builtin/info-tab/info-tab-json.jpg b/extensions-builtin/info-tab/info-tab-json.jpg new file mode 100644 index 000000000..4e5d3b2a4 Binary files /dev/null and b/extensions-builtin/info-tab/info-tab-json.jpg differ diff --git a/extensions-builtin/info-tab/info-tab-models.jpg b/extensions-builtin/info-tab/info-tab-models.jpg new file mode 100644 index 000000000..9d317005d Binary files /dev/null and b/extensions-builtin/info-tab/info-tab-models.jpg differ diff --git a/extensions-builtin/info-tab/info-tab.jpg b/extensions-builtin/info-tab/info-tab.jpg new file mode 100644 index 000000000..428114efb Binary files /dev/null and b/extensions-builtin/info-tab/info-tab.jpg differ diff --git a/extensions-builtin/info-tab/install.py b/extensions-builtin/info-tab/install.py new file mode 100644 index 000000000..5416cb0a4 --- /dev/null +++ b/extensions-builtin/info-tab/install.py @@ -0,0 +1 @@ +import launch diff --git a/extensions-builtin/info-tab/javascript/info-tab.js b/extensions-builtin/info-tab/javascript/info-tab.js new file mode 100644 index 000000000..a262587ca --- /dev/null +++ b/extensions-builtin/info-tab/javascript/info-tab.js @@ -0,0 +1,39 @@ +// this would not be needed if automatic run gradio with loop enabled + +let loaded = false; +let interval; + +function refresh() { + const btn = gradioApp().getElementById('info_tab_refresh_btn') // we could cache this dom element + if (!btn) return // but ui may get destroyed + btn.click() // actual refresh is done from python code we just trigger it but simulating button click +} + +function onHidden() { // stop refresh interval when tab is not visible + if (!interval) return + clearInterval(interval); + interval = undefined; +} + +function onVisible() { // start refresh interval tab is when visible + if (interval) return // interval already started so dont start it again + interval = setInterval(refresh, 1000); +} + +function initLoading() { // triggered on gradio change to monitor when ui gets sufficiently constructed + if (loaded) return + const block = gradioApp().getElementById('info_tab'); + if (!block) return + intersectionObserver = new IntersectionObserver((entries) => { + if (entries[0].intersectionRatio <= 0) onHidden(); + if (entries[0].intersectionRatio > 0) onVisible(); + }); + intersectionObserver.observe(block); // monitor visibility of tab +} + +function initInitial() { // just setup monitor for gradio events + const mutationObserver = new MutationObserver(initLoading) + mutationObserver.observe(gradioApp(), { childList: true, subtree: true }); // monitor changes to gradio +} + +document.addEventListener('DOMContentLoaded', initInitial); diff --git a/extensions-builtin/info-tab/preload.py b/extensions-builtin/info-tab/preload.py new file mode 100644 index 000000000..b37effc27 --- /dev/null +++ b/extensions-builtin/info-tab/preload.py @@ -0,0 +1,2 @@ +def preload(parser): + pass diff --git a/extensions-builtin/info-tab/scripts/info-tab.py b/extensions-builtin/info-tab/scripts/info-tab.py new file mode 100644 index 000000000..d50754e6d --- /dev/null +++ b/extensions-builtin/info-tab/scripts/info-tab.py @@ -0,0 +1,301 @@ +import datetime +import os +import platform +import subprocess +import time + +import accelerate +import gradio as gr +import psutil +import pytorch_lightning +import safetensors +import torch +import transformers +from modules import paths, script_callbacks, sd_hijack, sd_models, sd_samplers, shared + +data = {} + +def get_cuda(): + if not torch.cuda.is_available(): + return {} + else: + try: + return { + 'version': torch.version.cuda, + 'devices': torch.cuda.device_count(), + 'current': torch.cuda.get_device_name(torch.cuda.current_device()), + 'arch': torch.cuda.get_arch_list()[-1], + 'capability': torch.cuda.get_device_capability(shared.device), + } + except Exception as e: + return { 'error': e } + +def get_state(): + s = vars(shared.state) + flags = 'skipped ' if s.get('skipped', False) else '' + flags += 'interrupted ' if s.get('interrupted', False) else '' + flags += 'needs restart' if s.get('need_restart', False) else '' + return { + 'started': time.strftime('%c', time.localtime(s.get('time_start', time.time()))), + 'step': f'{s.get("sampling_step", 0)} / {s.get("sampling_steps", 0)}', + 'jobs': f'{s.get("job_no", 0)} / {s.get("job_count", 0)}', # pylint: disable=consider-using-f-string + 'flags': flags, + 'job': s.get('job', ''), + 'text': s.get('textinfo', ''), + } + +def get_memory(): + def gb(val: float): + return round(val / 1024 / 1024 / 1024, 2) + mem = {} + try: + process = psutil.Process(os.getpid()) + res = process.memory_info() + ram_total = 100 * res.rss / process.memory_percent() + ram = { 'free': gb(ram_total - res.rss), 'used': gb(res.rss), 'total': gb(ram_total) } + mem.update({ 'ram': ram }) + except Exception as e: + mem.update({ 'ram': e }) + try: + if torch.cuda.is_available(): + s = torch.cuda.mem_get_info() + gpu = { 'free': gb(s[0]), 'used': gb(s[1] - s[0]), 'total': gb(s[1]) } + s = dict(torch.cuda.memory_stats(shared.device)) + allocated = { 'current': gb(s['allocated_bytes.all.current']), 'peak': gb(s['allocated_bytes.all.peak']) } + reserved = { 'current': gb(s['reserved_bytes.all.current']), 'peak': gb(s['reserved_bytes.all.peak']) } + active = { 'current': gb(s['active_bytes.all.current']), 'peak': gb(s['active_bytes.all.peak']) } + inactive = { 'current': gb(s['inactive_split_bytes.all.current']), 'peak': gb(s['inactive_split_bytes.all.peak']) } + warnings = { 'retries': s['num_alloc_retries'], 'oom': s['num_ooms'] } + mem.update({ + 'gpu': gpu, + 'gpu-active': active, + 'gpu-allocated': allocated, + 'gpu-reserved': reserved, + 'gpu-inactive': inactive, + 'events': warnings, + }) + except: + pass + return mem + +def get_optimizations(): + ram = [] + if shared.cmd_opts.medvram: + ram.append('medvram') + if shared.cmd_opts.lowvram: + ram.append('lowvram') + if shared.cmd_opts.lowram: + ram.append('lowram') + if len(ram) == 0: + ram.append('none') + return ram + +def get_libs(): + return { + 'xformers': shared.xformers_available, + 'accelerate': accelerate.__version__, + 'transformers': transformers.__version__, + 'safetensors': safetensors.__version__, + 'lightning': pytorch_lightning.__version__, + } + +def get_repos(): + repos = {} + for key, val in paths.paths.items(): + try: + cmd = f'git -C {val} log --pretty=format:"%h %ad" -1 --date=short' + res = subprocess.run(f'{cmd} {val}', stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell=True, check=True) + stdout = res.stdout.decode(encoding = 'utf8', errors='ignore') if len(res.stdout) > 0 else '' + words = stdout.split(' ') + repos[key] = f'[{words[0]}] {words[1]}' + except: + repos[key] = '(unknown)' + return repos + +def get_model(): + try: + return { + 'configured': shared.opts.data['sd_model_checkpoint'], + 'current': shared.sd_model.sd_checkpoint_info.title, + 'configuration': os.path.basename(sd_models.find_checkpoint_config(shared.sd_model.sd_checkpoint_info)), + } + except: + return { 'error': 'no model config found' } + +def get_vae(): + try: + return { + 'configured': shared.opts.sd_vae, + 'current': os.path.basename(shared.sd_vae.loaded_vae_file), + } + except: + return { 'error': 'no vae config found' } + +def get_platform(): + try: + return { + 'host': platform.node(), + 'arch': platform.machine(), + 'cpu': platform.processor(), + 'system': platform.system(), + 'platform': platform.platform(aliased = True, terse = False), + 'release': platform.release(), + 'version': platform.version(), + 'python': platform.python_version(), + } + except Exception as e: + return { 'error': e } + +def get_torch(): + return { + 'version': torch.__version__, + 'precision': shared.cmd_opts.precision + (' fp32' if shared.cmd_opts.no_half else ' fp16'), + } + +def get_version(): + try: + res = subprocess.run('git log --pretty=format:"%h %ad" -1 --date=short', stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell=True, check=True) + ver = res.stdout.decode(encoding = 'utf8', errors='ignore') if len(res.stdout) > 0 else '' + githash, updated = ver.split(' ') + res = subprocess.run('git remote get-url origin', stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell=True, check=True) + origin = res.stdout.decode(encoding = 'utf8', errors='ignore') if len(res.stdout) > 0 else '' + res = subprocess.run('git branch --show-current', stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell=True, check=True) + branch = res.stdout.decode(encoding = 'utf8', errors='ignore') if len(res.stdout) > 0 else '' + return { + 'updated': updated, + 'hash': githash, + 'origin': origin.replace('\n', ''), + 'branch': branch.replace('\n', ''), + } + except: + return {} + +def get_embeddings(): + return [f'{v} ({sd_hijack.model_hijack.embedding_db.word_embeddings[v].vectors})' for i, v in enumerate(sd_hijack.model_hijack.embedding_db.word_embeddings)] + +def get_skipped(): + return [k for k in sd_hijack.model_hijack.embedding_db.skipped_embeddings.keys()] + +def get_crossattention(): + try: + return sd_hijack.model_hijack.optimization_method + except: + return 'unknown' + +def get_models(): + return [x.title for x in sd_models.checkpoints_list.values()] + +def get_samplers(): + return [sampler[0] for sampler in sd_samplers.all_samplers] + +def get_full_data(): + global data # pylint: disable=global-statement + data = { + 'date': datetime.datetime.now().strftime('%c'), + 'timestamp': datetime.datetime.now().strftime('%X'), + 'version': get_version(), + 'model': get_model(), + 'vae': get_vae(), + 'torch': get_torch(), + 'cuda': get_cuda(), + 'state': get_state(), + 'memory': get_memory(), + 'optimizations': get_optimizations(), + 'libs': get_libs(), + 'repos': get_repos(), + 'models': get_models(), + 'hypernetworks': [name for name in shared.hypernetworks], + 'embeddings': get_embeddings(), + 'skipped': get_skipped(), + 'schedulers': get_samplers(), + 'platform': get_platform(), + 'crossattention': get_crossattention(), + 'api': shared.cmd_opts.api, + 'webui': not shared.cmd_opts.nowebui, + } + return data + +def get_quick_data(): + data['timestamp'] = datetime.datetime.now().strftime('%X') + data['state'] = get_state() + data['memory'] = get_memory() + +def list2text(lst: list): + return '\n'.join(lst) + +def dict2str(d: dict): + arr = [f'{name}: {d[name]}' for i, name in enumerate(d)] + return ' '.join(arr) + +def dict2text(d: dict): + arr = ['{name}: {val}'.format(name = name, val = d[name] if not type(d[name]) is dict else dict2str(d[name])) for i, name in enumerate(d)] # pylint: disable=consider-using-f-string + return list2text(arr) + +def refresh_info_quick(): + get_quick_data() + return dict2text(data['state']), dict2text(data['memory']), data['timestamp'], data + +def refresh_info_full(): + get_full_data() + return dict2text(data['state']), dict2text(data['memory']), data['models'], data['hypernetworks'], data['embeddings'], data['skipped'], dict2text(data['model']), dict2text(data['vae']), data['timestamp'], data + +def on_ui_tabs(): + get_full_data() + with gr.Blocks(analytics_enabled = False) as info_tab: + with gr.Row(elem_id = 'info_tab'): + with gr.Column(scale = 9): + with gr.Box(): + with gr.Row(): + with gr.Column(): + gr.Textbox(dict2text(data['version']), label = 'Version', lines = len(data['version'])) + with gr.Column(): + model = gr.Textbox(dict2text(data['model']), label = 'Model', lines = len(data['model'])) + vae = gr.Textbox(dict2text(data['vae']), label = 'VAE', lines = len(data['vae'])) + with gr.Column(): + state = gr.Textbox(dict2text(data['state']), label = 'State', lines = len(data['state'])) + with gr.Column(): + memory = gr.Textbox(dict2text(data['memory']), label = 'Memory', lines = len(data['memory'])) + with gr.Box(): + with gr.Accordion('System data', open = True, visible = True): + with gr.Row(): + with gr.Column(): + gr.Textbox(dict2text(data['platform']), label = 'Platform', lines = len(data['platform'])) + with gr.Column(): + gr.Textbox(dict2text(data['torch']), label = 'Torch', lines = len(data['torch'])) + gr.Textbox(dict2text(data['cuda']), label = 'CUDA', lines = len(data['cuda'])) + with gr.Row(): + gr.Textbox(list2text(data['optimizations']), label = 'Memory optimization') + gr.Textbox(data['crossattention'], label = 'Cross-attention') + gr.Textbox((data['api']), label = 'API') + with gr.Column(): + gr.Textbox(dict2text(data['libs']), label = 'Libs', lines = len(data['libs'])) + gr.Textbox(dict2text(data['repos']), label = 'Repos', lines = len(data['repos'])) + with gr.Box(): + with gr.Accordion('Models...', open = False, visible = True): + with gr.Row(): + with gr.Column(): + models = gr.JSON(data['models'], label = 'Models', lines = len(data['models'])) + hypernetworks = gr.JSON(data['hypernetworks'], label = 'Hypernetworks', lines = len(data['hypernetworks'])) + with gr.Column(): + embeddings = gr.JSON(data['embeddings'], label = 'Embeddings: loaded', lines = len(data['embeddings'])) + skipped = gr.JSON(data['skipped'], label = 'Embeddings: skipped', lines = len(data['embeddings'])) + with gr.Box(): + with gr.Accordion('Info object', open = False, visible = True): + # reduce json data to avoid private info + data.pop('models', None) + data.pop('embeddings', None) + data.pop('skipped', None) + data.pop('hypernetworks', None) + data.pop('schedulers', None) + json = gr.JSON(data) + with gr.Column(scale = 1, min_width = 120): + timestamp = gr.Text(data['timestamp'], label = '', elem_id = 'info_tab_last_update') + refresh_quick = gr.Button('Refresh state', elem_id = 'info_tab_refresh_btn', visible = False).style(full_width = False) # quick refresh is used from js interval + refresh_quick.click(refresh_info_quick, inputs = [], outputs = [state, memory, timestamp, json]) + refresh_full = gr.Button('Refresh data').style(full_width = False) + refresh_full.click(refresh_info_full, inputs = [], outputs = [state, memory, models, hypernetworks, embeddings, skipped, model, vae, timestamp, json]) + interrupt = gr.Button('Send interrupt') + interrupt.click(shared.state.interrupt, inputs = [], outputs = []) + return (info_tab, 'Info', 'info_tab'), + +script_callbacks.on_ui_tabs(on_ui_tabs) diff --git a/modules/ui.py b/modules/ui.py index 99483130c..ef24b8240 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -1673,11 +1673,11 @@ def create_ui(): ) interfaces = [ - (txt2img_interface, "txt2img", "txt2img"), - (img2img_interface, "img2img", "img2img"), - (extras_interface, "Extras", "extras"), - (pnginfo_interface, "PNG Info", "pnginfo"), - (modelmerger_interface, "Checkpoint Merger", "modelmerger"), + (txt2img_interface, "Text", "txt2img"), + (img2img_interface, "Image", "img2img"), + (extras_interface, "Upscale", "extras"), + (pnginfo_interface, "Image Info", "pnginfo"), + # (modelmerger_interface, "Checkpoint Merger", "modelmerger"), (train_interface, "Train", "ti"), ] diff --git a/scripts/save_steps_animation.py b/scripts/save_steps_animation.py deleted file mode 120000 index 80e208dfe..000000000 --- a/scripts/save_steps_animation.py +++ /dev/null @@ -1 +0,0 @@ -/home/vlado/dev/sd-extensions/scripts/save_steps_animation.py \ No newline at end of file diff --git a/scripts/save_steps_animation.py b/scripts/save_steps_animation.py new file mode 100644 index 000000000..c92be7af9 --- /dev/null +++ b/scripts/save_steps_animation.py @@ -0,0 +1,142 @@ +import json +import os +import shutil + +import gradio as gr +from modules import scripts +from modules.images import save_image +from modules.sd_samplers import KDiffusionSampler, sample_to_image + +# configurable section +video_rate = 30 +author = 'https://github.com/vladmandic' +cli_template = "ffmpeg -hide_banner -loglevel {loglevel} -hwaccel auto -y -framerate {framerate} -i {inpath}/%5d.jpg -r {videorate} {preset} {minterpolate} {flags} -metadata title='{description}' -metadata description='{info}' -metadata author='stable-diffusion' -metadata album_artist='{author}' '{outfile}'" # note: +presets = { + 'x264': '-vcodec libx264 -preset medium -crf 23', + 'x265': '-vcodec libx265 -preset faster -crf 28', + 'vpx-vp9': '-vcodec libvpx-vp9 -crf 34 -b:v 0 -deadline realtime -cpu-used 4', + 'aom-av1': '-vcodec libaom-av1 -crf 28 -b:v 0 -usage realtime -cpu-used 8 -pix_fmt yuv444p', +} + +# internal state variables +current_step = 0 +orig_callback_state = KDiffusionSampler.callback_state + + +class Script(scripts.Script): + # script title to show in ui + def title(self): + return "Save animation of intermediate steps" + + + # is ui visible: process/postprocess triggers for always-visible scripts otherwise use run as entry point + def show(self, is_img2img): + return scripts.AlwaysVisible + + + # ui components + def ui(self, is_visible): + with gr.Accordion("Save animation", open = False, elem_id="save-animation"): + gr.HTML(""" + + Creates animation sequence from denoised intermediate steps with video frame interpolation to achieve desired animation duration
""") + with gr.Row(): + is_enabled = gr.Checkbox(label = "Script Enabled", value = False) + codec = gr.Radio(label = 'Codec', choices = ['x264', 'x265', 'vpx-vp9', 'aom-av1'], value = 'x264') + interpolation = gr.Radio(label = 'Interpolation', choices = ['none', 'mci', 'blend'], value = 'mci') + with gr.Row(): + duration = gr.Slider(label = "Duration", minimum = 0.5, maximum = 120, step = 0.1, value = 10) + skip_steps = gr.Slider(label = "Skip steps", minimum = 0, maximum = 100, step = 1, value = 5) + with gr.Row(): + debug = gr.Checkbox(label = "Debug info", value = False) + run_incomplete = gr.Checkbox(label = "Run on incomplete", value = True) + tmp_delete = gr.Checkbox(label = "Delete intermediate", value = True) + out_create = gr.Checkbox(label = "Create animation", value = True) + with gr.Row(): + tmp_path = gr.Textbox(label = "Path for intermediate files", lines = 1, value = "intermediate") + out_path = gr.Textbox(label = "Path for output animation file", lines = 1, value = "animation") + + return [is_enabled, codec, interpolation, duration, skip_steps, debug, run_incomplete, tmp_delete, out_create, tmp_path, out_path] + + + # runs on each step for always-visible scripts + def process(self, p, is_enabled, codec, interpolation, duration, skip_steps, debug, run_incomplete, tmp_delete, out_create, tmp_path, out_path): + if is_enabled: + def callback_state(self, d): + global current_step + current_step = d["i"] + 1 + if (skip_steps == 0) or (current_step > skip_steps): + image = sample_to_image(samples = d["denoised"], index = 0, approximation = None) + inpath = os.path.join(p.outpath_samples, tmp_path) + save_image(image, inpath, "", extension = 'jpg', short_filename = True, no_prompt = True) # filename using 00000 format so its easier for ffmpeg sequence parsing + return orig_callback_state(self, d) + + setattr(KDiffusionSampler, "callback_state", callback_state) + + + # run at the end of sequence for always-visible scripts + def postprocess(self, p, processed, is_enabled, codec, interpolation, duration, skip_steps, debug, run_incomplete, tmp_delete, out_create, tmp_path, out_path): + global current_step + setattr(KDiffusionSampler, "callback_state", orig_callback_state) + if not is_enabled: + return + # callback happened too early, it happens with large number of steps and some samplers or if interrupted + if vars(processed)['steps'] != current_step: + print('Save animation warning: postprocess early call', { 'current': current_step, 'target': vars(processed)['steps'] }) + if not run_incomplete: + return + # create dictionary with all input and output parameters + v = vars(processed) + params = { + 'prompt': v['prompt'], + 'negative': v['negative_prompt'], + 'seed': v['seed'], + 'sampler': v['sampler_name'], + 'cfgscale': v['cfg_scale'], + 'steps': v['steps'], + 'current': current_step, + 'skip': skip_steps, + 'info': v['info'].replace('\n', ' '), + 'model': v['info'].split('Model:')[1].split()[0] if ("Model:" in v['info']) else "unknown", # parse string if model info is present + 'embedding': v['info'].split('Used embeddings:')[1].split()[0] if ("Used embeddings:" in v['info']) else "none", # parse string if embedding info is present + 'faces': v['face_restoration_model'], + 'timestamp': v['job_timestamp'], + 'inpath': os.path.join(p.outpath_samples, tmp_path), + 'outpath': os.path.join(p.outpath_samples, out_path), + 'codec': 'lib' + codec, + 'duration': duration, + 'interpolation': interpolation, + 'loglevel': 'error', + 'cli': cli_template, + 'framerate': 1.0 * (current_step - skip_steps) / duration, + 'videorate': video_rate, + 'author': author, + 'preset': presets[codec], + 'flags': "-movflags +faststart", + 'ffmpeg': shutil.which("ffmpeg"), # detect if ffmpeg executable is present in path + } + if debug: + params['loglevel'] = 'info' + print("Save animation params:", json.dumps(params, indent = 2)) + if out_create: + if not os.path.isdir(params['inpath']) or not os.path.isdir(params['outpath']): + print('Save animation error: folder not found', params['inpath'], params['outpath']) + return + if params['ffmpeg'] is None: + print("Save animation error: ffmpeg not found:") + return + # append conditionals to dictionary + params['minterpolate'] = "" if (params['interpolation'] == "none") else "-vf minterpolate=mi_mode={mi},fifo".format(mi = params['interpolation']) + params['outfile'] = os.path.join(params['outpath'], str(params['seed']) + "-" + str(params['prompt'])) + ('.webm' if (params['codec'] == 'libvpx-vp9') else '.mp4') + params['description'] = "{prompt} | negative {negative} | seed {seed} | sampler {sampler} | cfgscale {cfgscale} | steps {steps} | current {current} | model {model} | embedding {embedding} | faces {faces} | timestamp {timestamp} | interpolation {interpolation}".format(**params) + print("Save animation creating movie sequence:", params['outfile']) + cmd = params['cli'].format(**params) + # actual ffmpeg call + os.system(cmd) + if tmp_delete: + for root, _dirs, files in os.walk(params['inpath']): + print("Save animation removing {n} files from temp folder: {path}".format(path = root, n = len(files))) + for file in files: + f = os.path.join(root, file) + if os.path.isfile(f): + os.remove(f) diff --git a/ui-config.json b/ui-config.json index f1ccf0c87..0eac8bd9d 100644 --- a/ui-config.json +++ b/ui-config.json @@ -1,6 +1,6 @@ { "txt2img/Prompt/visible": true, - "txt2img/Prompt/value": "photorealistic, high detailed, sharp focus, depth of field", + "txt2img/Prompt/value": "photorealistic, high detailed, sharp focus, depth of field, 4k", "txt2img/Negative prompt/visible": true, "txt2img/Negative prompt/value": "foggy, blurry, blurred, duplicate, ugly, mutilated, mutation, mutated, out of frame, bad anatomy, disfigured, deformed, censored, low res, watermark, text, poorly drawn face, signature", "txt2img/Style 1/value": "None", diff --git a/user.css b/user.css index 91edbf24e..08af99a87 100644 --- a/user.css +++ b/user.css @@ -24,7 +24,7 @@ div.gradio-container.dark > div.w-full.flex.flex-col.min-h-screen > div { backgr .dark .gr-form { border-radius: 0; border-width: 0; } .dark .gr-check-radio { background-color: var(--inactive-color); border-width: 0; border-radius: 2px; box-shadow: 2px 2px 3px #111111; } .dark .gr-check-radio:checked { background-color: var(--highlight-color); } -.dark .gr-button { border-radius: 0; font-weight: normal; box-shadow: 2px 2px 3px #111111; font-size: 0.9rem; min-width: 32px; } +.dark .gr-button { border-radius: 0; font-weight: normal; box-shadow: 2px 2px 3px #111111; font-size: 0.9rem; min-width: 42px; min-height: 42px; } .dark .gr-box { border-radius: 0; background-color: #222222; box-shadow: 2px 2px 3px #111111; border-width: 0; padding-bottom: 12px; } .dark .dark\:bg-gray-900 { background-color: black; } .dark .bg-white { color: lightyellow; border-radius: 0; } @@ -36,6 +36,8 @@ div.gradio-container.dark > div.w-full.flex.flex-col.min-h-screen > div { backgr .px-4 { padding-lefT: 1rem; padding-right: 1rem; } .py-6 { padding-bottom: 0; } .overflow-hidden .flex .flex-col .relative col .gap-4 { min-width: var(--left-column); max-width: var(--left-column); } /* this is a problematic one */ +.rounded-lg { border-radius: 0; } +.p-2 { padding: 0 } /* automatic style classes */ .progressDiv .progress { background: var(--highlight-color); border-radius: 2px; } @@ -49,7 +51,7 @@ div.gradio-container.dark > div.w-full.flex.flex-col.min-h-screen > div { backgr #img2img_neg_prompt > label > textarea { font-size: 1.2rem; } #txt2img_generate, #img2img_generate, #txt2img_interrupt, #img2img_interrupt, #txt2img_skip, #img2img_skip { margin-top: 10px; min-height: 2rem; height: 63px; } #txt2img_interrupt, #img2img_interrupt, #txt2img_skip, #img2img_skip { background-color: var(--inactive-color); } -#txt2img_gallery { background: black; } +#txt2img_gallery, #img2img_gallery, #extras_gallery { background: black; } #tab_extensions table { background-color: #222222; } #style_pos_col, #style_neg_col, #roll_col { display: none; } #interrogate_col { margin-top: 10px; } @@ -57,3 +59,5 @@ div.gradio-container.dark > div.w-full.flex.flex-col.min-h-screen > div { backgr #save-animation { border-radius: 0 !important; margin-bottom: 16px; background-color: #111111; } #open_folder_txt2img, #open_folder_img2img, #open_folder_extras { display: none } #footer { display: none; } +#txt2img_seed_row { padding: 0; margin-top: 8px; } +#txt2img_subseed_show { min-width: 74px; padding: 0 0 0 6px;}