diff --git a/TODO.md b/TODO.md index ada239ef4..8c0f7599d 100644 --- a/TODO.md +++ b/TODO.md @@ -49,13 +49,16 @@ Tech that can be integrated as part of the core workflow... - - - -- kubernetes dnsname -- rife -- add sd-webui-agent-scheduler: -- remove sd-webui-model-converter -- update training to use interrogator -- update training to use rembg +- update `train.py` to use `interrogator` +- update `train.py` to use `rembg` - - shared.info - hints -- import-hooks + +shutdown instance -> edit +and on the right hand side you'll see kubernetes config for the instance +which also includes dns name for the instance + external-dns.alpha.kubernetes.io/hostname: sdnext-a6000.tenant-91a92d-prod.coreweave.cloud + +ui -> namespaces -> tenant-91a92d-prod +dns name for the instance is ..coreweave.cloud diff --git a/cli/train.py b/cli/train.py index 14ef28d6a..8ee5a675e 100755 --- a/cli/train.py +++ b/cli/train.py @@ -120,7 +120,7 @@ def parse_args(): def prepare_server(): try: - server_status = util.Map(sdapi.progress()) + server_status = util.Map(sdapi.progresssync()) server_state = server_status['state'] except: log.error(f'server error: {server_status}') diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index e78d486ce..bdcd34d21 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit e78d486ce0e5cb9adc52549370d71e0433bf2111 +Subproject commit bdcd34d21bb77e8280710302240731431509e555 diff --git a/installer.py b/installer.py index 16414fa5e..10113fc0c 100644 --- a/installer.py +++ b/installer.py @@ -9,6 +9,7 @@ import subprocess import io import pstats import cProfile +import pkg_resources try: from modules.cmd_args import parser @@ -98,7 +99,6 @@ def print_profile(profile: cProfile.Profile, msg: str): # check if package is installed def installed(package, friendly: str = None): - import pkg_resources ok = True try: if friendly: @@ -132,6 +132,23 @@ def installed(package, friendly: str = None): return False +def pip(arg: str, ignore: bool = False): + arg = arg.replace('>=', '==') + log.info(f'Installing package: {arg.replace("install", "").replace("--upgrade", "").replace("--no-deps", "").replace("--force", "").replace(" ", " ").strip()}') + log.debug(f"Running pip: {arg}") + result = subprocess.run(f'"{sys.executable}" -m pip {arg}', shell=True, check=False, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + txt = result.stdout.decode(encoding="utf8", errors="ignore") + if len(result.stderr) > 0: + txt += ('\n' if len(txt) > 0 else '') + result.stderr.decode(encoding="utf8", errors="ignore") + txt = txt.strip() + if result.returncode != 0 and not ignore: + global errors # pylint: disable=global-statement + errors += 1 + log.error(f'Error running pip: {arg}') + log.debug(f'Pip output: {txt}') + return txt + + # install package using pip if not already installed def install(package, friendly: str = None, ignore: bool = False): if args.reinstall: @@ -139,25 +156,8 @@ def install(package, friendly: str = None, ignore: bool = False): quick_allowed = False if args.use_ipex and package == "pytorch_lightning==1.9.4": package = "pytorch_lightning==1.8.6" - - def pip(arg: str): - arg = arg.replace('>=', '==') - log.info(f'Installing package: {arg.replace("install", "").replace("--upgrade", "").replace("--no-deps", "").replace("--force", "").replace(" ", " ").strip()}') - log.debug(f"Running pip: {arg}") - result = subprocess.run(f'"{sys.executable}" -m pip {arg}', shell=True, check=False, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - txt = result.stdout.decode(encoding="utf8", errors="ignore") - if len(result.stderr) > 0: - txt += ('\n' if len(txt) > 0 else '') + result.stderr.decode(encoding="utf8", errors="ignore") - txt = txt.strip() - if result.returncode != 0 and not ignore: - global errors # pylint: disable=global-statement - errors += 1 - log.error(f'Error running pip: {arg}') - log.debug(f'Pip output: {txt}') - return txt - if args.reinstall or not installed(package, friendly): - pip(f"install --upgrade {package}") + pip(f"install --upgrade {package}", ignore=ignore) # execute git command @@ -311,7 +311,6 @@ def check_torch(): try: if args.use_directml and allow_directml: import torch_directml # pylint: disable=import-error - import pkg_resources version = pkg_resources.get_distribution("torch-directml") log.info(f'Torch backend: DirectML ({version})') for i in range(0, torch_directml.device_count()): @@ -327,6 +326,11 @@ def check_torch(): try: if 'xformers' in xformers_package: install(f'--no-deps {xformers_package}', ignore=True) + else: + x = pkg_resources.working_set.by_key.get('xformers', None) + if x is not None: + log.warning(f'Not used, uninstalling: {x}') + pip('uninstall xformers --yes --quiet', ignore=True) except Exception as e: log.debug(f'Cannot install xformers package: {e}') try: @@ -428,7 +432,6 @@ def install_extensions(): if args.profile: pr = cProfile.Profile() pr.enable() - import pkg_resources pkg_resources._initialize_master_working_set() # pylint: disable=protected-access pkgs = [f'{p.project_name}=={p._version}' for p in pkg_resources.working_set] # pylint: disable=protected-access,not-an-iterable log.debug(f'Installed packages: {len(pkgs)}') diff --git a/javascript/progressbar.js b/javascript/progressbar.js index 404645f4b..adfb21f24 100644 --- a/javascript/progressbar.js +++ b/javascript/progressbar.js @@ -46,13 +46,23 @@ function checkPaused(state) { function setProgress(res) { elements = ['txt2img_generate', 'img2img_generate', 'extras_generate'] - perc = res ? `${Math.round((res?.progress || 0) * 100.0)}%` : '' - eta = res?.paused ? ' Paused' : ` ETA: ${Math.round(res?.eta || 0)}s`; + const progress = (res?.progress || 0) + const perc = res && (progress > 0) ? `${Math.round(100.0 * progress)}%` : '' + let sec = res?.eta || 0 + let eta = ''; + if (res?.paused) eta = 'Paused'; + else if (res?.completed || (progress > 0.99)) eta = 'Finishing'; + else if (sec === 0) eta = 'Starting'; + else { + min = Math.floor(sec / 60); + sec = sec % 60; + eta = min > 0 ? `ETA: ${Math.round(min)}m ${Math.round(sec)}s` : `ETA: ${Math.round(sec)}s`; + } document.title = 'SD.Next ' + perc; for (elId of elements) { el = document.getElementById(elId); el.innerText = res - ? perc + eta + ? `${perc} ${eta}` : 'Generate'; el.style.background = res ? `linear-gradient(to right, var(--primary-500) 0%, var(--primary-800) ${perc}, var(--neutral-700) ${perc})` @@ -66,19 +76,19 @@ function randomId() { // starts sending progress requests to "/internal/progress" uri, creating progressbar above progressbarContainer element and preview inside gallery element // Cleans up all created stuff when the task is over and calls atEnd. calls onProgress every time there is a progress update -function requestProgress(id_task, gallery, atEnd = null, onProgress = null, once = false) { +function requestProgress(id_task, progressEl, galleryEl, atEnd = null, onProgress = null, once = false) { localStorage.setItem('task', id_task); let hasStarted = false; const dateStart = new Date(); const prevProgress = null; - const parentGallery = gallery ? gallery.parentNode : null; + const parentGallery = galleryEl ? galleryEl.parentNode : null; let livePreview; const img = new Image(); if (parentGallery) { livePreview = document.createElement('div'); livePreview.className = 'livePreview'; - parentGallery.insertBefore(livePreview, gallery); - const rect = gallery.getBoundingClientRect(); + parentGallery.insertBefore(livePreview, galleryEl); + const rect = galleryEl.getBoundingClientRect(); if (rect.width) { livePreview.style.width = `${rect.width}px`; livePreview.style.height = `${rect.height}px`; @@ -108,7 +118,7 @@ function requestProgress(id_task, gallery, atEnd = null, onProgress = null, once return; } setProgress(res); - if (res.live_preview && gallery) img.src = res.live_preview; + if (res.live_preview && galleryEl) img.src = res.live_preview; if (onProgress) onProgress(res); setTimeout(() => start(id_task, res.id_live_preview), opts.live_preview_refresh_period || 250); }, done); diff --git a/javascript/ui.js b/javascript/ui.js index 2e2196f1d..bdde05da2 100644 --- a/javascript/ui.js +++ b/javascript/ui.js @@ -118,21 +118,23 @@ function create_submit_args(args) { return res; } +function showSubmitButtons(tabname, show) {} + function submit(...args) { - console.log('submit txt2img:', args); + console.log('Submit txt2img:', args); rememberGallerySelection('txt2img_gallery'); const id = randomId(); - requestProgress(id, gradioApp().getElementById('txt2img_gallery')); + requestProgress(id, null, gradioApp().getElementById('txt2img_gallery')); const res = create_submit_args(args); res[0] = id; return res; } function submit_img2img(...args) { - console.log('submit img2img:', args); + console.log('Submit img2img:', args); rememberGallerySelection('img2img_gallery'); const id = randomId(); - requestProgress(id, gradioApp().getElementById('img2img_gallery')); + requestProgress(id, null, gradioApp().getElementById('img2img_gallery')); const res = create_submit_args(args); res[0] = id; res[1] = get_tab_index('mode_img2img'); @@ -424,7 +426,7 @@ function reconnect_ui() { if (task_id) { console.debug('task check:', task_id); rememberGallerySelection('txt2img_gallery'); - requestProgress(task_id, gallery, null, null, true); + requestProgress(task_id, null, gallery, null, null, true); } const sd_model = gradioApp().getElementById('setting_sd_model_checkpoint'); diff --git a/modules/images.py b/modules/images.py index b2aa0cac0..e9656b365 100644 --- a/modules/images.py +++ b/modules/images.py @@ -465,14 +465,10 @@ def atomically_save_image(): if shared.opts.save_log_fn != '' and len(exifinfo_data) > 0: try: with open(os.path.join(paths.data_path, shared.opts.save_log_fn), mode='a+', encoding='utf-8') as f: - try: - entries = json.load(f) - except: - entries = [] - f.seek(0) - entries.append({ 'filename': filename, 'time': datetime.datetime.now().isoformat(), 'info': exifinfo_data }) - json.dump(entries, f, indent=4) - del entries + entry = { 'filename': filename, 'time': datetime.datetime.now().isoformat(), 'info': exifinfo_data } + json.dump(entry, f) + f.write(os.linesep) + shared.log.debug(f'Log file updated: {os.path.join(paths.data_path, shared.opts.save_log_fn)}') except Exception as e: shared.log.warning(f'Failed to save log file: {shared.opts.save_log_fn} {e}') save_queue.task_done() diff --git a/modules/import_hook.py b/modules/import_hook.py deleted file mode 100644 index c94c3b16b..000000000 --- a/modules/import_hook.py +++ /dev/null @@ -1,15 +0,0 @@ -import sys -from modules.shared import opts, log - -# this will break any attempt to import xformers which will prevent stability diffusion repo from trying to use it -try: - import xformers # pylint: disable=unused-import, import-error - import xformers.ops # pylint: disable=unused-import, import-error -except: - pass - -if opts.cross_attention_optimization != "xFormers": - if sys.modules.get("xformers", None) is not None: - log.info('Unloading xFormers') - sys.modules["xformers"] = None - sys.modules["xformers.ops"] = None diff --git a/modules/processing.py b/modules/processing.py index 61ebd961e..1d76ce839 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -147,6 +147,7 @@ class StableDiffusionProcessing: self.is_hr_pass = False opts.data['clip_skip'] = clip_skip + @property def sd_model(self): return shared.sd_model diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index 9349ed4e2..c1323eab5 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -1,3 +1,4 @@ +import sys from types import MethodType import torch from torch.nn.functional import silu diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py index d1ae932e1..cef0d8a6a 100644 --- a/modules/sd_hijack_optimizations.py +++ b/modules/sd_hijack_optimizations.py @@ -1,3 +1,4 @@ +import sys import math import psutil @@ -19,6 +20,11 @@ if shared.opts.cross_attention_optimization == "xFormers": shared.xformers_available = True except Exception: pass +else: + if sys.modules.get("xformers", None) is not None: + shared.log.debug('Unloading xFormers') + sys.modules["xformers"] = None + sys.modules["xformers.ops"] = None def get_available_vram(): diff --git a/modules/shared.py b/modules/shared.py index d684ca0e4..b213a94d1 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -331,7 +331,7 @@ options_templates.update(options_section(('saving-images', "Image Options"), { "grid_prevent_empty_spots": OptionInfo(True, "Prevent empty spots in grid (when set to autodetect)"), "n_rows": OptionInfo(-1, "Grid row count; use -1 for autodetect and 0 for it to be same as batch size", gr.Slider, {"minimum": -1, "maximum": 16, "step": 1}), "save_txt": OptionInfo(False, "Create a text file next to every image with generation parameters"), - "save_log_fn": OptionInfo("", "Create a log file with image information for each saved image", component_args=hide_dirs), + "save_log_fn": OptionInfo("", "Create a JSON log file with image information for each saved image", component_args=hide_dirs), "save_images_before_face_restoration": OptionInfo(False, "Save a copy of image before doing face restoration"), "save_images_before_highres_fix": OptionInfo(False, "Save a copy of image before applying highres fix"), "save_images_before_color_correction": OptionInfo(False, "Save a copy of image before applying color correction to img2img results"), @@ -484,7 +484,7 @@ options_templates.update(options_section(('ui', "Live previews"), { })) options_templates.update(options_section(('sampler-params', "Sampler parameters"), { - "show_samplers": OptionInfo(["Euler a", "UniPC", "DDIM", "DPM++ SDE", "DPM++ SDE", "DPM2 Karras", "DPM++ 2M Karras"], "Show samplers in user interface", gr.CheckboxGroup, lambda: {"choices": [x.name for x in list_samplers() if x.name != "PLMS"]}), + "show_samplers": OptionInfo(["Euler a", "UniPC", "DDIM", "DPM++ 2M SDE", "DPM++ 2M SDE Karras", "DPM2 Karras", "DPM++ 2M Karras"], "Show samplers in user interface", gr.CheckboxGroup, lambda: {"choices": [x.name for x in list_samplers() if x.name != "PLMS"]}), "fallback_sampler": OptionInfo("Euler a", "Secondary sampler", gr.Dropdown, lambda: {"choices": ["None"] + [x.name for x in list_samplers()]}), "xyz_fallback_sampler": OptionInfo("None", "Force latent upscaler sampler", gr.Dropdown, lambda: {"choices": ["None"] + [x.name for x in list_samplers()]}), "eta_ancestral": OptionInfo(1.0, "Noise multiplier for ancestral samplers (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), diff --git a/webui.py b/webui.py index 666ff2083..5edba0ad3 100644 --- a/webui.py +++ b/webui.py @@ -27,7 +27,6 @@ warnings.filterwarnings(action="ignore", category=FutureWarning) warnings.filterwarnings(action="ignore", category=UserWarning, module="torchvision") startup_timer.record("torch") -from modules import import_hook # pylint: disable=W0611,C0411,C0412 from fastapi import FastAPI # pylint: disable=W0611,C0411 import gradio # pylint: disable=W0611,C0411 startup_timer.record("gradio")