update

TODO.md

@@ -49,13 +49,16 @@ Tech that can be integrated as part of the core workflow...
 - <https://towardsdatascience.com/mastering-memoization-in-python-dcdd8b435189>
 - <https://github.com/AUTOMATIC1111/stable-diffusion-webui/compare/89f9faa...20ae71f>
 - <https://github.com/vladmandic/automatic/issues/1056>
-- kubernetes dnsname
-- rife
-- add sd-webui-agent-scheduler: <https://github.com/vladmandic/automatic/issues/559> <https://github.com/ArtVentureX/sd-webui-agent-scheduler/issues/2>
-- remove sd-webui-model-converter
-- update training to use interrogator
-- update training to use rembg
+- update `train.py` to use `interrogator`
+- update `train.py` to use `rembg`
 - <https://github.com/vladmandic/automatic/discussions/1246>
 - shared.info
 - hints
-- import-hooks
+
+shutdown instance -> edit
+and on the right hand side you'll see kubernetes config for the instance
+which also includes dns name for the instance
+    external-dns.alpha.kubernetes.io/hostname: sdnext-a6000.tenant-91a92d-prod.coreweave.cloud
+
+ui -> namespaces -> tenant-91a92d-prod
+dns name for the instance is <instance-name>.<tenant-id>.coreweave.cloud

cli/train.py

@@ -120,7 +120,7 @@ def parse_args():
 
 def prepare_server():
     try:
-        server_status = util.Map(sdapi.progress())
+        server_status = util.Map(sdapi.progresssync())
         server_state = server_status['state']
     except:
         log.error(f'server error: {server_status}')

installer.py

@@ -9,6 +9,7 @@ import subprocess
 import io
 import pstats
 import cProfile
+import pkg_resources
 
 try:
     from modules.cmd_args import parser
@@ -98,7 +99,6 @@ def print_profile(profile: cProfile.Profile, msg: str):
 
 # check if package is installed
 def installed(package, friendly: str = None):
-    import pkg_resources
     ok = True
     try:
         if friendly:
@@ -132,6 +132,23 @@ def installed(package, friendly: str = None):
         return False
 
 
+def pip(arg: str, ignore: bool = False):
+    arg = arg.replace('>=', '==')
+    log.info(f'Installing package: {arg.replace("install", "").replace("--upgrade", "").replace("--no-deps", "").replace("--force", "").replace("  ", " ").strip()}')
+    log.debug(f"Running pip: {arg}")
+    result = subprocess.run(f'"{sys.executable}" -m pip {arg}', shell=True, check=False, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    txt = result.stdout.decode(encoding="utf8", errors="ignore")
+    if len(result.stderr) > 0:
+        txt += ('\n' if len(txt) > 0 else '') + result.stderr.decode(encoding="utf8", errors="ignore")
+    txt = txt.strip()
+    if result.returncode != 0 and not ignore:
+        global errors # pylint: disable=global-statement
+        errors += 1
+        log.error(f'Error running pip: {arg}')
+        log.debug(f'Pip output: {txt}')
+    return txt
+
+
 # install package using pip if not already installed
 def install(package, friendly: str = None, ignore: bool = False):
     if args.reinstall:
@@ -139,25 +156,8 @@ def install(package, friendly: str = None, ignore: bool = False):
         quick_allowed = False
     if args.use_ipex and package == "pytorch_lightning==1.9.4":
         package = "pytorch_lightning==1.8.6"
-
-    def pip(arg: str):
-        arg = arg.replace('>=', '==')
-        log.info(f'Installing package: {arg.replace("install", "").replace("--upgrade", "").replace("--no-deps", "").replace("--force", "").replace("  ", " ").strip()}')
-        log.debug(f"Running pip: {arg}")
-        result = subprocess.run(f'"{sys.executable}" -m pip {arg}', shell=True, check=False, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        txt = result.stdout.decode(encoding="utf8", errors="ignore")
-        if len(result.stderr) > 0:
-            txt += ('\n' if len(txt) > 0 else '') + result.stderr.decode(encoding="utf8", errors="ignore")
-        txt = txt.strip()
-        if result.returncode != 0 and not ignore:
-            global errors # pylint: disable=global-statement
-            errors += 1
-            log.error(f'Error running pip: {arg}')
-            log.debug(f'Pip output: {txt}')
-        return txt
-
     if args.reinstall or not installed(package, friendly):
-        pip(f"install --upgrade {package}")
+        pip(f"install --upgrade {package}", ignore=ignore)
 
 
 # execute git command
@@ -311,7 +311,6 @@ def check_torch():
                 try:
                     if args.use_directml and allow_directml:
                         import torch_directml # pylint: disable=import-error
-                        import pkg_resources
                         version = pkg_resources.get_distribution("torch-directml")
                         log.info(f'Torch backend: DirectML ({version})')
                         for i in range(0, torch_directml.device_count()):
@@ -327,6 +326,11 @@ def check_torch():
     try:
         if 'xformers' in xformers_package:
             install(f'--no-deps {xformers_package}', ignore=True)
+        else:
+            x = pkg_resources.working_set.by_key.get('xformers', None)
+            if x is not None:
+                log.warning(f'Not used, uninstalling: {x}')
+                pip('uninstall xformers --yes --quiet', ignore=True)
     except Exception as e:
         log.debug(f'Cannot install xformers package: {e}')
     try:
@@ -428,7 +432,6 @@ def install_extensions():
     if args.profile:
         pr = cProfile.Profile()
         pr.enable()
-    import pkg_resources
     pkg_resources._initialize_master_working_set() # pylint: disable=protected-access
     pkgs = [f'{p.project_name}=={p._version}' for p in pkg_resources.working_set] # pylint: disable=protected-access,not-an-iterable
     log.debug(f'Installed packages: {len(pkgs)}')

javascript/progressbar.js

@@ -46,13 +46,23 @@ function checkPaused(state) {
 
 function setProgress(res) {
   elements = ['txt2img_generate', 'img2img_generate', 'extras_generate']
-  perc = res ? `${Math.round((res?.progress || 0) * 100.0)}%` : ''
-  eta = res?.paused ? ' Paused' : ` ETA: ${Math.round(res?.eta || 0)}s`;
+  const progress = (res?.progress || 0)
+  const perc = res && (progress > 0) ? `${Math.round(100.0 * progress)}%` : ''
+  let sec = res?.eta || 0
+  let eta = '';
+  if (res?.paused) eta = 'Paused';
+  else if (res?.completed || (progress > 0.99)) eta = 'Finishing';
+  else if (sec === 0) eta = 'Starting';
+  else {
+    min = Math.floor(sec / 60);
+    sec = sec % 60;
+    eta = min > 0 ? `ETA: ${Math.round(min)}m ${Math.round(sec)}s` : `ETA: ${Math.round(sec)}s`; 
+  }
   document.title = 'SD.Next ' + perc;
   for (elId of elements) {
     el = document.getElementById(elId);
     el.innerText = res
-      ? perc + eta
+      ? `${perc} ${eta}`
       : 'Generate';
     el.style.background = res 
       ? `linear-gradient(to right, var(--primary-500) 0%, var(--primary-800) ${perc}, var(--neutral-700) ${perc})`
@@ -66,19 +76,19 @@ function randomId() {
 
 // starts sending progress requests to "/internal/progress" uri, creating progressbar above progressbarContainer element and preview inside gallery element
 // Cleans up all created stuff when the task is over and calls atEnd. calls onProgress every time there is a progress update
-function requestProgress(id_task, gallery, atEnd = null, onProgress = null, once = false) {
+function requestProgress(id_task, progressEl, galleryEl, atEnd = null, onProgress = null, once = false) {
   localStorage.setItem('task', id_task);
   let hasStarted = false;
   const dateStart = new Date();
   const prevProgress = null;
-  const parentGallery = gallery ? gallery.parentNode : null;
+  const parentGallery = galleryEl ? galleryEl.parentNode : null;
   let livePreview;
   const img = new Image();
   if (parentGallery) {
     livePreview = document.createElement('div');
     livePreview.className = 'livePreview';
-    parentGallery.insertBefore(livePreview, gallery);
-    const rect = gallery.getBoundingClientRect();
+    parentGallery.insertBefore(livePreview, galleryEl);
+    const rect = galleryEl.getBoundingClientRect();
     if (rect.width) {
       livePreview.style.width = `${rect.width}px`;
       livePreview.style.height = `${rect.height}px`;
@@ -108,7 +118,7 @@ function requestProgress(id_task, gallery, atEnd = null, onProgress = null, once
         return;
       }
       setProgress(res);
-      if (res.live_preview && gallery) img.src = res.live_preview;
+      if (res.live_preview && galleryEl) img.src = res.live_preview;
       if (onProgress) onProgress(res);
       setTimeout(() => start(id_task, res.id_live_preview), opts.live_preview_refresh_period || 250);
     }, done);

javascript/ui.js

@@ -118,21 +118,23 @@ function create_submit_args(args) {
   return res;
 }
 
+function showSubmitButtons(tabname, show) {}
+
 function submit(...args) {
-  console.log('submit txt2img:', args);
+  console.log('Submit txt2img:', args);
   rememberGallerySelection('txt2img_gallery');
   const id = randomId();
-  requestProgress(id, gradioApp().getElementById('txt2img_gallery'));
+  requestProgress(id, null, gradioApp().getElementById('txt2img_gallery'));
   const res = create_submit_args(args);
   res[0] = id;
   return res;
 }
 
 function submit_img2img(...args) {
-  console.log('submit img2img:', args);
+  console.log('Submit img2img:', args);
   rememberGallerySelection('img2img_gallery');
   const id = randomId();
-  requestProgress(id, gradioApp().getElementById('img2img_gallery'));
+  requestProgress(id, null, gradioApp().getElementById('img2img_gallery'));
   const res = create_submit_args(args);
   res[0] = id;
   res[1] = get_tab_index('mode_img2img');
@@ -424,7 +426,7 @@ function reconnect_ui() {
   if (task_id) {
     console.debug('task check:', task_id);
     rememberGallerySelection('txt2img_gallery');
-    requestProgress(task_id, gallery, null, null, true);
+    requestProgress(task_id, null, gallery, null, null, true);
   }
 
   const sd_model = gradioApp().getElementById('setting_sd_model_checkpoint');

modules/images.py

@@ -465,14 +465,10 @@ def atomically_save_image():
         if shared.opts.save_log_fn != '' and len(exifinfo_data) > 0:
             try:
                 with open(os.path.join(paths.data_path, shared.opts.save_log_fn), mode='a+', encoding='utf-8') as f:
-                    try:
-                        entries = json.load(f)
-                    except:
-                        entries = []
-                    f.seek(0)
-                    entries.append({ 'filename': filename, 'time': datetime.datetime.now().isoformat(), 'info': exifinfo_data })
-                    json.dump(entries, f, indent=4)
-                    del entries
+                    entry = { 'filename': filename, 'time': datetime.datetime.now().isoformat(), 'info': exifinfo_data }
+                    json.dump(entry, f)
+                    f.write(os.linesep)
+                    shared.log.debug(f'Log file updated: {os.path.join(paths.data_path, shared.opts.save_log_fn)}')
             except Exception as e:
                 shared.log.warning(f'Failed to save log file: {shared.opts.save_log_fn} {e}')
         save_queue.task_done()

modules/import_hook.py

@@ -1,15 +0,0 @@
-import sys
-from modules.shared import opts, log
-
-# this will break any attempt to import xformers which will prevent stability diffusion repo from trying to use it
-try:
-    import xformers # pylint: disable=unused-import, import-error
-    import xformers.ops # pylint: disable=unused-import, import-error
-except:
-    pass
-
-if opts.cross_attention_optimization != "xFormers":
-    if sys.modules.get("xformers", None) is not None:
-        log.info('Unloading xFormers')
-    sys.modules["xformers"] = None
-    sys.modules["xformers.ops"] = None

modules/processing.py

@@ -147,6 +147,7 @@ class StableDiffusionProcessing:
         self.is_hr_pass = False
         opts.data['clip_skip'] = clip_skip
 
+
     @property
     def sd_model(self):
         return shared.sd_model

modules/sd_hijack.py

@@ -1,3 +1,4 @@
+import sys
 from types import MethodType
 import torch
 from torch.nn.functional import silu

modules/sd_hijack_optimizations.py

@@ -1,3 +1,4 @@
+import sys
 import math
 import psutil
 
@@ -19,6 +20,11 @@ if shared.opts.cross_attention_optimization == "xFormers":
         shared.xformers_available = True
     except Exception:
         pass
+else:
+    if sys.modules.get("xformers", None) is not None:
+        shared.log.debug('Unloading xFormers')
+    sys.modules["xformers"] = None
+    sys.modules["xformers.ops"] = None
 
 
 def get_available_vram():

modules/shared.py

@@ -331,7 +331,7 @@ options_templates.update(options_section(('saving-images', "Image Options"), {
     "grid_prevent_empty_spots": OptionInfo(True, "Prevent empty spots in grid (when set to autodetect)"),
     "n_rows": OptionInfo(-1, "Grid row count; use -1 for autodetect and 0 for it to be same as batch size", gr.Slider, {"minimum": -1, "maximum": 16, "step": 1}),
     "save_txt": OptionInfo(False, "Create a text file next to every image with generation parameters"),
-    "save_log_fn": OptionInfo("", "Create a log file with image information for each saved image", component_args=hide_dirs),
+    "save_log_fn": OptionInfo("", "Create a JSON log file with image information for each saved image", component_args=hide_dirs),
     "save_images_before_face_restoration": OptionInfo(False, "Save a copy of image before doing face restoration"),
     "save_images_before_highres_fix": OptionInfo(False, "Save a copy of image before applying highres fix"),
     "save_images_before_color_correction": OptionInfo(False, "Save a copy of image before applying color correction to img2img results"),
@@ -484,7 +484,7 @@ options_templates.update(options_section(('ui', "Live previews"), {
 }))
 
 options_templates.update(options_section(('sampler-params', "Sampler parameters"), {
-    "show_samplers": OptionInfo(["Euler a", "UniPC", "DDIM", "DPM++ SDE", "DPM++ SDE", "DPM2 Karras", "DPM++ 2M Karras"], "Show samplers in user interface", gr.CheckboxGroup, lambda: {"choices": [x.name for x in list_samplers() if x.name != "PLMS"]}),
+    "show_samplers": OptionInfo(["Euler a", "UniPC", "DDIM", "DPM++ 2M SDE", "DPM++ 2M SDE Karras", "DPM2 Karras", "DPM++ 2M Karras"], "Show samplers in user interface", gr.CheckboxGroup, lambda: {"choices": [x.name for x in list_samplers() if x.name != "PLMS"]}),
     "fallback_sampler": OptionInfo("Euler a", "Secondary sampler", gr.Dropdown, lambda: {"choices": ["None"] + [x.name for x in list_samplers()]}),
     "xyz_fallback_sampler": OptionInfo("None", "Force latent upscaler sampler", gr.Dropdown, lambda: {"choices": ["None"] + [x.name for x in list_samplers()]}),
     "eta_ancestral": OptionInfo(1.0, "Noise multiplier for ancestral samplers (eta)", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}),

webui.py

@@ -27,7 +27,6 @@ warnings.filterwarnings(action="ignore", category=FutureWarning)
 warnings.filterwarnings(action="ignore", category=UserWarning, module="torchvision")
 startup_timer.record("torch")
 
-from modules import import_hook # pylint: disable=W0611,C0411,C0412
 from fastapi import FastAPI # pylint: disable=W0611,C0411
 import gradio # pylint: disable=W0611,C0411
 startup_timer.record("gradio")