minor patches and nvml cli

CHANGELOG.md

@@ -131,6 +131,8 @@ Upgrades are still possible and supported, but above is recommended for best exp
       for example `--data-dir <path>` can be specified as `SD_DATADIR=<path>` before starting SD.Next  
   - **Logging**
     - get browser session info in server log  
+    - allow custom log file destination  
+      see `webui --log`
     - when running with `--debug` flag, log is force-rotated  
       so each `sdnext.log.*` represents exactly one server run  
     - internal server job state tracking

cli/nvml.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+
+import logging
+import pynvml as nv
+from rich.pretty import install
+
+
+install()
+logging.basicConfig(level = logging.INFO, format = '%(asctime)s %(levelname)s: %(message)s')
+log = logging.getLogger(__name__)
+
+def get_reason(val):
+    throttle = {
+        1: 'gpu idle',
+        2: 'applications clocks setting',
+        4: 'sw power cap',
+        8: 'hw slowdown',
+        16: 'sync boost',
+        32: 'sw thermal slowdown',
+        64: 'hw thermal slowdown',
+        128: 'hw power brake slowdown',
+        256: 'display clock setting',
+    }
+    reason = ', '.join([throttle[i] for i in throttle if i & val])
+    return reason if len(reason) > 0 else 'none'
+
+def main():
+    nv.nvmlInit()
+    log.info(f"version cuda={nv.nvmlSystemGetCudaDriverVersion()} driver={nv.nvmlSystemGetDriverVersion()}")
+    for i in range(nv.nvmlDeviceGetCount()):
+        dev = nv.nvmlDeviceGetHandleByIndex(i)
+        log.info(f"device#{i}: {nv.nvmlDeviceGetName(dev)}")
+        log.info(f" version vbios={nv.nvmlDeviceGetVbiosVersion(dev)} rom={nv.nvmlDeviceGetInforomImageVersion(dev)}")
+        log.info(f" cuda capabilities: {nv.nvmlDeviceGetCudaComputeCapability(dev)}")
+        log.info(f" pci link={nv.nvmlDeviceGetCurrPcieLinkGeneration(dev)} width={nv.nvmlDeviceGetCurrPcieLinkWidth(dev)} busid={nv.nvmlDeviceGetPciInfo(dev).busId} deviceid={nv.nvmlDeviceGetPciInfo(dev).pciDeviceId}")
+        log.info(f" memory total={round(nv.nvmlDeviceGetMemoryInfo(dev).total/1024/1024, 2)}Mb free={round(nv.nvmlDeviceGetMemoryInfo(dev).free/1024/1024,2)}Mb used={round(nv.nvmlDeviceGetMemoryInfo(dev).used/1024/1024,2)}Mb")
+        log.info(f" clock graphics={nv.nvmlDeviceGetClockInfo(dev, 0)}Mhz sm={nv.nvmlDeviceGetClockInfo(dev, 1)}Mhz memory={nv.nvmlDeviceGetClockInfo(dev, 2)}Mhz")
+        log.info(f" utilization gpu={nv.nvmlDeviceGetUtilizationRates(dev).gpu}% memory={nv.nvmlDeviceGetUtilizationRates(dev).memory}% temp={nv.nvmlDeviceGetTemperature(dev, 0)}C fan={nv.nvmlDeviceGetFanSpeed(dev)}%")
+        log.info(f" power usage={round(nv.nvmlDeviceGetPowerUsage(dev)/1000, 2)}w limit={round(nv.nvmlDeviceGetEnforcedPowerLimit(dev)/1000, 2)}w energy={round(nv.nvmlDeviceGetTotalEnergyConsumption(dev)/1000000000, 2)}Mj")
+        log.info(f" throttle={get_reason(nv.nvmlDeviceGetCurrentClocksThrottleReasons(dev))} state power={nv.nvmlDeviceGetPowerState(dev)} performance={nv.nvmlDeviceGetPerformanceState(dev)}")
+    nv.nvmlShutdown()
+
+
+if __name__ == "__main__":
+    main()

cli/simple-txt2img.js

@@ -1,7 +1,9 @@
+#!/usr/bin/env node
+
 // simple nodejs script to test sdnext api
 
-const fs = require('fs');
-const process = require('process');
+const fs = require('fs'); // eslint-disable-line no-undef
+const process = require('process'); // eslint-disable-line no-undef
 
 const sd_url = process.env.SDAPI_URL || 'http://127.0.0.1:7860';
 const sd_username = process.env.SDAPI_USR;
@@ -50,7 +52,7 @@ async function main() {
   } else {
     const json = await res.json();
     console.log('result:', json.info);
-    for (const i in json.images) {
+    for (const i in json.images) { // eslint-disable-line guard-for-in
       const f = `/tmp/test-{${i}.jpg`;
       fs.writeFileSync(f, atob(json.images[i]), 'binary');
       console.log('image saved:', f);

installer.py

@@ -21,6 +21,7 @@ class Dot(dict): # dot notation access to dictionary attributes
 log = logging.getLogger("sd")
 log_file = os.path.join(os.path.dirname(__file__), 'sdnext.log')
 log_rolled = False
+first_call = True
 quick_allowed = True
 errors = 0
 opts = {}
@@ -73,6 +74,10 @@ def setup_logging():
     from rich.pretty import install as pretty_install
     from rich.traceback import install as traceback_install
 
+    if args.log:
+        global log_file # pylint: disable=global-statement
+        log_file = args.log
+
     level = logging.DEBUG if args.debug else logging.INFO
     log.setLevel(logging.DEBUG) # log to file is always at level debug for facility `sd`
     console = Console(log_time=True, log_time_format='%H:%M:%S-%f', theme=Theme({
@@ -93,10 +98,16 @@ def setup_logging():
 
     fh = RotatingFileHandler(log_file, maxBytes=32*1024*1024, backupCount=9, encoding='utf-8', delay=True) # 10MB default for log rotation
     global log_rolled # pylint: disable=global-statement
-    if not log_rolled and args.debug:
+    if not log_rolled and args.debug and not args.log:
         fh.doRollover()
-        log.debug(f'Logger: file={log_file} level={level}')
         log_rolled = True
+
+    global first_call # pylint: disable=global-statement
+    if first_call:
+        log_size = os.path.getsize(log_file) if os.path.exists(log_file) else 0
+        log.debug(f'Logger: file={log_file} level={level} size={log_size} mode={"append" if not log_rolled else "create"}')
+        first_call = False
+
     fh.formatter = logging.Formatter('%(asctime)s | %(name)s | %(levelname)s | %(module)s | %(message)s')
     fh.setLevel(logging.DEBUG)
     log.addHandler(fh)
@@ -886,6 +897,7 @@ def check_timestamp():
 
 def add_args(parser):
     group = parser.add_argument_group('Setup options')
+    group.add_argument("--log", type=str, default=os.environ.get("SD_LOG", None), help="Set log file, default: %(default)s")
     group.add_argument('--debug', default = os.environ.get("SD_DEBUG",False), action='store_true', help = "Run installer with debug logging, default: %(default)s")
     group.add_argument('--reset', default = os.environ.get("SD_RESET",False), action='store_true', help = "Reset main repository to latest version, default: %(default)s")
     group.add_argument('--upgrade', default = os.environ.get("SD_UPGRADE",False), action='store_true', help = "Upgrade main repository to latest version, default: %(default)s")

launch.py

@@ -165,7 +165,10 @@ if __name__ == "__main__":
     installer.args = args
     installer.setup_logging()
     installer.log.info('Starting SD.Next')
-    sys.excepthook = installer.custom_excepthook
+    try:
+        sys.excepthook = installer.custom_excepthook
+    except Exception:
+        pass
     installer.read_options()
     if args.skip_all:
         args.quick = True

modules/ui.py

@@ -380,6 +380,7 @@ def create_ui(startup_timer = None):
         with FormRow(variant='compact', elem_id="txt2img_extra_networks", visible=False) as extra_networks_ui:
             from modules import ui_extra_networks
             extra_networks_ui = ui_extra_networks.create_ui(extra_networks_ui, extra_networks_button, 'txt2img', skip_indexing=opts.extra_network_skip_indexing)
+            timer.startup.record('ui-extra-networks')
 
         with gr.Row().style(equal_height=False, elem_id="txt2img_interface"):
             with gr.Column(variant='compact', elem_id="txt2img_settings"):

modules/ui_extra_networks.py

@@ -19,7 +19,7 @@ import modules.ui_symbols as symbols
 
 allowed_dirs = []
 dir_cache = {} # key=path, value=(mtime, listdir(path))
-refresh_time = None
+refresh_time = 0
 extra_pages = shared.extra_networks
 
 
@@ -108,7 +108,9 @@ class ExtraNetworksPage:
         self.html = ''
         self.items = []
         self.missing_thumbs = []
-        self.refresh_time = None
+        self.refresh_time = 0
+        self.page_time = 0
+        self.list_time = 0
         # class additional is to keep old extensions happy
         self.card = '''
             <div class='card' onclick={card_click} title='{name}' data-tab='{tabname}' data-page='{page}' data-name='{name}' data-filename='{filename}' data-tags='{tags}'>
@@ -187,10 +189,25 @@ class ExtraNetworksPage:
             shared.log.info(f"Extra network thumbnails: {self.name} created={created}")
             self.missing_thumbs.clear()
 
-    def create_page(self, tabname, skip = False):
-        if self.refresh_time is not None and self.refresh_time > refresh_time: # cached page
-            return self.html
+    def create_items(self, tabname):
+        if self.refresh_time is not None and self.refresh_time > refresh_time: # cached results
+            return
         t0 = time.time()
+        try:
+            self.items = list(self.list_items())
+            self.refresh_time = time.time()
+        except Exception as e:
+            self.items = []
+            shared.log.error(f'Extra networks error listing items: class={self.__class__} tab={tabname} {e}')
+        for item in self.items:
+            self.metadata[item["name"]] = item.get("metadata", {})
+        t1 = time.time()
+        self.list_time = round(t1-t0, 2)
+
+
+    def create_page(self, tabname, skip = False):
+        if self.page_time > refresh_time: # cached page
+            return self.html
         self_name_id = self.name.replace(" ", "_")
         if skip:
             return f"<div id='{tabname}_{self_name_id}_subdirs' class='extra-network-subdirs'></div><div id='{tabname}_{self_name_id}_cards' class='extra-network-cards'>Extra network page not ready<br>Click refresh to try again</div>"
@@ -211,24 +228,18 @@ class ExtraNetworksPage:
         subdirs_html = "<button class='lg secondary gradio-button custom-button search-all' onclick='extraNetworksSearchButton(event)'>all</button><br>"
         subdirs_html += "".join([f"<button class='lg secondary gradio-button custom-button' onclick='extraNetworksSearchButton(event)'>{html.escape(subdir)}</button><br>" for subdir in subdirs if subdir != ''])
         self.html = ''
-        try:
-            self.items = list(self.list_items())
-            self.refresh_time = time.time()
-        except Exception as e:
-            self.items = []
-            shared.log.error(f'Extra networks error listing items: class={self.__class__} tab={tabname} {e}')
+        self.create_items(tabname)
         self.create_xyz_grid()
         htmls = []
         for item in self.items:
-            self.metadata[item["name"]] = item.get("metadata", {})
             htmls.append(self.create_html(item, tabname))
         self.html += ''.join(htmls)
+        self.page_time = time.time()
         if len(subdirs_html) > 0 or len(self.html) > 0:
             self.html = f"<div id='{tabname}_{self_name_id}_subdirs' class='extra-network-subdirs'>{subdirs_html}</div><div id='{tabname}_{self_name_id}_cards' class='extra-network-cards'>{self.html}</div>"
         else:
             return ''
-        t1 = time.time()
-        shared.log.debug(f"Extra networks: page='{self.name}' items={len(self.items)} subdirs={len(subdirs)} tab={tabname} dirs={self.allowed_directories_for_previews()} time={round(t1-t0, 2)}")
+        shared.log.debug(f"Extra networks: page='{self.name}' items={len(self.items)} subdirs={len(subdirs)} tab={tabname} dirs={self.allowed_directories_for_previews()} time={self.list_time}s")
         threading.Thread(target=self.create_thumb).start()
 
     def list_items(self):
@@ -464,13 +475,19 @@ def create_ui(container, button_parent, tabname, skip_indexing = False):
         if ui.tabname == 'txt2img': # refresh only once
             global refresh_time # pylint: disable=global-statement
             refresh_time = time.time()
+        threads = []
+        for page in get_pages():
+            # page.create_items(ui.tabname)
+            threads.append(threading.Thread(target=page.create_items, args=[ui.tabname]))
+            threads[-1].start()
+        for thread in threads:
+            thread.join()
         for page in get_pages():
             page.create_page(ui.tabname, skip_indexing)
             with gr.Tab(page.title, id=page.title.lower().replace(" ", "_"), elem_classes="extra-networks-tab") as tab:
                 hmtl = gr.HTML(page.html, elem_id=f'{tabname}{page.name}_extra_page', elem_classes="extra-networks-page")
                 ui.pages.append(hmtl)
                 tab.select(ui_tab_change, _js="getENActivePage", inputs=[ui.button_details], outputs=[ui.button_scan, ui.button_save, ui.button_model])
-
         # ui.tabs.change(fn=ui_tab_change, inputs=[], outputs=[ui.button_scan, ui.button_save])
 
     def fn_save_img():
@@ -624,7 +641,7 @@ def create_ui(container, button_parent, tabname, skip_indexing = False):
         for page in get_pages():
             if title is None or title == '' or title == page.title or len(page.html) == 0:
                 page.refresh()
-                page.refresh_time = None
+                page.refresh_time = 0
                 page.create_page(ui.tabname)
                 shared.log.debug(f"Refreshing Extra networks: page='{page.title}' items={len(page.items)} tab={ui.tabname}")
             pages.append(page.html)

webui.py

@@ -12,7 +12,7 @@ import modules.loader
 import torch # pylint: disable=wrong-import-order
 from modules import timer, errors, paths # pylint: disable=unused-import
 local_url = None
-from installer import log, git_commit, custom_excepthook
+from installer import log, git_commit
 import ldm.modules.encoders.modules # pylint: disable=W0611,C0411,E0401
 from modules import shared, extensions, extra_networks, ui_tempdir, ui_extra_networks, modelloader # pylint: disable=ungrouped-imports
 from modules.paths import create_paths
@@ -37,7 +37,12 @@ import modules.hypernetworks.hypernetwork
 from modules.middleware import setup_middleware
 
 
-sys.excepthook = custom_excepthook
+try:
+    from installer import custom_excepthook # pylint: disable=ungrouped-imports
+    sys.excepthook = custom_excepthook
+except Exception:
+    pass
+
 state = shared.state
 if not modules.loader.initialized:
     timer.startup.record("libraries")