From 4c98b8d1c7f677284c158df84e08b04fa80c01b3 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 30 Sep 2023 17:17:57 -0400 Subject: [PATCH] minor patches and nvml cli --- CHANGELOG.md | 2 ++ cli/nvml.py | 45 +++++++++++++++++++++++++++++++++ cli/simple-txt2img.js | 8 +++--- installer.py | 16 ++++++++++-- launch.py | 5 +++- modules/ui.py | 1 + modules/ui_extra_networks.py | 49 ++++++++++++++++++++++++------------ webui.py | 9 +++++-- 8 files changed, 111 insertions(+), 24 deletions(-) create mode 100755 cli/nvml.py mode change 100644 => 100755 cli/simple-txt2img.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 26af780b1..85c618ed3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -131,6 +131,8 @@ Upgrades are still possible and supported, but above is recommended for best exp for example `--data-dir ` can be specified as `SD_DATADIR=` before starting SD.Next - **Logging** - get browser session info in server log + - allow custom log file destination + see `webui --log` - when running with `--debug` flag, log is force-rotated so each `sdnext.log.*` represents exactly one server run - internal server job state tracking diff --git a/cli/nvml.py b/cli/nvml.py new file mode 100755 index 000000000..12a8da35d --- /dev/null +++ b/cli/nvml.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python + +import logging +import pynvml as nv +from rich.pretty import install + + +install() +logging.basicConfig(level = logging.INFO, format = '%(asctime)s %(levelname)s: %(message)s') +log = logging.getLogger(__name__) + +def get_reason(val): + throttle = { + 1: 'gpu idle', + 2: 'applications clocks setting', + 4: 'sw power cap', + 8: 'hw slowdown', + 16: 'sync boost', + 32: 'sw thermal slowdown', + 64: 'hw thermal slowdown', + 128: 'hw power brake slowdown', + 256: 'display clock setting', + } + reason = ', '.join([throttle[i] for i in throttle if i & val]) + return reason if len(reason) > 0 else 'none' + +def main(): + nv.nvmlInit() + log.info(f"version cuda={nv.nvmlSystemGetCudaDriverVersion()} driver={nv.nvmlSystemGetDriverVersion()}") + for i in range(nv.nvmlDeviceGetCount()): + dev = nv.nvmlDeviceGetHandleByIndex(i) + log.info(f"device#{i}: {nv.nvmlDeviceGetName(dev)}") + log.info(f" version vbios={nv.nvmlDeviceGetVbiosVersion(dev)} rom={nv.nvmlDeviceGetInforomImageVersion(dev)}") + log.info(f" cuda capabilities: {nv.nvmlDeviceGetCudaComputeCapability(dev)}") + log.info(f" pci link={nv.nvmlDeviceGetCurrPcieLinkGeneration(dev)} width={nv.nvmlDeviceGetCurrPcieLinkWidth(dev)} busid={nv.nvmlDeviceGetPciInfo(dev).busId} deviceid={nv.nvmlDeviceGetPciInfo(dev).pciDeviceId}") + log.info(f" memory total={round(nv.nvmlDeviceGetMemoryInfo(dev).total/1024/1024, 2)}Mb free={round(nv.nvmlDeviceGetMemoryInfo(dev).free/1024/1024,2)}Mb used={round(nv.nvmlDeviceGetMemoryInfo(dev).used/1024/1024,2)}Mb") + log.info(f" clock graphics={nv.nvmlDeviceGetClockInfo(dev, 0)}Mhz sm={nv.nvmlDeviceGetClockInfo(dev, 1)}Mhz memory={nv.nvmlDeviceGetClockInfo(dev, 2)}Mhz") + log.info(f" utilization gpu={nv.nvmlDeviceGetUtilizationRates(dev).gpu}% memory={nv.nvmlDeviceGetUtilizationRates(dev).memory}% temp={nv.nvmlDeviceGetTemperature(dev, 0)}C fan={nv.nvmlDeviceGetFanSpeed(dev)}%") + log.info(f" power usage={round(nv.nvmlDeviceGetPowerUsage(dev)/1000, 2)}w limit={round(nv.nvmlDeviceGetEnforcedPowerLimit(dev)/1000, 2)}w energy={round(nv.nvmlDeviceGetTotalEnergyConsumption(dev)/1000000000, 2)}Mj") + log.info(f" throttle={get_reason(nv.nvmlDeviceGetCurrentClocksThrottleReasons(dev))} state power={nv.nvmlDeviceGetPowerState(dev)} performance={nv.nvmlDeviceGetPerformanceState(dev)}") + nv.nvmlShutdown() + + +if __name__ == "__main__": + main() diff --git a/cli/simple-txt2img.js b/cli/simple-txt2img.js old mode 100644 new mode 100755 index 0519b6df0..34fb9a843 --- a/cli/simple-txt2img.js +++ b/cli/simple-txt2img.js @@ -1,7 +1,9 @@ +#!/usr/bin/env node + // simple nodejs script to test sdnext api -const fs = require('fs'); -const process = require('process'); +const fs = require('fs'); // eslint-disable-line no-undef +const process = require('process'); // eslint-disable-line no-undef const sd_url = process.env.SDAPI_URL || 'http://127.0.0.1:7860'; const sd_username = process.env.SDAPI_USR; @@ -50,7 +52,7 @@ async function main() { } else { const json = await res.json(); console.log('result:', json.info); - for (const i in json.images) { + for (const i in json.images) { // eslint-disable-line guard-for-in const f = `/tmp/test-{${i}.jpg`; fs.writeFileSync(f, atob(json.images[i]), 'binary'); console.log('image saved:', f); diff --git a/installer.py b/installer.py index fe4690565..1bc3aafa4 100644 --- a/installer.py +++ b/installer.py @@ -21,6 +21,7 @@ class Dot(dict): # dot notation access to dictionary attributes log = logging.getLogger("sd") log_file = os.path.join(os.path.dirname(__file__), 'sdnext.log') log_rolled = False +first_call = True quick_allowed = True errors = 0 opts = {} @@ -73,6 +74,10 @@ def setup_logging(): from rich.pretty import install as pretty_install from rich.traceback import install as traceback_install + if args.log: + global log_file # pylint: disable=global-statement + log_file = args.log + level = logging.DEBUG if args.debug else logging.INFO log.setLevel(logging.DEBUG) # log to file is always at level debug for facility `sd` console = Console(log_time=True, log_time_format='%H:%M:%S-%f', theme=Theme({ @@ -93,10 +98,16 @@ def setup_logging(): fh = RotatingFileHandler(log_file, maxBytes=32*1024*1024, backupCount=9, encoding='utf-8', delay=True) # 10MB default for log rotation global log_rolled # pylint: disable=global-statement - if not log_rolled and args.debug: + if not log_rolled and args.debug and not args.log: fh.doRollover() - log.debug(f'Logger: file={log_file} level={level}') log_rolled = True + + global first_call # pylint: disable=global-statement + if first_call: + log_size = os.path.getsize(log_file) if os.path.exists(log_file) else 0 + log.debug(f'Logger: file={log_file} level={level} size={log_size} mode={"append" if not log_rolled else "create"}') + first_call = False + fh.formatter = logging.Formatter('%(asctime)s | %(name)s | %(levelname)s | %(module)s | %(message)s') fh.setLevel(logging.DEBUG) log.addHandler(fh) @@ -886,6 +897,7 @@ def check_timestamp(): def add_args(parser): group = parser.add_argument_group('Setup options') + group.add_argument("--log", type=str, default=os.environ.get("SD_LOG", None), help="Set log file, default: %(default)s") group.add_argument('--debug', default = os.environ.get("SD_DEBUG",False), action='store_true', help = "Run installer with debug logging, default: %(default)s") group.add_argument('--reset', default = os.environ.get("SD_RESET",False), action='store_true', help = "Reset main repository to latest version, default: %(default)s") group.add_argument('--upgrade', default = os.environ.get("SD_UPGRADE",False), action='store_true', help = "Upgrade main repository to latest version, default: %(default)s") diff --git a/launch.py b/launch.py index cb6bc85de..52452136b 100644 --- a/launch.py +++ b/launch.py @@ -165,7 +165,10 @@ if __name__ == "__main__": installer.args = args installer.setup_logging() installer.log.info('Starting SD.Next') - sys.excepthook = installer.custom_excepthook + try: + sys.excepthook = installer.custom_excepthook + except Exception: + pass installer.read_options() if args.skip_all: args.quick = True diff --git a/modules/ui.py b/modules/ui.py index 59870aa44..58de042b9 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -380,6 +380,7 @@ def create_ui(startup_timer = None): with FormRow(variant='compact', elem_id="txt2img_extra_networks", visible=False) as extra_networks_ui: from modules import ui_extra_networks extra_networks_ui = ui_extra_networks.create_ui(extra_networks_ui, extra_networks_button, 'txt2img', skip_indexing=opts.extra_network_skip_indexing) + timer.startup.record('ui-extra-networks') with gr.Row().style(equal_height=False, elem_id="txt2img_interface"): with gr.Column(variant='compact', elem_id="txt2img_settings"): diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index 8f9263003..a374a6ccc 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -19,7 +19,7 @@ import modules.ui_symbols as symbols allowed_dirs = [] dir_cache = {} # key=path, value=(mtime, listdir(path)) -refresh_time = None +refresh_time = 0 extra_pages = shared.extra_networks @@ -108,7 +108,9 @@ class ExtraNetworksPage: self.html = '' self.items = [] self.missing_thumbs = [] - self.refresh_time = None + self.refresh_time = 0 + self.page_time = 0 + self.list_time = 0 # class additional is to keep old extensions happy self.card = '''
@@ -187,10 +189,25 @@ class ExtraNetworksPage: shared.log.info(f"Extra network thumbnails: {self.name} created={created}") self.missing_thumbs.clear() - def create_page(self, tabname, skip = False): - if self.refresh_time is not None and self.refresh_time > refresh_time: # cached page - return self.html + def create_items(self, tabname): + if self.refresh_time is not None and self.refresh_time > refresh_time: # cached results + return t0 = time.time() + try: + self.items = list(self.list_items()) + self.refresh_time = time.time() + except Exception as e: + self.items = [] + shared.log.error(f'Extra networks error listing items: class={self.__class__} tab={tabname} {e}') + for item in self.items: + self.metadata[item["name"]] = item.get("metadata", {}) + t1 = time.time() + self.list_time = round(t1-t0, 2) + + + def create_page(self, tabname, skip = False): + if self.page_time > refresh_time: # cached page + return self.html self_name_id = self.name.replace(" ", "_") if skip: return f"
Extra network page not ready
Click refresh to try again
" @@ -211,24 +228,18 @@ class ExtraNetworksPage: subdirs_html = "
" subdirs_html += "".join([f"
" for subdir in subdirs if subdir != '']) self.html = '' - try: - self.items = list(self.list_items()) - self.refresh_time = time.time() - except Exception as e: - self.items = [] - shared.log.error(f'Extra networks error listing items: class={self.__class__} tab={tabname} {e}') + self.create_items(tabname) self.create_xyz_grid() htmls = [] for item in self.items: - self.metadata[item["name"]] = item.get("metadata", {}) htmls.append(self.create_html(item, tabname)) self.html += ''.join(htmls) + self.page_time = time.time() if len(subdirs_html) > 0 or len(self.html) > 0: self.html = f"
{subdirs_html}
{self.html}
" else: return '' - t1 = time.time() - shared.log.debug(f"Extra networks: page='{self.name}' items={len(self.items)} subdirs={len(subdirs)} tab={tabname} dirs={self.allowed_directories_for_previews()} time={round(t1-t0, 2)}") + shared.log.debug(f"Extra networks: page='{self.name}' items={len(self.items)} subdirs={len(subdirs)} tab={tabname} dirs={self.allowed_directories_for_previews()} time={self.list_time}s") threading.Thread(target=self.create_thumb).start() def list_items(self): @@ -464,13 +475,19 @@ def create_ui(container, button_parent, tabname, skip_indexing = False): if ui.tabname == 'txt2img': # refresh only once global refresh_time # pylint: disable=global-statement refresh_time = time.time() + threads = [] + for page in get_pages(): + # page.create_items(ui.tabname) + threads.append(threading.Thread(target=page.create_items, args=[ui.tabname])) + threads[-1].start() + for thread in threads: + thread.join() for page in get_pages(): page.create_page(ui.tabname, skip_indexing) with gr.Tab(page.title, id=page.title.lower().replace(" ", "_"), elem_classes="extra-networks-tab") as tab: hmtl = gr.HTML(page.html, elem_id=f'{tabname}{page.name}_extra_page', elem_classes="extra-networks-page") ui.pages.append(hmtl) tab.select(ui_tab_change, _js="getENActivePage", inputs=[ui.button_details], outputs=[ui.button_scan, ui.button_save, ui.button_model]) - # ui.tabs.change(fn=ui_tab_change, inputs=[], outputs=[ui.button_scan, ui.button_save]) def fn_save_img(): @@ -624,7 +641,7 @@ def create_ui(container, button_parent, tabname, skip_indexing = False): for page in get_pages(): if title is None or title == '' or title == page.title or len(page.html) == 0: page.refresh() - page.refresh_time = None + page.refresh_time = 0 page.create_page(ui.tabname) shared.log.debug(f"Refreshing Extra networks: page='{page.title}' items={len(page.items)} tab={ui.tabname}") pages.append(page.html) diff --git a/webui.py b/webui.py index 0c8335058..5ef2bd781 100644 --- a/webui.py +++ b/webui.py @@ -12,7 +12,7 @@ import modules.loader import torch # pylint: disable=wrong-import-order from modules import timer, errors, paths # pylint: disable=unused-import local_url = None -from installer import log, git_commit, custom_excepthook +from installer import log, git_commit import ldm.modules.encoders.modules # pylint: disable=W0611,C0411,E0401 from modules import shared, extensions, extra_networks, ui_tempdir, ui_extra_networks, modelloader # pylint: disable=ungrouped-imports from modules.paths import create_paths @@ -37,7 +37,12 @@ import modules.hypernetworks.hypernetwork from modules.middleware import setup_middleware -sys.excepthook = custom_excepthook +try: + from installer import custom_excepthook # pylint: disable=ungrouped-imports + sys.excepthook = custom_excepthook +except Exception: + pass + state = shared.state if not modules.loader.initialized: timer.startup.record("libraries")