From dd25f2f515e1ec19401cac2aedb7202aba57fa96 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 15 Apr 2023 17:22:58 -0400 Subject: [PATCH] update setup --- .gitignore | 1 + .pylintrc | 2 + extensions-builtin/sd-extension-system-info | 2 +- extensions-builtin/sd-webui-controlnet | 2 +- launch.py | 1 + modules/api/api.py | 51 +++++++++++---------- modules/errors.py | 23 ++++++---- modules/shared.py | 2 +- modules/ui.py | 1 - setup.py | 32 +++++++++++-- webui.py | 12 +++-- 11 files changed, 81 insertions(+), 48 deletions(-) diff --git a/.gitignore b/.gitignore index df9ac9f36..ad2a58b22 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ __pycache__ /params.txt /cache.json /config.json +/ui-config.json /setup.log venv diff --git a/.pylintrc b/.pylintrc index 063e76202..2a66d3dfc 100644 --- a/.pylintrc +++ b/.pylintrc @@ -136,6 +136,8 @@ disable=raw-checker-failed, consider-iterating-dictionary, wrong-import-position, unnecessary-lambda, + consider-using-dict-items, + dangerous-default-value, enable=c-extension-no-member [METHOD_ARGS] diff --git a/extensions-builtin/sd-extension-system-info b/extensions-builtin/sd-extension-system-info index d88a80289..0254985c8 160000 --- a/extensions-builtin/sd-extension-system-info +++ b/extensions-builtin/sd-extension-system-info @@ -1 +1 @@ -Subproject commit d88a80289b6d252b695973d12dd6fd75f9c30829 +Subproject commit 0254985c819240fc7e065870828d3e38948c0dd7 diff --git a/extensions-builtin/sd-webui-controlnet b/extensions-builtin/sd-webui-controlnet index d575ccf5d..c9a88e5b6 160000 --- a/extensions-builtin/sd-webui-controlnet +++ b/extensions-builtin/sd-webui-controlnet @@ -1 +1 @@ -Subproject commit d575ccf5d63358d637224ead6ae62e944585cf4e +Subproject commit c9a88e5b66c396ae5ebe99528df25a7b9c924cc6 diff --git a/launch.py b/launch.py index 953b9271c..8b63d18a8 100644 --- a/launch.py +++ b/launch.py @@ -89,5 +89,6 @@ if __name__ == "__main__": setup.set_environment() setup.check_torch() setup.log.info(f"Server arguments: {sys.argv[1:]}") + setup.log.debug('Starting WebUI') import webui webui.webui() diff --git a/modules/api/api.py b/modules/api/api.py index 839e86f0c..fd93e4bf8 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -1,34 +1,37 @@ -import base64 import io import time +import base64 import datetime -import uvicorn -import gradio as gr -from threading import Lock from io import BytesIO -from gradio.processing_utils import decode_base64_to_file -# from gradio_client.utils import decode_base64_to_file +from typing import List +from threading import Lock +from secrets import compare_digest +import anyio +import starlette +import fastapi from fastapi import APIRouter, Depends, FastAPI, Request, Response from fastapi.security import HTTPBasic, HTTPBasicCredentials from fastapi.exceptions import HTTPException from fastapi.responses import JSONResponse from fastapi.encoders import jsonable_encoder -from secrets import compare_digest +from PIL import PngImagePlugin,Image +import piexif +import piexif.helper +import uvicorn +import gradio as gr +from gradio.processing_utils import decode_base64_to_file +# from gradio_client.utils import decode_base64_to_file -from modules import shared, errors, sd_samplers, deepbooru, sd_hijack, images, scripts, ui, postprocessing +from modules import errors, shared, sd_samplers, deepbooru, sd_hijack, images, scripts, ui, postprocessing from modules.api.models import * from modules.processing import StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img, process_images from modules.textual_inversion.textual_inversion import create_embedding, train_embedding from modules.textual_inversion.preprocess import preprocess from modules.hypernetworks.hypernetwork import create_hypernetwork, train_hypernetwork -from PIL import PngImagePlugin,Image from modules.sd_models import checkpoints_list, unload_model_weights, reload_model_weights from modules.sd_models_config import find_checkpoint_config_near_filename from modules.realesrgan_model import get_realesrgan_models from modules import devices -from typing import List -import piexif -import piexif.helper def upscaler_to_index(name: str): try: @@ -61,7 +64,7 @@ def decode_base64_to_image(encoding): try: image = Image.open(BytesIO(base64.b64decode(encoding))) return image - except Exception as err: + except Exception: raise HTTPException(status_code=500, detail="Invalid encoded image") def encode_pil_to_base64(image): @@ -124,7 +127,7 @@ def api_middleware(app: FastAPI): } print(f"API error: {request.method}: {request.url} {err}") if not isinstance(e, HTTPException): # do not print backtrace on known httpexceptions - errors.display(e, 'http api') + errors.display(e, 'http api', [anyio, fastapi, uvicorn, starlette]) return JSONResponse(status_code=vars(e).get('status_code', 500), content=jsonable_encoder(err)) @app.middleware("http") @@ -209,17 +212,17 @@ class Api: script_idx = script_name_to_index(script_name, script_runner.selectable_scripts) script = script_runner.selectable_scripts[script_idx] return script, script_idx - + def get_scripts_list(self): t2ilist = [str(title.lower()) for title in scripts.scripts_txt2img.titles] i2ilist = [str(title.lower()) for title in scripts.scripts_img2img.titles] - return ScriptsList(txt2img = t2ilist, img2img = i2ilist) + return ScriptsList(txt2img = t2ilist, img2img = i2ilist) def get_script(self, script_name, script_runner): if script_name is None or script_name == "": return None, None - + script_idx = script_name_to_index(script_name, script_runner.scripts) return script_runner.scripts[script_idx] @@ -254,10 +257,10 @@ class Api: if request.alwayson_scripts and (len(request.alwayson_scripts) > 0): for alwayson_script_name in request.alwayson_scripts.keys(): alwayson_script = self.get_script(alwayson_script_name, script_runner) - if alwayson_script == None: + if alwayson_script is None: raise HTTPException(status_code=422, detail=f"always on script {alwayson_script_name} not found") # Selectable script in always on script param check - if alwayson_script.alwayson == False: + if not alwayson_script.alwayson: raise HTTPException(status_code=422, detail=f"Cannot have a selectable script in the always on scripts params") # always on script with no arg should always run so you don't really need to add them to the requests if "args" in request.alwayson_scripts[alwayson_script_name]: @@ -298,7 +301,7 @@ class Api: p.outpath_samples = opts.outdir_txt2img_samples shared.state.begin() - if selectable_scripts != None: + if selectable_scripts is not None: p.script_args = script_args processed = scripts.scripts_txt2img.run(p, *p.script_args) # Need to pass args as list here else: @@ -355,7 +358,7 @@ class Api: p.outpath_samples = opts.outdir_img2img_samples shared.state.begin() - if selectable_scripts != None: + if selectable_scripts is not None: p.script_args = script_args processed = scripts.scripts_img2img.run(p, *p.script_args) # Need to pass args as list here else: @@ -398,7 +401,7 @@ class Api: return ExtrasBatchImagesResponse(images=list(map(encode_pil_to_base64, result[0])), html_info=result[1]) def pnginfoapi(self, req: PNGInfoRequest): - if(not req.image.strip()): + if not req.image.strip(): return PNGInfoResponse(info="") image = decode_base64_to_image(req.image.strip()) @@ -428,7 +431,7 @@ class Api: progress += 1 / shared.state.job_count * shared.state.sampling_step / shared.state.sampling_steps time_since_start = time.time() - shared.state.time_start - eta = (time_since_start/progress) + eta = time_since_start / progress eta_relative = eta-time_since_start progress = min(progress, 1) @@ -482,7 +485,7 @@ class Api: options = {} for key in shared.opts.data.keys(): metadata = shared.opts.data_labels.get(key) - if(metadata is not None): + if metadata is not None: options.update({key: shared.opts.data.get(key, shared.opts.data_labels.get(key).default)}) else: options.update({key: shared.opts.data.get(key, None)}) diff --git a/modules/errors.py b/modules/errors.py index e40389d88..abd83cd41 100644 --- a/modules/errors.py +++ b/modules/errors.py @@ -1,7 +1,5 @@ import sys -import anyio -import starlette -import gradio +import logging from rich import print # pylint: disable=redefined-builtin from rich.console import Console from rich.theme import Theme @@ -13,13 +11,18 @@ console = Console(log_time=True, log_time_format='%H:%M:%S-%f', theme=Theme({ "traceback.border.syntax_error": "black", "inspect.value.border": "black", })) + pretty_install(console=console) -traceback_install(console=console, extra_lines=1, width=console.width, word_wrap=False, indent_guides=False, suppress=[anyio, starlette, gradio]) +traceback_install(console=console, extra_lines=1, width=console.width, word_wrap=False, indent_guides=False) already_displayed = {} -def install(): + +def install(suppress=[]): pretty_install(console=console) - traceback_install(console=console, extra_lines=1, width=console.width, word_wrap=False, indent_guides=False, suppress=[anyio, starlette, gradio]) + traceback_install(console=console, extra_lines=1, width=console.width, word_wrap=False, indent_guides=False, suppress=suppress) + logging.basicConfig(level=logging.INFO, format='%(asctime)s | %(levelname)s | %(pathname)s | %(message)s') + for handler in logging.getLogger().handlers: + handler.setLevel(logging.INFO) def print_error_explanation(message): @@ -28,9 +31,9 @@ def print_error_explanation(message): print(line, file=sys.stderr) -def display(e: Exception, task): +def display(e: Exception, task, suppress=[]): print(f"{task or 'error'}: {type(e).__name__}", file=sys.stderr) - console.print_exception(show_locals=False, max_frames=2, extra_lines=1, suppress=[anyio, starlette, gradio], theme="ansi_dark", word_wrap=False, width=min([console.width, 200])) + console.print_exception(show_locals=False, max_frames=2, extra_lines=1, suppress=suppress, theme="ansi_dark", word_wrap=False, width=min([console.width, 200])) def display_once(e: Exception, task): @@ -47,5 +50,5 @@ def run(code, task): display(e, task) -def exception(): - console.print_exception(show_locals=False, max_frames=10, extra_lines=2, suppress=[anyio, starlette, gradio], theme="ansi_dark", word_wrap=False, width=min([console.width, 200])) +def exception(suppress=[]): + console.print_exception(show_locals=False, max_frames=10, extra_lines=2, suppress=suppress, theme="ansi_dark", word_wrap=False, width=min([console.width, 200])) diff --git a/modules/shared.py b/modules/shared.py index 45a57a639..a75b58fad 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -16,7 +16,7 @@ from modules.paths_internal import models_path, script_path, data_path, sd_confi import modules.paths_internal as paths from setup import log as setup_log # pylint: disable=E0611 -errors.install() +errors.install(gr) demo = None log = setup_log diff --git a/modules/ui.py b/modules/ui.py index 8f5b191e0..27f7dabf8 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -365,7 +365,6 @@ def create_output_panel(tabname, outdir): def create_sampler_and_steps_selection(choices, tabname): with FormRow(elem_id=f"sampler_selection_{tabname}"): - print(tabname, [x.name for x in choices]) sampler_index = gr.Dropdown(label='Sampling method', elem_id=f"{tabname}_sampling", choices=[x.name for x in choices], value="UniPC" if tabname == 'txt2img' else "Euler a", type="index") steps = gr.Slider(minimum=1, maximum=150, step=1, elem_id=f"{tabname}_steps", label="Sampling steps", value=10 if tabname == 'txt2img' else 20) diff --git a/setup.py b/setup.py index 45ed40dcd..baead2144 100644 --- a/setup.py +++ b/setup.py @@ -17,12 +17,13 @@ class Dot(dict): # dot notation access to dictionary attributes log = logging.getLogger("sd") args = Dot({ 'debug': False, 'upgrade': False, 'noupdate': False, 'skip-extensions': False, 'skip-requirements': False, 'reset': False }) quick_allowed = True +errors = 0 # setup console and file logging -def setup_logging(): +def setup_logging(clean=False): try: - if os.path.isfile('setup.log'): + if clean and os.path.isfile('setup.log'): os.remove('setup.log') time.sleep(0.1) # prevent race condition except: @@ -49,6 +50,7 @@ def setup_logging(): log.addHandler(sh) +# check if package is installed def installed(package): import pkg_resources ok = True @@ -76,6 +78,7 @@ def installed(package): log.debug(f"Package not installed: {pkgs}") return False + # install package using pip if not already installed def install(package): def pip(arg: str): @@ -85,6 +88,8 @@ def install(package): if len(result.stderr) > 0: txt = txt + '\n' + result.stderr.decode(encoding="utf8", errors="ignore") if result.returncode != 0: + global errors # pylint: disable=global-statement + errors += 1 log.error(f'Error running pip with args: {arg}') log.debug(f'Pip output: {txt}') return txt @@ -101,6 +106,8 @@ def git(arg: str): if len(result.stderr) > 0: txt = txt + '\n' + result.stderr.decode(encoding="utf8", errors="ignore") if result.returncode != 0: + global errors # pylint: disable=global-statement + errors += 1 log.error(f'Error running git with args: {arg}') log.debug(f'Git output: {txt}') return txt @@ -223,6 +230,8 @@ def run_extension_installer(folder): env['PYTHONPATH'] = os.path.abspath(".") result = subprocess.run(f'"{sys.executable}" "{path_installer}"', shell=True, env=env, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if result.returncode != 0: + global errors # pylint: disable=global-statement + errors += 1 txt = result.stdout.decode(encoding="utf8", errors="ignore") if len(result.stderr) > 0: txt = txt + '\n' + result.stderr.decode(encoding="utf8", errors="ignore") @@ -355,10 +364,19 @@ def check_timestamp(): return False if not os.path.isfile('setup.log'): return False - setup_time = os.path.getmtime('setup.log') - log.debug(f'Previous setup time: {time.ctime(setup_time)}') + setup_time = -1 + with open('setup.log', 'r', encoding='utf8') as f: + lines = f.readlines() + for line in lines: + if 'Setup complete without errors' in line: + setup_time = line.split(' ')[0] + return True + # setup_time = os.path.getmtime('setup.log') version_time = int(git('log -1 --pretty=format:"%at"')) log.debug(f'Repository update time: {time.ctime(int(version_time))}') + if setup_time == -1: + return False + log.debug(f'Previous setup time: {time.ctime(setup_time)}') if setup_time < version_time: return False extension_time = check_extensions() @@ -395,7 +413,7 @@ def git_reset(): # entry method when used as module def run_setup(): - setup_logging() + setup_logging(args.upgrade) check_python() if args.reset: git_reset() @@ -410,6 +428,10 @@ def run_setup(): install_repositories() install_submodules() install_extensions() + if errors == 0: + log.debug(f'Setup complete without errors: {round(time.time())}') + else: + log.warning(f'Setup complete with errors ({errors})') if __name__ == "__main__": diff --git a/webui.py b/webui.py index 22384c0b5..5aeb760ff 100644 --- a/webui.py +++ b/webui.py @@ -1,21 +1,20 @@ import os +import re import time import signal -import re -import logging import warnings -from fastapi.middleware.cors import CORSMiddleware -from fastapi.middleware.gzip import GZipMiddleware +import logging from setup import log from modules import timer, errors -errors.install() startup_timer = timer.Timer() +logging.basicConfig(level=logging.WARNING, format='%(asctime)s | %(levelname)s | %(pathname)s | %(message)s', force=True) # reset logging import torch # pylint: disable=C0411 import torchvision # pylint: disable=W0611,C0411 import pytorch_lightning # pytorch_lightning should be imported after torch, but it re-enables warnings on import so import once to disable them # pylint: disable=W0611,C0411 logging.getLogger("xformers").addFilter(lambda record: 'A matching Triton is not available' not in record.getMessage()) +logging.getLogger("pytorch_lightning").disabled = True warnings.filterwarnings(action="ignore", category=DeprecationWarning, module="pytorch_lightning") warnings.filterwarnings(action="ignore", category=UserWarning, module="torchvision") startup_timer.record("torch") @@ -23,6 +22,7 @@ startup_timer.record("torch") from modules import import_hook # pylint: disable=W0611,C0411,C0412 import gradio # pylint: disable=W0611,C0411 startup_timer.record("gradio") +errors.install([gradio]) import ldm.modules.encoders.modules # pylint: disable=W0611,C0411 from modules import extra_networks, ui_extra_networks_checkpoints # pylint: disable=C0411,C0412 @@ -144,6 +144,8 @@ def load_model(): def setup_middleware(app): + from fastapi.middleware.cors import CORSMiddleware + from fastapi.middleware.gzip import GZipMiddleware app.middleware_stack = None # reset current middleware to allow modifying user provided list app.add_middleware(GZipMiddleware, minimum_size=1024) if cmd_opts.cors_origins and cmd_opts.cors_regex: