From 773638202822ca7295306f53e1cbc02c7826faac Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 1 Jul 2025 13:12:21 -0400 Subject: [PATCH] major requirements refactor Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 14 +++++-- TODO.md | 5 ++- extensions-builtin/sdnext-modernui | 2 +- installer.py | 47 +++++++++++++++-------- launch.py | 1 + modules/face/insightface.py | 2 +- modules/loader.py | 22 +++++++++-- modules/pulid/__init__.py | 16 ++++++-- modules/pulid/pulid_sdxl.py | 2 +- modules/shared.py | 5 +-- modules/theme.py | 2 + repositories/ldm/models/diffusion/ddpm.py | 3 +- requirements.txt | 16 ++++---- scripts/infiniteyou_ext.py | 2 +- scripts/k_diff.py | 5 ++- scripts/pulid_ext.py | 2 +- webui.py | 2 + wiki | 2 +- 18 files changed, 99 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03f680dc6..a25ac0ab6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,17 @@ ## Update for 2025-07-01 -- **UI** - - major update to modernui layout - +- **UI** + - major update to modernui layout +- **Core** + - override `gradio` installer + - major refactoring of requirements and dependencies to unblock `numpy>=2.1.0` + - patch `insightface` + - patch `k-diffusion` + - better handle startup import errors +- **Fixes** + - allow theme type `None` to be set in config + ## Update for 2025-06-30 diff --git a/TODO.md b/TODO.md index 29a2dcf94..ce1305ea8 100644 --- a/TODO.md +++ b/TODO.md @@ -9,8 +9,9 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - Refactor: Move `model_*` stuff into subfolder - Refactor: sampler options - Common repo for `T5` and `CLiP` -- Upgrade: unblock `numpy`: see `gradio` -- Upgrade: unblock `pydantic`: see +- Upgrade: unblock `pydantic` and `albumentations` + - see + - blocked by `insightface` ### Complete Features diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index c4b07a0f7..b9ff142c7 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit c4b07a0f7ae121c183dba7974622fd0b93b72927 +Subproject commit b9ff142c7c75ec9833ccab6c1ee5125bc1fa5294 diff --git a/installer.py b/installer.py index 4de6f8103..be8ea712b 100644 --- a/installer.py +++ b/installer.py @@ -346,7 +346,7 @@ def pip(arg: str, ignore: bool = False, quiet: bool = True, uv = True): t_start = time.time() originalArg = arg arg = arg.replace('>=', '==') - package = arg.replace("install", "").replace("--upgrade", "").replace("--no-deps", "").replace("--force", "").replace(" ", " ").strip() + package = arg.replace("install", "").replace("--upgrade", "").replace("--no-deps", "").replace("--force-reinstall", "").replace(" ", " ").strip() uv = uv and args.uv and not package.startswith('git+') pipCmd = "uv pip" if uv else "pip" if not quiet and '-r ' not in arg: @@ -360,7 +360,7 @@ def pip(arg: str, ignore: bool = False, quiet: bool = True, uv = True): if len(result.stderr) > 0: if uv and result.returncode != 0: err = result.stderr.decode(encoding="utf8", errors="ignore") - log.warning('Install: cannot use uv, fallback to pip') + log.warning(f'Install: cmd="{pipCmd}" args="{all_args}" cannot use uv, fallback to pip') debug(f'Install: uv pip error: {err}') return pip(originalArg, ignore, quiet, uv=False) else: @@ -385,7 +385,7 @@ def install(package, friendly: str = None, ignore: bool = False, reinstall: bool quick_allowed = False if (args.reinstall) or (reinstall) or (not installed(package, friendly, quiet=quiet)): deps = '' if not no_deps else '--no-deps ' - cmd = f"install{' --upgrade' if not args.uv else ''}{' --force' if force else ''} {deps}{package}" + cmd = f"install{' --upgrade' if not args.uv else ''}{' --force-reinstall' if force else ''} {deps}{package}" res = pip(cmd, ignore=ignore, uv=package != "uv" and not package.startswith('git+')) try: importlib.reload(pkg_resources) @@ -827,6 +827,7 @@ def check_cudnn(): # check torch version def check_torch(): + log.info('Verifying torch installation') t_start = time.time() if args.skip_torch: log.info('Torch: skip tests') @@ -1163,31 +1164,43 @@ def ensure_base_requirements(): update_setuptools() # used by installler itself so must be installed before requirements - install('rich', 'rich', quiet=True) + install('rich==14.0.0', 'rich', quiet=True) install('psutil', 'psutil', quiet=True) - install('requests', 'requests', quiet=True) + install('requests==2.32.3', 'requests', quiet=True) ts('base', t_start) +def install_gradio(): + # pip install gradio==3.43.2 installs: + # aiofiles-23.2.1 altair-5.5.0 annotated-types-0.7.0 anyio-4.9.0 attrs-25.3.0 certifi-2025.6.15 charset_normalizer-3.4.2 click-8.2.1 contourpy-1.3.2 cycler-0.12.1 fastapi-0.115.14 ffmpy-0.6.0 filelock-3.18.0 fonttools-4.58.4 fsspec-2025.5.1 gradio-3.43.2 gradio-client-0.5.0 h11-0.16.0 hf-xet-1.1.5 httpcore-1.0.9 httpx-0.28.1 huggingface-hub-0.33.1 idna-3.10 importlib-resources-6.5.2 jinja2-3.1.6 jsonschema-4.24.0 jsonschema-specifications-2025.4.1 kiwisolver-1.4.8 markupsafe-2.1.5 matplotlib-3.10.3 narwhals-1.45.0 numpy-1.26.4 orjson-3.10.18 packaging-25.0 pandas-2.3.0 pillow-10.4.0 pydantic-2.11.7 pydantic-core-2.33.2 pydub-0.25.1 pyparsing-3.2.3 python-dateutil-2.9.0.post0 python-multipart-0.0.20 pytz-2025.2 pyyaml-6.0.2 referencing-0.36.2 requests-2.32.4 rpds-py-0.25.1 semantic-version-2.10.0 six-1.17.0 sniffio-1.3.1 starlette-0.46.2 tqdm-4.67.1 typing-extensions-4.14.0 typing-inspection-0.4.1 tzdata-2025.2 urllib3-2.5.0 uvicorn-0.35.0 websockets-11.0.3 + install('gradio==3.43.2', no_deps=True) + install('gradio-client==0.5.0', no_deps=True, quiet=True) + install('dctorch==0.1.2', no_deps=True, quiet=True) + pkgs = ['fastapi', 'websockets', 'aiofiles', 'ffmpy', 'pydub', 'uvicorn', 'semantic-version', 'altair', 'python-multipart'] + for pkg in pkgs: + if not installed(pkg, quiet=True): + install(pkg, quiet=True) + + def install_optional(): t_start = time.time() log.info('Installing optional requirements...') install('git+https://github.com/Disty0/BasicSR@2b6a12c28e0c81bfb13b7e984144f0b0f5461484', 'basicsr') install('git+https://github.com/Disty0/GFPGAN@09b1190eabbc77e5f15c61fa7c38a2064b403e20', 'gfpgan') - install('clean-fid') - install('pillow-jxl-plugin==1.3.3', ignore=True) - install('optimum-quanto==0.2.7', ignore=True) - install('torchao==0.10.0', ignore=True) - install('bitsandbytes==0.45.5', ignore=True) - install('pynvml', ignore=True) - install('ultralytics==8.3.40', ignore=True) - install('Cython', ignore=True) - install('insightface==0.7.3', ignore=True) # problematic build - install('albumentations==1.4.3', ignore=True) - install('pydantic==1.10.21', ignore=True) + install('clean-fid', quiet=True) + install('pillow-jxl-plugin==1.3.3', ignore=True, quiet=True) + install('optimum-quanto==0.2.7', ignore=True, quiet=True) + install('torchao==0.10.0', ignore=True, quiet=True) + install('bitsandbytes==0.45.5', ignore=True, quiet=True) + install('pynvml', ignore=True, quiet=True) + install('ultralytics==8.3.40', ignore=True, quiet=True) + install('Cython', ignore=True, quiet=True) + install('git+https://github.com/deepinsight/insightface@554a05561cb71cfebb4e012dfea48807f845a0c2#subdirectory=python-package', 'insightface') # insightface==0.7.3 with patches + install('albumentations==1.4.3', ignore=True, quiet=True) + install('pydantic==1.10.21', ignore=True, quiet=True) reload('pydantic', '1.10.21') install('gguf', ignore=True) - install('av', ignore=True) + install('av', ignore=True, quiet=True) try: import gguf scripts_dir = os.path.join(os.path.dirname(gguf.__file__), '..', 'scripts') diff --git a/launch.py b/launch.py index facf7d25a..082de3465 100755 --- a/launch.py +++ b/launch.py @@ -260,6 +260,7 @@ def main(): installer.set_environment() if args.uv: installer.install("uv", "uv") + installer.install_gradio() installer.check_torch() installer.check_onnx() installer.check_diffusers() diff --git a/modules/face/insightface.py b/modules/face/insightface.py index 7c9ec5af9..78e6c34a7 100644 --- a/modules/face/insightface.py +++ b/modules/face/insightface.py @@ -12,7 +12,7 @@ def get_app(mp_name, threshold=0.5, resolution=640): from installer import install, installed, reload if not installed('insightface', reload=False, quiet=True): - install('insightface==0.7.3', ignore=False) + install('git+https://github.com/deepinsight/insightface@554a05561cb71cfebb4e012dfea48807f845a0c2#subdirectory=python-package', 'insightface') # insightface==0.7.3 with patches install('albumentations==1.4.3', ignore=False, reinstall=True) install('pydantic==1.10.21', ignore=False, reinstall=True, force=True) reload('pydantic') diff --git a/modules/loader.py b/modules/loader.py index 674167d20..9d364daad 100644 --- a/modules/loader.py +++ b/modules/loader.py @@ -14,7 +14,22 @@ errors.install() logging.getLogger("DeepSpeed").disabled = True -# os.environ.setdefault('TORCH_LOGS', '-all') +np = None +try: + import numpy as np # pylint: disable=W0611,C0411 + import numpy.random # pylint: disable=W0611,C0411 # this causes failure if numpy version changed +except Exception as e: + errors.log.error(f'Loader: numpy=={np.__version__ if np is not None else None} {e}') + errors.log.error('Please restart the app to fix this issue') + sys.exit(1) + +try: + import scipy # pylint: disable=W0611,C0411 +except Exception as e: + errors.log.error(f'Loader: scipy=={np.__version__ if np is not None else None} {e}') + errors.log.error('Please restart the app to fix this issue') + sys.exit(1) + import torch # pylint: disable=C0411 if torch.__version__.startswith('2.5.0'): errors.log.warning(f'Disabling cuDNN for SDP on torch={torch.__version__}') @@ -34,6 +49,7 @@ logging.getLogger("pytorch_lightning").disabled = True warnings.filterwarnings(action="ignore", category=DeprecationWarning) warnings.filterwarnings(action="ignore", category=FutureWarning) warnings.filterwarnings(action="ignore", category=UserWarning, module="torchvision") +warnings.filterwarnings(action="ignore", message="numpy.dtype size changed") try: import torch._logging # pylint: disable=ungrouped-imports torch._logging._internal.DEFAULT_LOG_LEVEL = logging.ERROR # pylint: disable=protected-access @@ -139,7 +155,7 @@ try: except Exception: pass -try: # fix changed import in torchvision 0.17+, which breaks basicsr +try: import torchvision.transforms.functional_tensor # pylint: disable=unused-import, ungrouped-imports except ImportError: try: @@ -160,4 +176,4 @@ diffusers.utils.deprecate = deprecate_warn errors.log.info(f'Torch: torch=={torch.__version__} torchvision=={torchvision.__version__}') -errors.log.info(f'Packages: diffusers=={diffusers.__version__} transformers=={transformers.__version__} accelerate=={accelerate.__version__} gradio=={gradio.__version__} pydantic=={pydantic.__version__}') +errors.log.info(f'Packages: diffusers=={diffusers.__version__} transformers=={transformers.__version__} accelerate=={accelerate.__version__} gradio=={gradio.__version__} pydantic=={pydantic.__version__} numpy=={np.__version__}') diff --git a/modules/pulid/__init__.py b/modules/pulid/__init__.py index dcee2d7b9..958ee9c1e 100644 --- a/modules/pulid/__init__.py +++ b/modules/pulid/__init__.py @@ -4,8 +4,16 @@ Credit and original implementation: import os import sys +from modules.errors import log sys.path.append(os.path.dirname(__file__)) -from pulid_sdxl import StableDiffusionXLPuLIDPipeline, StableDiffusionXLPuLIDPipelineImage, StableDiffusionXLPuLIDPipelineInpaint -from pulid_utils import resize_numpy_image_long as resize -import attention_processor as attention -import pulid_sampling as sampling +try: + from pulid_sdxl import StableDiffusionXLPuLIDPipeline, StableDiffusionXLPuLIDPipelineImage, StableDiffusionXLPuLIDPipelineInpaint + from pulid_utils import resize_numpy_image_long as resize + import attention_processor as attention + import pulid_sampling as sampling +except Exception as e: + import traceback + log.error(f'PuLID import error: {e}') + print(traceback.format_exc()) + print(sys.exc_info()[0]) + raise ImportError(f'PuLID import error: {e}') from e diff --git a/modules/pulid/pulid_sdxl.py b/modules/pulid/pulid_sdxl.py index 0f4cf4f26..18bf0eb1c 100644 --- a/modules/pulid/pulid_sdxl.py +++ b/modules/pulid/pulid_sdxl.py @@ -1,7 +1,6 @@ from typing import Union import os import cv2 -import insightface import numpy as np import torch import torch.nn as nn @@ -14,6 +13,7 @@ from safetensors.torch import load_file from torchvision.transforms import InterpolationMode from torchvision.transforms.functional import normalize, resize +import insightface from basicsr.utils import img2tensor, tensor2img from facexlib.parsing import init_parsing_model from facexlib.utils.face_restoration_helper import FaceRestoreHelper diff --git a/modules/shared.py b/modules/shared.py index 624d7eae5..2cc4b0589 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -308,10 +308,7 @@ def temp_disable_extensions(): modules.shared.opts.data['theme_type'] = 'Modern' modules.shared.opts.data['gradio_theme'] = theme_name[7:] disable_themes.remove('sdnext-modernui') - elif theme_name.lower().startswith('gradio'): - modules.shared.opts.data['theme_type'] = 'None' - modules.shared.opts.data['gradio_theme'] = theme_name - elif theme_name.lower().startswith('huggingface'): + elif theme_name.lower().startswith('huggingface') or theme_name.lower().startswith('gradio') or theme_name.lower().startswith('none'): modules.shared.opts.data['theme_type'] = 'None' modules.shared.opts.data['gradio_theme'] = theme_name else: diff --git a/modules/theme.py b/modules/theme.py index f5e4e5455..c5924d96b 100644 --- a/modules/theme.py +++ b/modules/theme.py @@ -99,6 +99,8 @@ def reload_gradio_theme(): theme_name = 'black-teal' elif modules.shared.opts.theme_type == 'Modern': theme_name = 'Default' + elif modules.shared.opts.theme_type == 'None': + theme_name = 'gradio/default' else: modules.shared.opts.theme_type = 'Standard' theme_name = 'black-teal' diff --git a/repositories/ldm/models/diffusion/ddpm.py b/repositories/ldm/models/diffusion/ddpm.py index 99879b224..fe3f51fce 100644 --- a/repositories/ldm/models/diffusion/ddpm.py +++ b/repositories/ldm/models/diffusion/ddpm.py @@ -17,7 +17,8 @@ from functools import partial import itertools from tqdm import tqdm from torchvision.utils import make_grid -from pytorch_lightning.utilities.distributed import rank_zero_only +from pytorch_lightning.utilities.rank_zero import rank_zero_only + from omegaconf import ListConfig from ldm.util import log_txt_as_img, exists, default, ismap, isimage, mean_flat, count_params, instantiate_from_config diff --git a/requirements.txt b/requirements.txt index 0eea8fbac..66547dd6f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,30 +43,28 @@ antlr4-python3-runtime==4.9.3 requests==2.32.3 tqdm==4.67.1 accelerate==1.8.1 -opencv-contrib-python-headless==4.9.0.80 +opencv-contrib-python-headless==4.11.0.86 einops==0.4.1 -gradio==3.43.2 huggingface_hub==0.33.1 numexpr==2.10.2 -numpy==1.26.4 +numpy==2.1.2 pandas==2.3.0 numba==0.61.2 protobuf==4.25.3 -pytorch_lightning==1.9.4 -tokenizers==0.21.1 +pytorch_lightning==2.5.2 +tokenizers==0.21.2 transformers==4.53.0 urllib3==1.26.19 Pillow==10.4.0 timm==1.0.16 pydantic==1.10.21 -pyparsing==3.1.4 -typing-extensions==4.12.2 +pyparsing==3.1.2 +typing-extensions==4.14.0 # additional blendmodes -scipy +scipy==1.16.0 torchdiffeq -dctorch scikit-image seam-carving sentencepiece diff --git a/scripts/infiniteyou_ext.py b/scripts/infiniteyou_ext.py index a8ca9c9c3..7a702d858 100644 --- a/scripts/infiniteyou_ext.py +++ b/scripts/infiniteyou_ext.py @@ -15,7 +15,7 @@ orig_pipeline, orig_prompt_attention = None, None def verify_insightface(): from installer import installed, install, reload if not installed('insightface', reload=False, quiet=True): - install('insightface==0.7.3', ignore=False) + install('git+https://github.com/deepinsight/insightface@554a05561cb71cfebb4e012dfea48807f845a0c2#subdirectory=python-package', 'insightface') # insightface==0.7.3 with patches install('albumentations==1.4.3', ignore=False, reinstall=True) install('pydantic==1.10.21', ignore=False, reinstall=True, force=True) reload('pydantic') diff --git a/scripts/k_diff.py b/scripts/k_diff.py index 117df9c31..f5f7caab6 100644 --- a/scripts/k_diff.py +++ b/scripts/k_diff.py @@ -1,6 +1,6 @@ import inspect import gradio as gr -from modules import scripts, processing, shared, sd_models +from modules import scripts, processing, shared, sd_models, ui_common class Script(scripts.Script): @@ -17,7 +17,8 @@ class Script(scripts.Script): with gr.Row(): gr.HTML('  K-Diffusion Samplers
') with gr.Row(): - sampler = gr.Dropdown(label="Sampler", choices=self.samplers()) + sampler = gr.Dropdown(label="Sampler", choices=[]) + _refresh = ui_common.create_refresh_button(sampler, self.samplers, lambda: {"choices": self.samplers()}) return [sampler] def samplers(self): diff --git a/scripts/pulid_ext.py b/scripts/pulid_ext.py index 1c4b1a696..f000bc43a 100644 --- a/scripts/pulid_ext.py +++ b/scripts/pulid_ext.py @@ -30,7 +30,7 @@ class Script(scripts.Script): def dependencies(self): from installer import install, installed, reload if not installed('insightface==0.7.3', reload=False, quiet=True): - install('insightface==0.7.3', ignore=False) + install('git+https://github.com/deepinsight/insightface@554a05561cb71cfebb4e012dfea48807f845a0c2#subdirectory=python-package', 'insightface') # insightface==0.7.3 with patches install('albumentations==1.4.3', ignore=False, reinstall=True) install('pydantic==1.10.21', ignore=False, reinstall=True, force=True) reload('pydantic') diff --git a/webui.py b/webui.py index c24dbe179..927ea58e5 100644 --- a/webui.py +++ b/webui.py @@ -371,9 +371,11 @@ def webui(restart=False): app = start_ui() modules.script_callbacks.after_ui_callback() modules.sd_models.write_metadata() + load_model() mount_subpath(app) shared.opts.save(shared.config_filename) + if shared.cmd_opts.profile: for k, v in modules.script_callbacks.callback_map.items(): shared.log.debug(f'Registered callbacks: {k}={len(v)} {[c.script for c in v]}') diff --git a/wiki b/wiki index 45c389a37..a450f80ed 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 45c389a3747cb1caceb4035535041c400c29cad2 +Subproject commit a450f80edbdee9ba4daab34f05a374da7936cb5c