From 0ea7840608b2fbaddd7af5290e4070d71e90143e Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 30 Jan 2025 13:08:46 -0500 Subject: [PATCH] add tunable ops Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 6 ++++ installer.py | 23 +++++++++------ modules/control/proc/dwpose/__init__.py | 39 ++++++++++++++++++++----- modules/devices.py | 22 +++++++++++++- modules/shared.py | 2 ++ wiki | 2 +- 6 files changed, 76 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 293256f56..d04db3059 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Update for 2025-01-30 +- **Torch**: + - for cuda environemnts set default to `torch==2.6.0+cu126` + - add torch tunable ops and their max duration + *set in settings -> backend settings -> torch* - **Fixes**: - photomaker with offloading - photomaker with refine @@ -11,6 +15,8 @@ - handle invalid `triton` with `torch==2.6.0` - correct library import order - update requirements + - calculate dyn atten bmm slice rate + - dwpose update and patch `mmengine` installer ## Update for 2025-01-29 diff --git a/installer.py b/installer.py index 9dacf4682..01b75221c 100644 --- a/installer.py +++ b/installer.py @@ -527,9 +527,10 @@ def install_cuda(): if args.use_nightly: cmd = os.environ.get('TORCH_COMMAND', '--pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu126') else: - cmd = os.environ.get('TORCH_COMMAND', 'torch==2.5.1+cu124 torchvision==0.20.1+cu124 --index-url https://download.pytorch.org/whl/cu124') - # TODO torch no triton for torch==2.6 - # TODO blackwell requires cuda==12.8 + # cmd = os.environ.get('TORCH_COMMAND', 'torch==2.5.1+cu124 torchvision==0.20.1+cu124 --index-url https://download.pytorch.org/whl/cu124') + cmd = os.environ.get('TORCH_COMMAND', 'torch==2.6.0+cu126 torchvision==0.21.0+cu126 --index-url https://download.pytorch.org/whl/cu126') + os.environ.setdefault('TRITON_COMMAND', 'skip') + # TODO blackwell requires cuda==12.8 torch release is pending return cmd @@ -782,10 +783,14 @@ def check_cudnn(): import site site_packages = site.getsitepackages() cuda_path = os.environ.get('CUDA_PATH', '') - for site_package in site_packages: - folder = os.path.join(site_package, 'nvidia', 'cudnn', 'lib') - if os.path.exists(folder) and folder not in cuda_path: - os.environ['CUDA_PATH'] = f"{cuda_path}:{folder}" + if cuda_path == '': + for site_package in site_packages: + folder = os.path.join(site_package, 'nvidia', 'cudnn', 'lib') + if os.path.exists(folder) and folder not in cuda_path: + cuda_path = f"{cuda_path}:{folder}" + if cuda_path.startswith(':'): + cuda_path = cuda_path[1:] + os.environ['CUDA_PATH'] = cuda_path # check torch version @@ -1115,7 +1120,7 @@ def install_optional(): install('clean-fid') install('pillow-jxl-plugin==1.3.1', ignore=True) install('optimum-quanto==0.2.6', ignore=True) - install('bitsandbytes==0.45.0', ignore=True) + install('bitsandbytes==0.45.1', ignore=True) install('pynvml', ignore=True) install('ultralytics==8.3.40', ignore=True) install('Cython', ignore=True) @@ -1447,7 +1452,7 @@ def add_args(parser): group_compute.add_argument("--use-openvino", default=os.environ.get("SD_USEOPENVINO",False), action='store_true', help="Use Intel OpenVINO backend, default: %(default)s") group_compute.add_argument("--use-ipex", default=os.environ.get("SD_USEIPEX",False), action='store_true', help="Force use Intel OneAPI XPU backend, default: %(default)s") group_compute.add_argument("--use-cuda", default=os.environ.get("SD_USECUDA",False), action='store_true', help="Force use nVidia CUDA backend, default: %(default)s") - group_compute.add_argument("--use-nightly", default=os.environ.get("SD_USENIGHLY",False), action='store_true', help="Force use nightly torch builds, default: %(default)s") + group_compute.add_argument("--use-nightly", default=os.environ.get("SD_USENIGHTLY",False), action='store_true', help="Force use nightly torch builds, default: %(default)s") group_compute.add_argument("--use-rocm", default=os.environ.get("SD_USEROCM",False), action='store_true', help="Force use AMD ROCm backend, default: %(default)s") group_compute.add_argument('--use-zluda', default=os.environ.get("SD_USEZLUDA", False), action='store_true', help="Force use ZLUDA, AMD GPUs only, default: %(default)s") group_compute.add_argument("--use-xformers", default=os.environ.get("SD_USEXFORMERS",False), action='store_true', help="Force use xFormers cross-optimization, default: %(default)s") diff --git a/modules/control/proc/dwpose/__init__.py b/modules/control/proc/dwpose/__init__.py index d8fdb9618..350b88622 100644 --- a/modules/control/proc/dwpose/__init__.py +++ b/modules/control/proc/dwpose/__init__.py @@ -4,6 +4,7 @@ # 3rd Edited by ControlNet # 4th Edited by ControlNet (added face and correct hands) +from typing import Type, Optional, Union, List import os os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE" import cv2 @@ -16,37 +17,61 @@ checked_ok = False busy = False +def _register_module(self, module: Type, module_name: Optional[Union[str, List[str]]] = None, force: bool = False) -> None: + if not callable(module): + raise TypeError(f'module must be Callable, but got {type(module)}') + if module_name is None: + module_name = module.__name__ + if isinstance(module_name, str): + module_name = [module_name] + for name in module_name: + if not force and name in self._module_dict: # pylint: disable=protected-access + pass # patch for 'Adafactor is already registered in optimizer at torch.optim' + self._module_dict[name] = module # pylint: disable=protected-access + + def check_dependencies(): global checked_ok, busy # pylint: disable=global-statement + busy = True debug = log.trace if os.environ.get('SD_DWPOSE_DEBUG', None) is not None else lambda *args, **kwargs: None + # pip install --upgrade --no-deps --force-reinstall termcolor xtcocotools terminaltables pycocotools munkres shapely openmim==0.3.9 mmengine==0.10.5 mmcv==2.2.0 mmpose==1.3.2 mmdet==3.3.0 packages = [ 'termcolor', + 'xtcocotools', + 'terminaltables', + 'pycocotools', + 'munkres', + 'shapely', 'openmim==0.3.9', - 'mmengine==0.10.4', - 'mmcv==2.1.0', - 'mmpose==1.3.1', + 'mmengine==0.10.5', + 'mmcv==2.2.0', + 'mmpose==1.3.2', 'mmdet==3.3.0', ] - status = [installed(p, reload=False, quiet=False) for p in packages] + status = [installed(p, reload=False, quiet=True) for p in packages] debug(f'DWPose required={packages} status={status}') if not all(status): - log.info(f'Installing DWPose dependencies: {[packages]}') + log.info(f'Installing DWPose dependencies: {packages}') cmd = 'install --upgrade --no-deps --force-reinstall ' pkgs = ' '.join(packages) - res = pip(cmd + pkgs, ignore=False, quiet=False) - debug(f'DWPose pip install: {res}') + pip(cmd + pkgs, ignore=False, quiet=True, uv=False) try: import pkg_resources import imp # pylint: disable=deprecated-module imp.reload(pkg_resources) import mmcv # pylint: disable=unused-import import mmengine # pylint: disable=unused-import + from mmengine.registry import Registry + Registry._register_module = _register_module # pylint: disable=protected-access import mmpose # pylint: disable=unused-import import mmdet # pylint: disable=unused-import debug('DWPose import ok') checked_ok = True except Exception as e: log.error(f'DWPose: {e}') + # from modules import errors + # errors.display(e, 'DWPose') + busy = False return checked_ok diff --git a/modules/devices.py b/modules/devices.py index 874b24de8..db3ba3f89 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -288,6 +288,20 @@ def set_cuda_memory_limit(): log.warning(f'Torch CUDA memory limit: fraction={opts.cuda_mem_fraction:.2f} {e}') +def set_cuda_tunable(): + if not cuda_ok: + return + try: + if opts.torch_tunable_ops != 'default': + torch.cuda.tunable.enable(opts.torch_tunable_ops == 'true') + torch.cuda.tunable.tuning_enable(opts.torch_tunable_ops == 'true') + # torch.cuda.tunable.set_max_tuning_duration(100) + torch.cuda.tunable.set_max_tuning_iterations(opts.torch_tunable_limit) + # log.debug(f'Torce tunable: enabled={torch.cuda.tunable.is_enabled()} tuning={torch.cuda.tunable.tuning_is_enabled()} iterations={torch.cuda.tunable.get_max_tuning_iterations()} duration={torch.cuda.tunable.get_max_tuning_duration()}') + except Exception: + pass + + def test_fp16(): global fp16_ok # pylint: disable=global-statement if fp16_ok is not None: @@ -484,6 +498,7 @@ def set_dtype(): def set_cuda_params(): override_ipex_math() set_cuda_memory_limit() + set_cuda_tunable() set_cudnn_params() set_sdpa_params() set_dtype() @@ -492,7 +507,12 @@ def set_cuda_params(): device_name = get_raw_openvino_device() else: device_name = torch.device(get_optimal_device_name()) - log.info(f'Torch parameters: backend={backend} device={device_name} config={opts.cuda_dtype} dtype={dtype} context={inference_context.__name__} nohalf={opts.no_half} nohalfvae={opts.no_half_vae} upcast={opts.upcast_sampling} deterministic={opts.cudnn_deterministic} fp16={"pass" if fp16_ok else "fail"} bf16={"pass" if bf16_ok else "fail"} optimization="{opts.cross_attention_optimization}"') + try: + # tunable = torch._C._jit_get_tunable_op_enabled() # pylint: disable=protected-access + tunable = [torch.cuda.tunable.is_enabled(), torch.cuda.tunable.tuning_is_enabled()] + except Exception: + tunable = [False, False] + log.info(f'Torch parameters: backend={backend} device={device_name} config={opts.cuda_dtype} dtype={dtype} context={inference_context.__name__} nohalf={opts.no_half} nohalfvae={opts.no_half_vae} upcast={opts.upcast_sampling} deterministic={opts.cudnn_deterministic} tunable={tunable} fp16={"pass" if fp16_ok else "fail"} bf16={"pass" if bf16_ok else "fail"} optimization="{opts.cross_attention_optimization}"') def cond_cast_unet(tensor): diff --git a/modules/shared.py b/modules/shared.py index 6d4b8be78..269627af0 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -528,6 +528,8 @@ options_templates.update(options_section(('backends', "Backend Settings"), { "cudnn_benchmark": OptionInfo(False, "Full-depth cuDNN benchmark"), "diffusers_fuse_projections": OptionInfo(False, "Fused projections"), "torch_expandable_segments": OptionInfo(False, "Expandable segments"), + "torch_tunable_ops": OptionInfo("default", "Tunable ops", gr.Radio, {"choices": ["default", "true", "false"]}), + "torch_tunable_limit": OptionInfo(30, "Tunable ops limit", gr.Slider, {"minimum": 1, "maximum": 100, "step": 1}), "cuda_mem_fraction": OptionInfo(0.0, "Memory limit", gr.Slider, {"minimum": 0, "maximum": 2.0, "step": 0.05}), "torch_gc_threshold": OptionInfo(70, "GC threshold", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1}), "inference_mode": OptionInfo("no-grad", "Inference mode", gr.Radio, {"choices": ["no-grad", "inference-mode", "none"]}), diff --git a/wiki b/wiki index ba2f43a51..5b47edf5c 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit ba2f43a51370e98be99dadb9c2f19e3551f33cca +Subproject commit 5b47edf5cfedd9e89dbed6508ac86afb7c12777c