From dca4efb3ad92b0707fb873e8ef984885a54fb444 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 8 Oct 2023 08:10:34 -0400 Subject: [PATCH] handle xformers --- CHANGELOG.md | 15 +++++++++++---- installer.py | 10 ++++++---- modules/processing.py | 2 +- modules/sd_hijack_freeu.py | 10 ++++++---- modules/sd_hijack_hypertile.py | 2 ++ 5 files changed, 26 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a72300dc..d773094d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -135,9 +135,14 @@ or even free speedups and quality improvements (regardless of which workflows yo - to enable search, make sure all models have set hash values *Models -> Valida -> Calculate hashes* - **LoRA** - - for *backend:original*, lyco handler has been removed and replaced with new - unified lora/lyco handler that supports all variants of loras + - new unified LoRA handler for all LoRA types (lora, lyco, loha, lokr, locon, etc.) + applies to both original and diffusers backend + thanks @AI-Casanova for diffusers port + - for *backend:original*, separate lyco handler has been removed - **Compute** + - **CUDA**: + - default updated to `torch` *2.1.0* with cuda *12.1* + - testing moved to `torch` *2.2.0-dev/cu122* - **Intel Arc/IPEX**: - tons of optimizations, built-in binary wheels for Windows i have to say, intel arc/ipex is getting to be quite a player, especially with openvino @@ -145,8 +150,10 @@ or even free speedups and quality improvements (regardless of which workflows yo - **AMD ROCm**: - updated installer to support detect `ROCm` *5.4/5.5/5.6/5.7* - support for `torch-rocm-5.7` - - **CUDA**: - - testing moved to `torch` *2.2.0-dev/cu121* + - **xFormers**: + - default updated to *0.0.22* + - note that latest xformers are still not compatible with standard torch 2.1.0 with cuda 12.1 + either downgrade torch to 2.0.1 with cuda 11.8 or build xformers manually - **GC**: - custom garbage collect threshold to reduce vram memory usage, thanks @Disty0 see *settings -> compute -> gc* diff --git a/installer.py b/installer.py index a24406103..4da612567 100644 --- a/installer.py +++ b/installer.py @@ -341,8 +341,6 @@ def check_python(): # check torch version def check_torch(): - if args.quick: - return if args.skip_torch: log.info('Skipping Torch tests') return @@ -362,8 +360,8 @@ def check_torch(): pass elif allow_cuda and (shutil.which('nvidia-smi') is not None or os.path.exists(os.path.join(os.environ.get('SystemRoot') or r'C:\Windows', 'System32', 'nvidia-smi.exe'))): log.info('nVidia CUDA toolkit detected') - torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/cu118') - xformers_package = os.environ.get('XFORMERS_PACKAGE', 'xformers==0.0.20' if opts.get('cross_attention_optimization', '') == 'xFormers' else 'none') + torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/cu121') + xformers_package = os.environ.get('XFORMERS_PACKAGE', 'xformers==0.0.22' if opts.get('cross_attention_optimization', '') == 'xFormers' else 'none') elif allow_rocm and (shutil.which('rocminfo') is not None or os.path.exists('/opt/rocm/bin/rocminfo') or os.path.exists('/dev/kfd')): log.info('AMD ROCm toolkit detected') os.environ.setdefault('PYTORCH_HIP_ALLOC_CONF', 'garbage_collection_threshold:0.8,max_split_size_mb:512') @@ -483,6 +481,10 @@ def check_torch(): try: if 'xformers' in xformers_package: install(f'--no-deps {xformers_package}', ignore=True) + import torch + import xformers + if torch.__version__ != '2.0.1+cu118' and xformers.__version__ in ['0.0.22', '0.0.21', '0.0.20']: + log.warning(f'Likely incompatible torch with: xformers=={xformers.__version__} installed: torch=={torch.__version__} required: torch==2.0.1+cu118 - build xformers manually or downgrade torch') elif not args.experimental: x = pkg_resources.working_set.by_key.get('xformers', None) if x is not None: diff --git a/modules/processing.py b/modules/processing.py index 243863077..467cae2e2 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -643,7 +643,7 @@ def process_images(p: StableDiffusionProcessing) -> Processed: if not shared.opts.cuda_compile: modules.sd_models.apply_token_merging(p.sd_model, p.get_token_merging_ratio()) - modules.sd_hijack_freeu.apply_freeu(p.sd_model, shared.backend == shared.Backend.ORIGINAL) + modules.sd_hijack_freeu.apply_freeu(p, shared.backend == shared.Backend.ORIGINAL) if shared.cmd_opts.profile: """ diff --git a/modules/sd_hijack_freeu.py b/modules/sd_hijack_freeu.py index 2fa2dc10e..4d21601d4 100644 --- a/modules/sd_hijack_freeu.py +++ b/modules/sd_hijack_freeu.py @@ -126,11 +126,12 @@ def ratio_to_region(width: float, offset: float, n: int): return round(start), round(end), inverted -def apply_freeu(model, backend_original): +def apply_freeu(p, backend_original): global state_enabled # pylint: disable=global-statement global cat_original # pylint: disable=global-statement if backend_original: if opts.freeu_enabled: + p.extra_generation_params['FreeU'] = f'b1={opts.freeu_b1} b2={opts.freeu_b2} s1={opts.freeu_s1} s2={opts.freeu_s2}' if not state_enabled: # otherwise already patched cat_original = th.cat th.cat = functools.partial(free_u_cat_hijack, original_function=th.cat) @@ -139,12 +140,13 @@ def apply_freeu(model, backend_original): if cat_original is not None: th.cat = cat_original state_enabled = False - elif hasattr(model, 'enable_freeu'): + elif hasattr(p.sd_model, 'enable_freeu'): if opts.freeu_enabled: - model.enable_freeu(s1=opts.freeu_s1, s2=opts.freeu_s2, b1=opts.freeu_b1, b2=opts.freeu_b2) + p.extra_generation_params['FreeU'] = f'b1={opts.freeu_b1} b2={opts.freeu_b2} s1={opts.freeu_s1} s2={opts.freeu_s2}' + p.sd_model.enable_freeu(s1=opts.freeu_s1, s2=opts.freeu_s2, b1=opts.freeu_b1, b2=opts.freeu_b2) state_enabled = True elif state_enabled: - model.disable_freeu() + p.sd_model.disable_freeu() state_enabled = False if opts.freeu_enabled: log.info(f'Applying free-u: b1={opts.freeu_b1} b2={opts.freeu_b2} s1={opts.freeu_s1} s2={opts.freeu_s2}') diff --git a/modules/sd_hijack_hypertile.py b/modules/sd_hijack_hypertile.py index a163d0dc0..4ab38f986 100644 --- a/modules/sd_hijack_hypertile.py +++ b/modules/sd_hijack_hypertile.py @@ -142,6 +142,7 @@ def context_hypertile_vae(p): return nullcontext() else: shared.log.info(f'Applying hypertile: vae={shared.opts.hypertile_vae_tile}') + p.extra_generation_params['Hypertile VAE'] = shared.opts.hypertile_vae_tile return split_attention(vae, tile_size=shared.opts.hypertile_vae_tile, min_tile_size=128, swap_size=1) @@ -161,6 +162,7 @@ def context_hypertile_unet(p): return nullcontext() else: shared.log.info(f'Applying hypertile: unet={shared.opts.hypertile_unet_tile}') + p.extra_generation_params['Hypertile UNet'] = shared.opts.hypertile_unet_tile return split_attention(unet, tile_size=shared.opts.hypertile_unet_tile, min_tile_size=128, swap_size=1)