From 32dc4b3ab495894435cc129a8259f5f4db977158 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 5 Jul 2024 20:06:06 -0400 Subject: [PATCH] draft lumina work --- CHANGELOG.md | 10 +++++++--- html/reference.json | 1 + modules/model_lumina.py | 25 +++++++++++++++++++++++++ modules/sd_models.py | 11 +++++++++++ modules/sd_samplers_diffusers.py | 3 +++ modules/shared_items.py | 2 ++ requirements.txt | 2 +- 7 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 modules/model_lumina.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f1ccb53a..e04990523 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,22 +2,26 @@ TODO: - Requires `diffusers==0.30.0` -- Alpha Lumina: https://github.com/huggingface/diffusers/pull/8652 +- AlphaVLLM Lumina-Next: https://github.com/huggingface/diffusers/pull/8652 - LavenderFlow: https://github.com/huggingface/diffusers/pull/8796 +- FlowMatchHeunDiscreteScheduler ## Update for 2024-07-05 - massive updates to [Wiki](https://github.com/vladmandic/automatic/wiki) with over 20 new pages and articles, now includes guides for nearly all major features thanks @GenesisArtemis! -- support for DoRA networks, thanks @AI-Casanova! +- support for [AlphaVLLM Lumina-Next-SFT](https://huggingface.co/Alpha-VLLM/Lumina-Next-SFT-diffusers) + note: this is a large model at 8.6GB and uses T5 XXL variation of text encoder + (previous version of Lumina used Gemma 2B as text encoder) - support for [HunyuanDiT 1.2](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT-v1.2-Diffusers) - support for [CogFlorence 2 Large](https://huggingface.co/thwri/CogFlorence-2-Large-Freeze) VLM model - support for [AuraSR](https://huggingface.co/fal/AuraSR) high-quality 4x GAN-style upscaling model note: this is a large upscaler at 2.5GB - support for [uv](https://pypi.org/project/uv/), extremely fast installer, thanks @Yoinky3000! to use, simply add `--uv` to your command line params -- enable `Florence` VLM for all platforms, thanks @lshqqytiger! +- support for **DoRA** networks, thanks @AI-Casanova! +- enable **Florence VLM** for all platforms, thanks @lshqqytiger! - add SD3 with FP16 T5 to list of detected models - fix executing extensions with zero params - add support for embeddings bundled in LoRA, thanks @AI-Casanova! diff --git a/html/reference.json b/html/reference.json index c8eaee3f0..7c5101841 100644 --- a/html/reference.json +++ b/html/reference.json @@ -193,6 +193,7 @@ "path": "Alpha-VLLM/Lumina-Next-SFT-diffusers", "desc": "The Lumina-Next-SFT is a Next-DiT model containing 2B parameters and utilizes Gemma-2B as the text encoder, enhanced through high-quality supervised fine-tuning (SFT).", "preview": "Alpha-VLLM-Lumina-Next-SFT-diffusers.jpg", + "skip": true, "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 2.0" }, diff --git a/modules/model_lumina.py b/modules/model_lumina.py new file mode 100644 index 000000000..72bf04e2a --- /dev/null +++ b/modules/model_lumina.py @@ -0,0 +1,25 @@ +import diffusers + + +def load_lumina(_checkpoint_info, diffusers_load_config={}): + from modules import shared, devices, modelloader + modelloader.hf_login() + # {'low_cpu_mem_usage': True, 'torch_dtype': torch.float16, 'load_connected_pipeline': True, 'safety_checker': None, 'requires_safety_checker': False} + if 'torch_dtype' not in diffusers_load_config: + diffusers_load_config['torch_dtype'] = 'torch.float16' + if 'low_cpu_mem_usage' in diffusers_load_config: + del diffusers_load_config['low_cpu_mem_usage'] + if 'load_connected_pipeline' in diffusers_load_config: + del diffusers_load_config['load_connected_pipeline'] + if 'safety_checker' in diffusers_load_config: + del diffusers_load_config['safety_checker'] + if 'requires_safety_checker' in diffusers_load_config: + del diffusers_load_config['requires_safety_checker'] + pipe = diffusers.LuminaText2ImgPipeline.from_pretrained( + 'Alpha-VLLM/Lumina-Next-SFT-diffusers', + cache_dir = shared.opts.diffusers_dir, + **diffusers_load_config, + ) + print('HERE2', diffusers_load_config) + devices.torch_gc() + return pipe diff --git a/modules/sd_models.py b/modules/sd_models.py index 236cd821d..4f433b90a 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -611,6 +611,8 @@ def detect_pipeline(f: str, op: str = 'model', warning=True, quiet=False): guess = 'Stable Cascade' if 'pixart-sigma' in f.lower(): guess = 'PixArt-Sigma' + if 'lumina-next' in f.lower(): + guess = 'Lumina-Next' # switch for specific variant if guess == 'Stable Diffusion' and 'inpaint' in f.lower(): guess = 'Stable Diffusion Inpaint' @@ -992,6 +994,15 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No if debug_load: errors.display(e, 'Load') return + elif model_type in ['Lumina-Next']: # forced pipeline + try: + from modules.model_lumina import load_lumina + sd_model = load_lumina(checkpoint_info, diffusers_load_config) + except Exception as e: + shared.log.error(f'Diffusers Failed loading {op}: {checkpoint_info.path} {e}') + if debug_load: + errors.display(e, 'Load') + return elif model_type in ['Stable Diffusion 3']: try: from modules.model_sd3 import load_sd3 diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index 997ca5c3a..fa0c7aebb 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -33,6 +33,7 @@ try: PNDMScheduler, SASolverScheduler, FlowMatchEulerDiscreteScheduler, + # FlowMatchHeunDiscreteScheduler, ) except Exception as e: import diffusers @@ -67,6 +68,7 @@ config = { 'DPM++ 2M EDM': { 'solver_order': 2, 'solver_type': 'midpoint', 'final_sigmas_type': 'zero', 'algorithm_type': 'dpmsolver++' }, 'CMSI': { }, #{ 'sigma_min': 0.002, 'sigma_max': 80.0, 'sigma_data': 0.5, 's_noise': 1.0, 'rho': 7.0, 'clip_denoised': True }, 'Euler FlowMatch': { 'shift': 1, }, + # 'Heun FlowMatch': { 'shift': 1, }, 'IPNDM': { }, } @@ -99,6 +101,7 @@ samplers_data_diffusers = [ sd_samplers_common.SamplerData('TCD', lambda model: DiffusionSampler('TCD', TCDScheduler, model), [], {}), sd_samplers_common.SamplerData('CMSI', lambda model: DiffusionSampler('CMSI', CMStochasticIterativeScheduler, model), [], {}), sd_samplers_common.SamplerData('Euler FlowMatch', lambda model: DiffusionSampler('Euler FlowMatch', FlowMatchEulerDiscreteScheduler, model), [], {}), + # sd_samplers_common.SamplerData('Heun FlowMatch', lambda model: DiffusionSampler('Heun FlowMatch', FlowMatchHeunDiscreteScheduler, model), [], {}), sd_samplers_common.SamplerData('Same as primary', None, [], {}), ] diff --git a/modules/shared_items.py b/modules/shared_items.py index cc352f14f..cfeea03c6 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -99,6 +99,8 @@ def get_pipelines(): if hasattr(diffusers, 'StableDiffusion3Pipeline'): pipelines['Stable Diffusion 3'] = getattr(diffusers, 'StableDiffusion3Pipeline', None) pipelines['Stable Diffusion 3 Img2Img'] = getattr(diffusers, 'StableDiffusion3Img2ImgPipeline', None) + if hasattr(diffusers, 'LuminaText2ImgPipeline'): + pipelines['Lumina-Next'] = getattr(diffusers, 'LuminaText2ImgPipeline', None) for k, v in pipelines.items(): if k != 'Autodetect' and v is None: diff --git a/requirements.txt b/requirements.txt index 555d99e65..69a74cac8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -53,7 +53,7 @@ pandas protobuf==4.25.3 pytorch_lightning==1.9.4 tokenizers==0.19.1 -transformers==4.41.2 +transformers==4.42.3 urllib3==1.26.19 Pillow==10.3.0 timm==0.9.16