draft lumina work

CHANGELOG.md

@@ -2,22 +2,26 @@
 
 TODO:
 - Requires `diffusers==0.30.0`
-- Alpha Lumina: https://github.com/huggingface/diffusers/pull/8652
+- AlphaVLLM Lumina-Next: https://github.com/huggingface/diffusers/pull/8652
 - LavenderFlow: https://github.com/huggingface/diffusers/pull/8796
+- FlowMatchHeunDiscreteScheduler
 
 ## Update for 2024-07-05
 
 - massive updates to [Wiki](https://github.com/vladmandic/automatic/wiki)  
   with over 20 new pages and articles, now includes guides for nearly all major features  
   thanks @GenesisArtemis!  
-- support for DoRA networks, thanks @AI-Casanova!
+- support for [AlphaVLLM Lumina-Next-SFT](https://huggingface.co/Alpha-VLLM/Lumina-Next-SFT-diffusers)  
+  note: this is a large model at 8.6GB and uses T5 XXL variation of text encoder  
+  (previous version of Lumina used Gemma 2B as text encoder)  
 - support for [HunyuanDiT 1.2](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT-v1.2-Diffusers)
 - support for [CogFlorence 2 Large](https://huggingface.co/thwri/CogFlorence-2-Large-Freeze) VLM model
 - support for [AuraSR](https://huggingface.co/fal/AuraSR) high-quality 4x GAN-style upscaling model  
   note: this is a large upscaler at 2.5GB  
 - support for [uv](https://pypi.org/project/uv/), extremely fast installer, thanks @Yoinky3000!  
   to use, simply add `--uv` to your command line params  
-- enable `Florence` VLM for all platforms, thanks @lshqqytiger!  
+- support for **DoRA** networks, thanks @AI-Casanova!
+- enable **Florence VLM**  for all platforms, thanks @lshqqytiger!  
 - add SD3 with FP16 T5 to list of detected models
 - fix executing extensions with zero params  
 - add support for embeddings bundled in LoRA, thanks @AI-Casanova!

html/reference.json

@@ -193,6 +193,7 @@
     "path": "Alpha-VLLM/Lumina-Next-SFT-diffusers",
     "desc": "The Lumina-Next-SFT is a Next-DiT model containing 2B parameters and utilizes Gemma-2B as the text encoder, enhanced through high-quality supervised fine-tuning (SFT).",
     "preview": "Alpha-VLLM-Lumina-Next-SFT-diffusers.jpg",
+    "skip": true,
     "extras": "width: 1024, height: 1024, sampler: Default, cfg_scale: 2.0"
   }, 
 

modules/model_lumina.py

@@ -0,0 +1,25 @@
+import diffusers
+
+
+def load_lumina(_checkpoint_info, diffusers_load_config={}):
+    from modules import shared, devices, modelloader
+    modelloader.hf_login()
+    # {'low_cpu_mem_usage': True, 'torch_dtype': torch.float16, 'load_connected_pipeline': True, 'safety_checker': None, 'requires_safety_checker': False}
+    if 'torch_dtype' not in diffusers_load_config:
+        diffusers_load_config['torch_dtype'] = 'torch.float16'
+    if 'low_cpu_mem_usage' in diffusers_load_config:
+        del diffusers_load_config['low_cpu_mem_usage']
+    if 'load_connected_pipeline' in diffusers_load_config:
+        del diffusers_load_config['load_connected_pipeline']
+    if 'safety_checker' in diffusers_load_config:
+        del diffusers_load_config['safety_checker']
+    if 'requires_safety_checker' in diffusers_load_config:
+        del diffusers_load_config['requires_safety_checker']
+    pipe = diffusers.LuminaText2ImgPipeline.from_pretrained(
+        'Alpha-VLLM/Lumina-Next-SFT-diffusers',
+        cache_dir = shared.opts.diffusers_dir,
+        **diffusers_load_config,
+    )
+    print('HERE2', diffusers_load_config)
+    devices.torch_gc()
+    return pipe

modules/sd_models.py

@@ -611,6 +611,8 @@ def detect_pipeline(f: str, op: str = 'model', warning=True, quiet=False):
                 guess = 'Stable Cascade'
             if 'pixart-sigma' in f.lower():
                 guess = 'PixArt-Sigma'
+            if 'lumina-next' in f.lower():
+                guess = 'Lumina-Next'
             # switch for specific variant
             if guess == 'Stable Diffusion' and 'inpaint' in f.lower():
                 guess = 'Stable Diffusion Inpaint'
@@ -992,6 +994,15 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No
                     if debug_load:
                         errors.display(e, 'Load')
                     return
+            elif model_type in ['Lumina-Next']: # forced pipeline
+                try:
+                    from modules.model_lumina import load_lumina
+                    sd_model = load_lumina(checkpoint_info, diffusers_load_config)
+                except Exception as e:
+                    shared.log.error(f'Diffusers Failed loading {op}: {checkpoint_info.path} {e}')
+                    if debug_load:
+                        errors.display(e, 'Load')
+                    return
             elif model_type in ['Stable Diffusion 3']:
                 try:
                     from modules.model_sd3 import load_sd3

modules/sd_samplers_diffusers.py

@@ -33,6 +33,7 @@ try:
         PNDMScheduler,
         SASolverScheduler,
         FlowMatchEulerDiscreteScheduler,
+        # FlowMatchHeunDiscreteScheduler,
     )
 except Exception as e:
     import diffusers
@@ -67,6 +68,7 @@ config = {
     'DPM++ 2M EDM': { 'solver_order': 2, 'solver_type': 'midpoint', 'final_sigmas_type': 'zero', 'algorithm_type': 'dpmsolver++' },
     'CMSI': { }, #{ 'sigma_min':  0.002, 'sigma_max': 80.0, 'sigma_data': 0.5, 's_noise': 1.0, 'rho': 7.0, 'clip_denoised': True },
     'Euler FlowMatch': { 'shift': 1, },
+    # 'Heun FlowMatch': { 'shift': 1, },
     'IPNDM': { },
 }
 
@@ -99,6 +101,7 @@ samplers_data_diffusers = [
     sd_samplers_common.SamplerData('TCD', lambda model: DiffusionSampler('TCD', TCDScheduler, model), [], {}),
     sd_samplers_common.SamplerData('CMSI', lambda model: DiffusionSampler('CMSI', CMStochasticIterativeScheduler, model), [], {}),
     sd_samplers_common.SamplerData('Euler FlowMatch', lambda model: DiffusionSampler('Euler FlowMatch', FlowMatchEulerDiscreteScheduler, model), [], {}),
+    # sd_samplers_common.SamplerData('Heun FlowMatch', lambda model: DiffusionSampler('Heun FlowMatch', FlowMatchHeunDiscreteScheduler, model), [], {}),
 
     sd_samplers_common.SamplerData('Same as primary', None, [], {}),
 ]

modules/shared_items.py

@@ -99,6 +99,8 @@ def get_pipelines():
     if hasattr(diffusers, 'StableDiffusion3Pipeline'):
         pipelines['Stable Diffusion 3'] = getattr(diffusers, 'StableDiffusion3Pipeline', None)
         pipelines['Stable Diffusion 3 Img2Img'] = getattr(diffusers, 'StableDiffusion3Img2ImgPipeline', None)
+    if hasattr(diffusers, 'LuminaText2ImgPipeline'):
+        pipelines['Lumina-Next'] = getattr(diffusers, 'LuminaText2ImgPipeline', None)
 
     for k, v in pipelines.items():
         if k != 'Autodetect' and v is None:

requirements.txt

@@ -53,7 +53,7 @@ pandas
 protobuf==4.25.3
 pytorch_lightning==1.9.4
 tokenizers==0.19.1
-transformers==4.41.2
+transformers==4.42.3
 urllib3==1.26.19
 Pillow==10.3.0
 timm==0.9.16