add lumina-dimmo

Signed-off-by: Vladimir Mandic <mandic00@live.com>
2026-09-20 01:31:13 +02:00 · 2026-05-06 09:11:33 +02:00
parent 75b0a4da59
commit a7d3723014
9 changed files with 2945 additions and 35 deletions
@@ -34,6 +34,10 @@ For full details, see [ChangeLog](https://github.com/vladmandic/automatic/blob/m
  - [VIBE Image Edit](https://huggingface.co/iitolstykh/VIBE-Image-Edit) text-guided image editing model  
    built on Sana1.5-1.6B diffusion backbone with Qwen3-VL-2B multimodal conditioning  
    supports both image editing and text-to-image generation; uses multi-scale resolution binning up to 2048px  
+  - [AlphaVLLM Lumina-DiMOO](https://huggingface.co/Alpha-VLLM/Lumina-DiMOO) unified multimodal diffusion model  
+    includes *T2I*, *I2I edit*, and *MMU* capabilities in a single pipeline  
+    *note* in addition to normal prompt-based image editing, model also supports special prompts: *dense, canny_pred, control, subject, edit, ref_transfer, multi_view*
+    *note* as any multi-modal/unified model, it needs higher step count: recommended is 64 steps  
  - [Owen777 UltraFlux-v1](https://huggingface.co/Owen777/UltraFlux-v1) native 4K text-to-image model based on FLUX.1-dev  
    *note*: UltraFlux is capable of rendering images up to 4K resolution, but it doesnt mean it will do that on any hardware - it will depend on your VRAM!
 - **UI**
@@ -53,12 +53,9 @@ TODO: Investigate which models are diffusers-compatible and prioritize!
 ### Image

 - [JoyAI-Image-Edit](https://github.com/huggingface/diffusers/pull/13444) (pr in-progress)
- [Lumina-DiMOO](https://github.com/huggingface/diffusers/pull/12468) (pr stalled)
 - [nVidia Cosmos-Predict-2.5](https://huggingface.co/nvidia/Cosmos-Predict2.5-2B) (in diffusers)
 - [nVidia Cosmos-Transfer-2.5](https://huggingface.co/nvidia/Cosmos-Transfer2.5-2B) (in diffusers)
 - [Tencent HY-WU](https://huggingface.co/tencent/HY-WU) (transformers-compatible)
- [Mugen](https://huggingface.co/CabalResearch/Mugen) (sdxl with flux vae experiment, not clean)
- [Liquid](https://github.com/FoundationVision/Liquid) (autoregressive, not clean)

 ### Video

@@ -149,6 +146,11 @@ TODO: Investigate which models are diffusers-compatible and prioritize!
  - Background removal model trained on Bria FIBO dataset
  - Created: 2025-08 | Updated: 2025-09 | Stars: N/A (private model)

+### Rejected
+
+- [Mugen](https://huggingface.co/CabalResearch/Mugen) (sdxl with flux vae experiment, not clean)
+- [Liquid](https://github.com/FoundationVision/Liquid) (autoregressive, not clean)
+
 ## Code TODO

 > npm run todo
@@ -632,6 +632,15 @@
    "size": 20.75,
    "date": "2025 January"
  },
+  "AlphaVLLM Lumina DiMOO": {
+    "path": "Alpha-VLLM/Lumina-DiMOO",
+    "desc": "Lumina-DiMOO is an omni diffusion large language model for multimodal generation and understanding with text-to-image, image editing, and multimodal understanding capabilities.",
+    "preview": "Alpha-VLLM--Lumina-DiMOO.jpg",
+    "skip": true,
+    "extras": "sampler: Default",
+    "size": 0,
+    "date": "2025 September"
+  },

  "HiDream-I1 Fast": {
    "path": "HiDream-ai/HiDream-I1-Fast",
@@ -31,7 +31,6 @@ pipelines = {
    'Chroma': getattr(diffusers, 'ChromaPipeline', None),
    'Sana': getattr(diffusers, 'SanaPipeline', None),
    'Lumina-Next': getattr(diffusers, 'LuminaText2ImgPipeline', None),
-    # 'Lumina-DiMOO': getattr(diffusers, 'LuminaDiMOOPipeline', None),
    'Lumina 2': getattr(diffusers, 'Lumina2Pipeline', None),
    'AuraFlow': getattr(diffusers, 'AuraFlowPipeline', None),
    'Kandinsky 2.1': getattr(diffusers, 'KandinskyCombinedPipeline', None),
@@ -56,7 +55,6 @@ pipelines = {
    'FLUX2 Klein': getattr(diffusers, 'Flux2KleinPipeline', None),
    'LongCat': getattr(diffusers, 'LongCatImagePipeline', None),
    'GLM-Image': getattr(diffusers, 'GlmImagePipeline', None),
-    'VIBE': getattr(diffusers, 'VIBESanaEditingPipeline', getattr(diffusers, 'DiffusionPipeline', None)),
    # dynamically imported and redefined later
    'Meissonic': getattr(diffusers, 'DiffusionPipeline', None),
    'OmniGen2': getattr(diffusers, 'DiffusionPipeline', None),
@@ -70,6 +68,8 @@ pipelines = {
    'Anima': getattr(diffusers, 'DiffusionPipeline', None),
    'SDXS': getattr(diffusers, 'DiffusionPipeline', None),
    'Step1X-Edit': getattr(diffusers, 'DiffusionPipeline', None),
+    'VIBE': getattr(diffusers, 'DiffusionPipeline', None),
+    'Lumina-DiMOO': getattr(diffusers, 'DiffusionPipeline', None),
 }


@@ -0,0 +1,163 @@
+
+To load a custom pipeline you just need to pass the `custom_pipeline` argument to `DiffusionPipeline`, as one of the files in `diffusers/examples/community`. Feel free to send a PR with your own pipelines, we will merge them quickly.
+
+
+## Example Usage
+
+# Lumina-DiMOO
+[Project](https://synbol.github.io/Lumina-DiMOO/) / [GitHub](https://github.com/Alpha-VLLM/Lumina-DiMOO/) / [Model](https://huggingface.co/Alpha-VLLM/Lumina-DiMOO)
+
+Lumina-DiMOO is a discrete-diffusion omni-modal foundation model unifying generation and understanding. This implementation integrates a Lumina-DiMOO switch for T2I, I2I editing, and MMU.
+
+#### Key features
+
+- **Unified Discrete Diffusion Architecture**: Employs a fully discrete diffusion framework to process inputs and outputs across diverse modalities.
+- **Versatile Multimodal Capabilities**: Supports a wide range of multimodal tasks, including text-to-image generation (arbitrary and high-resolution), image-to-image generation (e.g., image editing, subject-driven generation, inpainting), and advanced image understanding.
+- **Higher Sampling Efficiency**:  Outperforms previous autoregressive (AR) or hybrid AR-diffusion models with significantly faster sampling. A custom caching mechanism further boosts sampling speed by up to 2×.
+
+
+### Example Usage
+
+The Lumina-DiMOO pipeline provides three core functions — T2I, I2I, and MMU.
+For detailed implementation examples and creative applications, please visit the [GitHub](https://github.com/Alpha-VLLM/Lumina-DiMOO)
+
+
+#### Text-to-Image
+**prompt**             |  **image** 
+:-------------------------:|:-------------------------:
+| "A striking photograph of a glass of orange juice on a wooden kitchen table, capturing a playful moment. The orange juice splashes out of the glass and forms the word \"Smile\" in a whimsical, swirling script just above the glass. The background is softly blurred, revealing a cozy, homely kitchen with warm lighting and a sense of comfort." | <img src="https://github-production-user-asset-6210df.s3.amazonaws.com/73575386/500095460-5490df4b-5ed9-4db7-bbca-3768a32ac840.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVCODYLSA53PQK4ZA%2F20251011%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20251011T015852Z&X-Amz-Expires=300&X-Amz-Signature=c9dc21e9414ed9069d35c72e21c4226aaea86142ab18dc9571aefb9bbdce9642&X-Amz-SignedHeaders=host">
+
+```python
+import torch
+
+from diffusers import VQModel, DiffusionPipeline
+from transformers import AutoTokenizer
+
+vqvae = VQModel.from_pretrained("Alpha-VLLM/Lumina-DiMOO", subfolder="vqvae").to(device='cuda', dtype=torch.bfloat16)
+tokenizer = AutoTokenizer.from_pretrained("Alpha-VLLM/Lumina-DiMOO", trust_remote_code=True)
+
+pipe = DiffusionPipeline.from_pretrained(
+    "Alpha-VLLM/Lumina-DiMOO",
+    vqvae=vqvae,
+    tokenizer=tokenizer,
+    torch_dtype=torch.bfloat16,
+    custom_pipeline="lumina_dimoo",
+)
+pipe.to("cuda")
+
+prompt = '''A striking photograph of a glass of orange juice on a wooden kitchen table, capturing a playful moment. The orange juice splashes out of the glass and forms the word \"Smile\" in a whimsical, swirling script just above the glass. The background is softly blurred, revealing a cozy, homely kitchen with warm lighting and a sense of comfort.'''
+
+img = pipe(
+    prompt=prompt,
+    task="text_to_image",
+    height=768,
+    width=1536,
+    num_inference_steps=64,
+    cfg_scale=4.0,     
+    use_cache=True,
+    cache_ratio=0.9, 
+    warmup_ratio=0.3,
+    refresh_interval=5
+).images[0]
+
+img.save("t2i_test_output.png")
+```
+
+#### Image-to-Image
+**prompt**             |  **image_before**   |  **image_after**  
+:-------------------------:|:-------------------------:|:-------------------------:
+| "A functional wooden printer stand.Nestled next to a brick wall in a bustling city street, it stands firm as pedestrians hustle by, illuminated by the warm glow of vintage street lamps." | <img src="https://github-production-user-asset-6210df.s3.amazonaws.com/73575386/500095462-7b451c2f-ec15-4cb9-8a6a-4eab9d2f5760.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVCODYLSA53PQK4ZA%2F20251011%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20251011T015605Z&X-Amz-Expires=300&X-Amz-Signature=a0e4be32af521b555747b794e660eb0ab116a7ced1cded55448f324fca9fa901&X-Amz-SignedHeaders=host"> | <img src="https://github-production-user-asset-6210df.s3.amazonaws.com/73575386/500095459-1d6631ef-8286-4402-a74b-85fb1ea684c3.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVCODYLSA53PQK4ZA%2F20251011%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20251011T015729Z&X-Amz-Expires=300&X-Amz-Signature=4fb8f2e5eb9645b5e8efb1022e01c329f4a54e793a9e3c4e36fdd357e076964a&X-Amz-SignedHeaders=host"> |
+
+```python
+import torch
+
+from diffusers import VQModel, DiffusionPipeline
+from transformers import AutoTokenizer
+from diffusers.utils import load_image
+
+vqvae = VQModel.from_pretrained("Alpha-VLLM/Lumina-DiMOO", subfolder="vqvae").to(device='cuda', dtype=torch.bfloat16)
+tokenizer = AutoTokenizer.from_pretrained("Alpha-VLLM/Lumina-DiMOO", trust_remote_code=True)
+
+pipe = DiffusionPipeline.from_pretrained(
+    "Alpha-VLLM/Lumina-DiMOO",
+    vqvae=vqvae,
+    tokenizer=tokenizer,
+    torch_dtype=torch.bfloat16,
+    custom_pipeline="lumina_dimoo",
+)
+pipe.to("cuda")
+
+input_image = load_image(
+    "https://raw.githubusercontent.com/Alpha-VLLM/Lumina-DiMOO/main/examples/example_2.jpg"
+).convert("RGB")
+
+prompt = "A functional wooden printer stand.Nestled next to a brick wall in a bustling city street, it stands firm as pedestrians hustle by, illuminated by the warm glow of vintage street lamps."
+
+img = pipe(
+    prompt=prompt,
+    image=input_image,
+    edit_type="depth_control",
+    num_inference_steps=64,
+    temperature=1.0,
+    cfg_scale=2.5,
+    cfg_img=4.0,
+    task="image_to_image"
+).images[0]
+
+img.save("i2i_test_output.png")
+
+```
+
+
+#### Multimodal Understanding
+**question**       |   **image**      |   **answer** 
+:-------------------------:|:-------------------------:|:-------------------------:
+| "Please describe the image." | <img src="https://github-production-user-asset-6210df.s3.amazonaws.com/73575386/500095461-9ae63fc0-b992-4652-9af5-b87be647048f.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVCODYLSA53PQK4ZA%2F20251011%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20251011T015701Z&X-Amz-Expires=300&X-Amz-Signature=acd3b53d43c81324b2135fde46a0ad25d04dfb71f987456a0b02d1b9af2f2e2e&X-Amz-SignedHeaders=host"> | "The image shows a vibrant orange sports car parked in a showroom. The car has a sleek, aerodynamic design with a prominent front grille and side vents. The body is adorned with black and orange racing stripes, creating a striking contrast against the orange paint. The car is equipped with black alloy wheels and a low-profile body style. The background features a white wall with a large emblem that reads "BREITZEN" and includes a silhouette of a horse and text. The floor is tiled with dark tiles, and the showroom is well-lit, highlighting the car. The overall setting suggests a high-end, possibly luxury, automotive environment."|
+
+
+```python
+import torch
+result.images[0].save(f"flux_fill_controlnet_inpaint_depth{timestamp}.jpg")
+```
+
+
+```python
+import torch
+
+from diffusers import VQModel, DiffusionPipeline
+from transformers import AutoTokenizer
+from diffusers.utils import load_image
+
+vqvae = VQModel.from_pretrained("Alpha-VLLM/Lumina-DiMOO", subfolder="vqvae").to(device='cuda', dtype=torch.bfloat16)
+tokenizer = AutoTokenizer.from_pretrained("Alpha-VLLM/Lumina-DiMOO", trust_remote_code=True)
+
+pipe = DiffusionPipeline.from_pretrained(
+    "Alpha-VLLM/Lumina-DiMOO",
+    vqvae=vqvae,
+    tokenizer=tokenizer,
+    torch_dtype=torch.bfloat16,
+    custom_pipeline="lumina_dimoo",
+)
+pipe.to("cuda")
+
+question = "Please describe the image."
+
+input_image = load_image(
+    "https://raw.githubusercontent.com/Alpha-VLLM/Lumina-DiMOO/main/examples/example_8.png"
+).convert("RGB")
+
+out = pipe(
+    prompt=question,
+    image=input_image,
+    task="multimodal_understanding",
+    num_inference_steps=128,
+    gen_length=128,
+    block_length=32,
+    temperature=0.0,
+    cfg_scale=0.0,
+)
+
+text = getattr(out, "text", out)
+with open("mmu_answer.txt", "w", encoding="utf-8") as f:
+    f.write(text.strip() + "\n")
+```
@@ -0,0 +1,58 @@
+from pipelines.lumina_dimmo.lumina_dimoo import LuminaDiMOOPipeline
+from modules.logger import log
+
+
+class LuminaDiMOOTextPipeline(LuminaDiMOOPipeline):
+    def __call__(self, prompt: list[str], height: int = 1024, width: int = 1024, num_inference_steps: int = 64, cfg_scale: float = 4.0, temperature: float = 1.0, painting_mode=None, painting_image=None, mask_h_ratio: float = 1.0, mask_w_ratio: float = 0.2, use_cache: bool = True, cache_ratio: float = 0.9, refresh_interval: int = 5, warmup_ratio: float = 0.3, **kwargs):
+        if isinstance(prompt, list):
+            prompt = prompt[0].strip()
+        task = "text_to_image"
+        log.debug(f'Base args: task={task} height={height} width={width} steps={num_inference_steps} cfg_scale={cfg_scale} temperature={temperature} painting_mode={painting_mode} mask_h_ratio={mask_h_ratio} mask_w_ratio={mask_w_ratio} use_cache={use_cache} cache_ratio={cache_ratio} refresh_interval={refresh_interval} warmup_ratio={warmup_ratio}')
+        return super().__call__(
+            prompt=prompt,
+            task=task,
+            height=height,
+            width=width,
+            num_inference_steps=num_inference_steps,
+            cfg_scale=cfg_scale,
+            temperature=temperature,
+            painting_mode=painting_mode,
+            painting_image=painting_image,
+            mask_h_ratio=mask_h_ratio,
+            mask_w_ratio=mask_w_ratio,
+            use_cache=use_cache,
+            cache_ratio=cache_ratio,
+            refresh_interval=refresh_interval,
+            warmup_ratio=warmup_ratio,
+            **kwargs)
+
+
+class LuminaDiMOOImagePipeline(LuminaDiMOOPipeline):
+    def __call__(self, prompt: list[str], image, num_inference_steps: int = 64, temperature: float = 1.0, cfg_scale: float = 2.5, cfg_img: float = 4.0, **kwargs):
+        edit_types = ['dense', 'canny_pred', 'control', 'subject', 'edit', 'ref_transfer', 'multi_view']
+        if isinstance(prompt, list):
+            prompt = prompt[0].strip()
+        task = "image_to_image"
+        edit_type = 'default'
+        ref_image = None
+        for et in edit_types:
+            if prompt.startswith(et):
+                edit_type = et
+                break
+        if isinstance(image, list):
+            if len(image) > 1:
+                ref_image = image[1]
+            if len(image) > 0:
+                image = image[0]
+        log.debug(f'Base args: task={task} edit_type={edit_type} steps={num_inference_steps} cfg_scale={cfg_scale} cfg_img={cfg_img} temperature={temperature} image={image} ref_image={ref_image}')
+        return super().__call__(
+            prompt=prompt,
+            task=task,
+            image=image,
+            ref_image=ref_image,
+            edit_type=edit_type,
+            num_inference_steps=num_inference_steps,
+            temperature=temperature,
+            cfg_scale=cfg_scale,
+            cfg_img=cfg_img,
+            **kwargs)
@@ -66,38 +66,31 @@ def load_lumina_dimoo(checkpoint_info, diffusers_load_config=None):
    repo_id = sd_models.path_to_repo(checkpoint_info)
    sd_models.hf_auth_check(checkpoint_info)

-    load_config, _quant_args = model_quant.get_dit_args(diffusers_load_config, allow_quant=False)
+    load_config, quant_args = model_quant.get_dit_args(diffusers_load_config)
    log.debug(f'Load model: type=LuminaDiMOO repo="{repo_id}" config={diffusers_load_config} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_config}')

-    pipe_cls = getattr(diffusers, 'LuminaDiMOOPipeline', None)
-    if pipe_cls is not None:
-        pipe = pipe_cls.from_pretrained(
-            repo_id,
-            cache_dir=shared.opts.diffusers_dir,
-            **load_config,
-        )
-    else:
-        try:
-            pipe = diffusers.DiffusionPipeline.from_pretrained(
-                repo_id,
-                cache_dir=shared.opts.diffusers_dir,
-                trust_remote_code=True,
-                **load_config,
-            )
-        except Exception as e:
-            raise RuntimeError(f'Lumina-DiMOO is not available in installed diffusers={diffusers.__version__}. Please update diffusers to a version that includes LuminaDiMOOPipeline.') from e
+    from pipelines.lumina_dimmo.pipelines import LuminaDiMOOTextPipeline, LuminaDiMOOImagePipeline
+    diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["luminadimoo"] = LuminaDiMOOTextPipeline
+    diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["luminadimoo"] = LuminaDiMOOImagePipeline

+    # Force slow tokenizer path for Lumina-DiMOO to avoid fast-tokenizer conversion failures.
+    tokenizer = transformers.AutoTokenizer.from_pretrained(
+        repo_id,
+        cache_dir=shared.opts.diffusers_dir,
+        trust_remote_code=True,
+        use_fast=False,
+    )
+
+    pipe = LuminaDiMOOTextPipeline.from_pretrained(
+        repo_id,
+        cache_dir=shared.opts.diffusers_dir,
+        tokenizer=tokenizer,
+        **load_config,
+        **quant_args,
+    )
+    pipe.skip_processing = True
+    pipe.task_args = {'output_type': 'np'}
+
+    del tokenizer
    devices.torch_gc(force=True, reason='load')
    return pipe
-
-""" Reference
-  "AlphaVLLM Lumina DiMOO": {
-    "path": "Alpha-VLLM/Lumina-DiMOO",
-    "desc": "Lumina-DiMOO is an omni diffusion large language model for multimodal generation and understanding with text-to-image, image editing and understanding capabilities.",
-    "preview": "Alpha-VLLM--Lumina-DiMOO.jpg",
-    "skip": true,
-    "extras": "sampler: Default",
-    "size": 0,
-    "date": "2025 September"
-  },
-"""
@@ -36,6 +36,7 @@ exclude = [
    "pipelines/step1x",
    "pipelines/vibe",
    "pipelines/ultraflux",
+    "pipelines/lumina_dimmo",
    "scripts/lbm",
    "scripts/daam",
    "scripts/xadapter",
@@ -172,6 +173,7 @@ main.ignore-paths=[
    "pipelines/step1x",
    "pipelines/vibe",
    "pipelines/ultraflux",
+    "pipelines/lumina_dimmo",
    "scripts/consistory",
    "scripts/ctrlx",
    "scripts/daam",