diff --git a/CHANGELOG.md b/CHANGELOG.md index 285a10303..db39a3aba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,14 +5,18 @@ - **Models** - [Zeta-Chroma](https://huggingface.co/lodestones/Zeta-Chroma) pixel-space diffusion transformer image model generates images directly in RGB space using NextDiT-style architecture - - [Bria FIBO](https://huggingface.co/briaai/FIBO) 8B parameter text-to-image model using Flow Matching - includes *Normal*, *Edit*, and *Lite* (distilled) variants - features lightweight SmolLM3-3B text encoder with efficient inference - [AiArtLab SDXS-1B](https://huggingface.co/AiArtLab/sdxs-1b) Simple Diffusion XS *(training still in progress)* this model combines Qwen3.5-1.8B text encoder with SDXL-style UNET with only 1.6B parameters and custom 32ch VAE - [Anima Preview-v3](https://huggingface.co/circlestone-labs/Anima) new version of Anima + - [Bria FIBO](https://huggingface.co/briaai/FIBO) 8B parameter text-to-image model using Flow Matching + includes *Normal*, *Edit*, and *Lite* (distilled) variants + features lightweight SmolLM3-3B text encoder with efficient inference + *note*: this is a [gated model](https://vladmandic.github.io/sdnext-docs/Gated/) + - [Lumina-DiMOO](https://huggingface.co/Alpha-VLLM/Lumina-DiMOO) omni diffusion large language model for multimodal generation and understanding + supports *text-to-image*, *image editing*, and *image understanding* in a unified discrete diffusion framework + uses a VQ-VAE backed token space with any-to-any task coverage - **Caption & Prompt Enhance** - - [Google Gemma 4] in *E2B* and *E4B* variants + - [Google Gemma 4] in *E2B* and *E4B* variants as well as *heretic* fine-tune - **Compute** - **ROCm** futher work on advanced configuration and tuning, thanks @resonantsky now covers both ROCm on Windows and Linux diff --git a/data/reference.json b/data/reference.json index 78ba210d9..d2a0508f0 100644 --- a/data/reference.json +++ b/data/reference.json @@ -601,6 +601,15 @@ "size": 20.75, "date": "2025 January" }, + "AlphaVLLM Lumina DiMOO": { + "path": "Alpha-VLLM/Lumina-DiMOO", + "desc": "Lumina-DiMOO is an omni diffusion large language model for multimodal generation and understanding with text-to-image, image editing and understanding capabilities.", + "preview": "Alpha-VLLM--Lumina-DiMOO.jpg", + "skip": true, + "extras": "sampler: Default", + "size": 0, + "date": "2025 September" + }, "HiDream-I1 Fast": { "path": "HiDream-ai/HiDream-I1-Fast", diff --git a/models/Reference/Alpha-VLLM--Lumina-DiMOO.jpg b/models/Reference/Alpha-VLLM--Lumina-DiMOO.jpg new file mode 100644 index 000000000..e69de29bb diff --git a/modules/caption/models_def.py b/modules/caption/models_def.py index f91b2e5f2..475fb5a0d 100644 --- a/modules/caption/models_def.py +++ b/modules/caption/models_def.py @@ -7,6 +7,7 @@ vlm_models = { "Google Gemma 3n E4B": "google/gemma-3n-E4B-it", # 1.5GB "Google Gemma 4 E2B": "google/gemma-4-E2B-it", "Google Gemma 4 E4B": "google/gemma-4-E4B-it", + "Heretic Gemma 4 E4B": "p-e-w/gemma-4-E2B-it-heretic-ara", "Nidum Gemma 3 4B Uncensored": "nidum/Nidum-Gemma-3-4B-it-Uncensored", "Allura Gemma 3 Glitter 4B": "allura-org/Gemma-3-Glitter-4B", # Qwen3.5 diff --git a/modules/modeldata.py b/modules/modeldata.py index 20c633b15..be9fa821f 100644 --- a/modules/modeldata.py +++ b/modules/modeldata.py @@ -52,6 +52,8 @@ def get_model_type(pipe): model_type = 'f1' elif "ZImage" in name or "Z-Image" in name: model_type = 'zimage' + elif "LuminaDiMOO" in name: + model_type = 'luminadimoo' elif "Lumina2" in name: model_type = 'lumina2' elif "Lumina" in name: diff --git a/modules/sd_detect.py b/modules/sd_detect.py index e1cefb21a..7c2170986 100644 --- a/modules/sd_detect.py +++ b/modules/sd_detect.py @@ -67,6 +67,8 @@ def guess_by_name(fn, current_guess): new_guess = 'Sana' elif 'lumina-next' in fn.lower(): new_guess = 'Lumina-Next' + elif 'lumina-dimoo' in fn.lower(): + new_guess = 'Lumina-DiMOO' elif 'lumina-image-2' in fn.lower(): new_guess = 'Lumina 2' elif 'kolors' in fn.lower(): diff --git a/modules/sd_models.py b/modules/sd_models.py index 9a6eea317..0a72a8ac8 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -361,6 +361,10 @@ def load_diffuser_force(detected_model_type, checkpoint_info, diffusers_load_con from pipelines.model_lumina import load_lumina sd_model = load_lumina(checkpoint_info, diffusers_load_config) allow_post_quant = True + elif model_type in ['Lumina-DiMOO']: + from pipelines.model_lumina import load_lumina_dimoo + sd_model = load_lumina_dimoo(checkpoint_info, diffusers_load_config) + allow_post_quant = False elif model_type in ['Kolors']: from pipelines.model_kolors import load_kolors sd_model = load_kolors(checkpoint_info, diffusers_load_config) diff --git a/modules/shared_items.py b/modules/shared_items.py index a2a52a3f7..ac14aef3c 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -30,6 +30,7 @@ pipelines = { 'Chroma': getattr(diffusers, 'ChromaPipeline', None), 'Sana': getattr(diffusers, 'SanaPipeline', None), 'Lumina-Next': getattr(diffusers, 'LuminaText2ImgPipeline', None), + 'Lumina-DiMOO': getattr(diffusers, 'LuminaDiMOOPipeline', None), 'Lumina 2': getattr(diffusers, 'Lumina2Pipeline', None), 'AuraFlow': getattr(diffusers, 'AuraFlowPipeline', None), 'Kandinsky 2.1': getattr(diffusers, 'KandinskyCombinedPipeline', None), diff --git a/pipelines/model_lumina.py b/pipelines/model_lumina.py index 5b85bf0fd..b0f1ad795 100644 --- a/pipelines/model_lumina.py +++ b/pipelines/model_lumina.py @@ -53,3 +53,34 @@ def load_lumina2(checkpoint_info, diffusers_load_config=None): devices.torch_gc(force=True, reason='load') return pipe + + +def load_lumina_dimoo(checkpoint_info, diffusers_load_config=None): + if diffusers_load_config is None: + diffusers_load_config = {} + repo_id = sd_models.path_to_repo(checkpoint_info) + sd_models.hf_auth_check(checkpoint_info) + + load_config, _quant_args = model_quant.get_dit_args(diffusers_load_config, allow_quant=False) + log.debug(f'Load model: type=LuminaDiMOO repo="{repo_id}" config={diffusers_load_config} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_config}') + + pipe_cls = getattr(diffusers, 'LuminaDiMOOPipeline', None) + if pipe_cls is not None: + pipe = pipe_cls.from_pretrained( + repo_id, + cache_dir=shared.opts.diffusers_dir, + **load_config, + ) + else: + try: + pipe = diffusers.DiffusionPipeline.from_pretrained( + repo_id, + cache_dir=shared.opts.diffusers_dir, + trust_remote_code=True, + **load_config, + ) + except Exception as e: + raise RuntimeError(f'Lumina-DiMOO is not available in installed diffusers={diffusers.__version__}. Please update diffusers to a version that includes LuminaDiMOOPipeline.') from e + + devices.torch_gc(force=True, reason='load') + return pipe