From 70beb1ee7c756f05a153850447f6bce62f659749 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 13 Sep 2026 12:49:57 +0200 Subject: [PATCH] add minimax-nunchaku-lite Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 22 ++++++++++++++-------- data/reference-distilled.json | 15 ++++++++------- data/reference-nunchaku.json | 7 +++++++ extensions-builtin/sdnext-modernui | 2 +- modules/attention/dispatcher.py | 2 +- modules/lora/native_adapter.py | 17 ++++++----------- modules/minimax/minimax_ui.py | 6 ++++-- modules/minimax/minimax_video.py | 6 +++--- modules/modular_load.py | 20 +++++++++++++++----- modules/video_models/models_def.py | 12 ++++++++++++ modules/video_models/video_minimax.py | 9 ++++++++- pipelines/generic_transformer.py | 7 +++---- pipelines/minimax/minimax_nunchaku.py | 20 ++++++++++++++++++++ pipelines/model_minimax.py | 14 ++++++++++++-- 14 files changed, 114 insertions(+), 45 deletions(-) create mode 100644 pipelines/minimax/minimax_nunchaku.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2202cf2a8..b7ed11f00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,12 +6,12 @@ *What's New*? Well, code-wise, this is a big one... First, a-lot-of-optimizations: -- updated core packages -- improved **LoRA** performance and quality, especially with quantized models -- newly structured **attention** mechanisms -- modular pipelines with new **guidance** methods -- support for different **caching** stacks -- compute updates across the board +- Updated core packages +- Improved **LoRA** performance and quality, especially with quantized models +- Newly structured **attention** mechanisms +- Modular pipelines with new **guidance** methods +- Support for different **caching** stacks +- Compute updates across the board And some cool new stuff and models: - **DLSSv5** integration @@ -32,7 +32,10 @@ Plus inevitable bug-fixes... LLaDA-Image is a 6.5B transformer with massive 16.3B fully-custom MoE text-encoder and optional 1.3B SigVQ conditioning model with support for text-to-image, vq-conditioned text-to-image and image-editing workflows *note* model is extremely quantization sensitive so minimum allowed quant type is `uint8` - - [MiniMax-H3](https://huggingface.co/MiniMaxAI/MiniMax-H3) update with pre-quantized `sdnq-uint8` *pruned* variants + - [MiniMax-H3](https://huggingface.co/MiniMaxAI/MiniMax-H3) updates + new [SDNQ-uint8](https://huggingface.co/OzzyGT/MiniMax_H3_sdnq_8bit_pruned) pre-quantized *pruned* variants + new [Nunchaku-Lite](https://huggingface.co/rootonchair/MiniMax-H3-nunchaku-lite-int4) variant + new [VDN](https://huggingface.co/OpenVDN/vdn-minimax-h3) *video-delta-net* variant - **LoRA** - see [LoRA docs](https://vladmandic.github.io/sdnext-docs/LoRA) for all of the improvements and usage instructions *note*: lora now has its own settings section in *settings -> lora* @@ -108,7 +111,10 @@ Plus inevitable bug-fixes... *note*: requires grok api key - remove `/redocs` as `/docs` are primary api docs - rebuild docs site index - - **ui**: ability to filter samplers and upscalers, thanks @emecii +- **UI** + - some (light) re-styling of the *Default* theme + - add new *Tillerz-CleanDark* theme, thanks @Tillerz + - ability to filter samplers and upscalers, thanks @emecii - **Wiki/Docs**: - new articles: *Attention, Modular-Pipelines* - updated: *LoRA, MiniMax* diff --git a/data/reference-distilled.json b/data/reference-distilled.json index c7d831f5f..9e894dc53 100644 --- a/data/reference-distilled.json +++ b/data/reference-distilled.json @@ -25,6 +25,14 @@ "size": 19.38, "date": "2023 November" }, + "SDXL Flash Mini": { + "path": "SDXL-Flash_Mini.safetensors@https://huggingface.co/sd-community/sdxl-flash-mini/resolve/main/SDXL-Flash_Mini.safetensors?download=true", + "preview": "SDXL-Flash_Mini.jpg", + "desc": "Introducing the new fast model SDXL Flash (Mini), we learned that all fast XL models work fast, but the quality decreases, and we also made a fast model, but it is not as fast as LCM, Turbo, Lightning and Hyper, but the quality is higher.", + "extras": "sampler: DEIS, steps: 40, cfg_scale: 6.0", + "experimental": true, + "date": "2024 May" + }, "Tencent HunyuanDiT 1.1 Distilled": { "path": "Tencent-Hunyuan/HunyuanDiT-v1.1-Diffusers-Distilled", "desc": "Hunyuan-DiT : A Powerful Multi-Resolution Diffusion Transformer with Fine-Grained Chinese Understanding.", @@ -258,12 +266,5 @@ "extras": "steps: 4, cfg_scale: 1.0", "size": 37.15, "date": "2026 September" - }, - "SDXL Flash Mini": { - "path": "SDXL-Flash_Mini.safetensors@https://huggingface.co/sd-community/sdxl-flash-mini/resolve/main/SDXL-Flash_Mini.safetensors?download=true", - "preview": "SDXL-Flash_Mini.jpg", - "desc": "Introducing the new fast model SDXL Flash (Mini), we learned that all fast XL models work fast, but the quality decreases, and we also made a fast model, but it is not as fast as LCM, Turbo, Lightning and Hyper, but the quality is higher.", - "extras": "sampler: DEIS, steps: 40, cfg_scale: 6.0", - "experimental": true } } diff --git a/data/reference-nunchaku.json b/data/reference-nunchaku.json index 17d49437e..15b129ce9 100644 --- a/data/reference-nunchaku.json +++ b/data/reference-nunchaku.json @@ -295,5 +295,12 @@ "desc": "Nunchaku-Lite quantization using precompiled Kernels", "size": 10.92, "date": "2026 August" + }, + "MiniMaxAI MiniMax-H3 Nunchaku-Lite": { + "path": "rootonchair/MiniMax-H3-nunchaku-lite-int4", + "preview": "OzzyGT/MiniMax_H3_sdnq_dynamic_4bit.jpg", + "desc": "Nunchaku-Lite quantization using precompiled Kernels", + "size": 23.70, + "date": "2026 September" } } diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index e95c40585..a9ed76cd1 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit e95c40585f8eb2d8ad0238ea7512ddf3e7fdcb23 +Subproject commit a9ed76cd116aa666b5bab00e37eddd8edf48b01c diff --git a/modules/attention/dispatcher.py b/modules/attention/dispatcher.py index 94c817949..6a8912347 100644 --- a/modules/attention/dispatcher.py +++ b/modules/attention/dispatcher.py @@ -72,7 +72,7 @@ def get_hf_api_hijack(user_agent = None): # pylint: disable=unused-argument def hijack_kernels(): global orig_get_kernel # pylint: disable=global-statement try: - install('kernels==0.16.0') + install('kernels==0.16.1') import kernels import kernels.utils log.debug(f'Attention dispatcher: kernels={kernels.__version__}') diff --git a/modules/lora/native_adapter.py b/modules/lora/native_adapter.py index 289dbaf2f..40cd29e97 100644 --- a/modules/lora/native_adapter.py +++ b/modules/lora/native_adapter.py @@ -644,21 +644,16 @@ def try_load_lora(name, network_on_disk, lora_scale, *, continue if not shapes_match(sd_module, target_w["lora_down.weight"], target_w["lora_up.weight"]): - log.warning( - f'Network load: type=LoRA name="{name}" arch={arch_name} key={network_key}' - f' lora={target_w["lora_down.weight"].shape[1]}x{target_w["lora_up.weight"].shape[0]}' - f' module={getattr(sd_module, "weight", None).shape if hasattr(sd_module, "weight") else "?"}' - f' shape mismatch' - ) + if l.debug: + _module = f'{getattr(sd_module, "weight", None).shape if hasattr(sd_module, "weight") else "?"}' + log.warning(f'Network load: type=LoRA name="{name}" arch={arch_name} key={network_key} lora={target_w["lora_down.weight"].shape[1]}x{target_w["lora_up.weight"].shape[0]} module={_module} shape mismatch') mismatch += 1 continue if "diff_b" in target_w and not bias_delta_fits(sd_module, target_w["diff_b"]): - log.warning( - f'Network load: type=LoRA name="{name}" arch={arch_name} key={network_key}' - f' bias={tuple(target_w["diff_b"].shape)} module={tuple(sd_module.bias.shape)}' - f' bias shape mismatch' - ) + if l.debug: + _bias = f'bias={tuple(target_w["diff_b"].shape)} module={tuple(sd_module.bias.shape)}' + log.warning(f'Network load: type=LoRA name="{name}" arch={arch_name} key={network_key} {_bias} bias shape mismatch') mismatch += 1 continue diff --git a/modules/minimax/minimax_ui.py b/modules/minimax/minimax_ui.py index a101f521a..d452c7f75 100644 --- a/modules/minimax/minimax_ui.py +++ b/modules/minimax/minimax_ui.py @@ -36,7 +36,8 @@ def create_ui(prompt, _negative, styles, overrides, script_inputs, mp4_fps, mp4_ random_seed = ToolButton(ui_symbols.random, elem_id='minimax_seed_random') random_seed.click(fn=lambda: -1, show_progress='hidden', inputs=[], outputs=[seed]) with gr.Row(): - audio_enable = gr.Checkbox(label='Enable audio', value=True, elem_id="minimax_audio_enable") + enable_audio = gr.Checkbox(label='Enable audio', value=True, elem_id="minimax_audio_enable") + enable_preview = gr.Checkbox(label='Enable preview', value=True, elem_id="minimax_preview_enable") with gr.Accordion(open=False, label="Input media", elem_id='minimax_input_media_accordion', visible=True) as input_accordion: with gr.Row(): init_image = gr.Image(label='Image', elem_id='minimax_init_image', type='pil', image_mode='RGB', width=256, height=256) @@ -97,7 +98,8 @@ def create_ui(prompt, _negative, styles, overrides, script_inputs, mp4_fps, mp4_ mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, mp4_scale, mp4_upscaler, - audio_enable, + enable_audio, + enable_preview, overrides, ] video_outputs = [ diff --git a/modules/minimax/minimax_video.py b/modules/minimax/minimax_video.py index 85d86fd73..a95ba1cc4 100644 --- a/modules/minimax/minimax_video.py +++ b/modules/minimax/minimax_video.py @@ -88,7 +88,7 @@ def generate(task_id, _ui_state, mp4_fps, mp4_interpolate, mp4_codec, mp4_ext, mp4_opt, mp4_video, mp4_frames, mp4_sf, mp4_thumb, mp4_scale, mp4_upscaler, - audio_enable, + enable_audio, enable_preview, _overrides, *args, **_kwargs, @@ -144,7 +144,7 @@ def generate(task_id, _ui_state, outpath_samples=paths.resolve_output_path(shared.opts.outdir_samples, shared.opts.outdir_video), ops=['video'], ) - video_minimax.apply_overrides(p, shared.sd_model, still=False, audio=audio_enable) + video_minimax.apply_overrides(p, shared.sd_model, still=False, audio=enable_audio, preview=enable_preview) video_minimax.set_sampler_shift(shared.sd_model, video_shift=video_shift, audio_shift=audio_shift) log.debug(f'Video: engine="{engine}" model="{model}" workflow={workflow} cls={shared.sd_model.__class__.__name__} shift={video_shift}:{audio_shift} kwargs={p.task_args}') processing.fix_seed(p) @@ -168,7 +168,7 @@ def generate(task_id, _ui_state, sd_models.offload_ondemand(shared.sd_model, reason='finish', force=True) # force offload all loaded modules to cpu devices.torch_gc(force=True) # free gpu memory before saving video - audio = getattr(processed, 'audio', None) if audio_enable else None + audio = getattr(processed, 'audio', None) if enable_audio else None if audio is not None: audio = audio[0].float().cpu() if audio.ndim == 3 else audio.float().cpu() aac_sample_rate = getattr(shared.sd_model, 'audio_sampling_rate', 32000) diff --git a/modules/modular_load.py b/modules/modular_load.py index 3579b6f72..97ac13a12 100644 --- a/modules/modular_load.py +++ b/modules/modular_load.py @@ -38,7 +38,7 @@ def is_modular(obj) -> bool: return 'Modular' in cls.__name__ -def preload_components(pipe, workflow: str | None, load_config: dict | None = None) -> dict: +def preload_components(pipe, workflow: str | None, load_config: dict | None = None, loaded: dict | None = None) -> dict: """Load the denoiser and text encoder through the shared loaders rather than the pipeline's own. `load_components` fetches every component into the pipeline's cache directory with no @@ -53,8 +53,10 @@ def preload_components(pipe, workflow: str | None, load_config: dict | None = No """ from pipelines import generic specs = getattr(pipe, '_component_specs', {}) # pylint: disable=protected-access - loaded = {} + loaded = loaded or {} for name in missing_components(pipe, workflow): + if name in loaded: + continue spec = specs.get(name) if spec is None or getattr(spec, 'default_creation_method', None) != 'from_pretrained': continue @@ -71,7 +73,7 @@ def preload_components(pipe, workflow: str | None, load_config: dict | None = No elif origin.startswith('transformers') and ('text_encoder' in name): # shared substitution is on: the map matches class plus a substring of the repo name, so its entries have to run narrow before broad component = generic.load_text_encoder(repo, cls_name=cls, load_config=load_config, subfolder=subfolder) - if 'transformer' in name: + if ('transformer' in name) and (component is None): # fallback for component with remote-code as it does not have resolvable cls component = generic.load_transformer(repo, cls_name=None, load_config=load_config, subfolder=subfolder, trust_remote_code=True) if component is not None: @@ -96,7 +98,15 @@ def missing_components(pipe, workflow: str | None) -> list: return [name for name in names if getattr(pipe, name, None) is None] -def load_modular_pipe(repo_cls, repo: str, workflow: str | None = None, revision: str | None = None, offline_args: dict | None = None, base: bool = False, load_config: dict | None = None): +def load_modular_pipe(repo_cls, + repo: str, + workflow: str | None = None, + revision: str | None = None, + offline_args: dict | None = None, + base: bool = False, + load_config: dict | None = None, + loaded: dict | None = None, + ): if repo_cls is None or isinstance(repo_cls, str): log.error(f'Load modular: repo="{repo}" cls="{repo_cls}" pipeline class not found: diffusers too old') return None @@ -112,7 +122,7 @@ def load_modular_pipe(repo_cls, repo: str, workflow: str | None = None, revision **offline_args, ) # the workflow restricts the component fetch only: passing it to from_pretrained instead would prune the blocks tree to one task and disable runtime dispatch between them - preloaded = preload_components(pipe, workflow, load_config=load_config) + preloaded = preload_components(pipe, workflow, load_config=load_config, loaded=loaded) if preloaded: pipe.update_components(**preloaded) # registered before the rest, which load_components then skips log.debug(f'Load modular: cls={pipe.__class__.__name__} preloaded={list(preloaded)}') diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py index 758d56ac2..cbf37b192 100644 --- a/modules/video_models/models_def.py +++ b/modules/video_models/models_def.py @@ -771,6 +771,18 @@ try: image_hijack=False, vae_hijack=False, vae_remote=False), + Model(name='MiniMax H3 Nunchaku-Lite', + url='https://huggingface.co/rootonchair/MiniMax-H3-nunchaku-lite-int4', + repo='rootonchair/MiniMax-H3-nunchaku-lite-int4', + repo_cls='MiniMaxH3ModularPipeline', + workflow='fl2va', + base=True, + te_cls=None, + dit_cls=None, + te_hijack=False, + image_hijack=False, + vae_hijack=False, + vae_remote=False), Model(name='MiniMax H3 VDN', url='https://huggingface.co/OpenVDN/vdn-minimax-h3', repo='OpenVDN/vdn-minimax-h3', diff --git a/modules/video_models/video_minimax.py b/modules/video_models/video_minimax.py index 658ac1c75..f52697689 100644 --- a/modules/video_models/video_minimax.py +++ b/modules/video_models/video_minimax.py @@ -5,7 +5,7 @@ from modules.logger import log MIN_LATENT_FRAMES = 7 # decoder floor: fewer latent frames leave the chunked decode with nothing to emit -def apply_overrides(p, pipe, still: bool = False, audio: bool = True): +def apply_overrides(p, pipe, still: bool = False, audio: bool = True, preview: bool = False): """Per-generation constraints shared by the video tab and the image path: canvas and frame alignment, the bespoke scheduler guard, tiling, and the audio/still toggles.""" if still: @@ -40,6 +40,13 @@ def apply_overrides(p, pipe, still: bool = False, audio: bool = True): p.task_args['output_type'] = 'pil' if still else 'np' p.video_still = still + if preview: + from pipelines.minimax.minimax_latents import unpack_latents + pipe.custom_unpack_latents = unpack_latents # add a helper to unpack the video latents from the block state + else: + if hasattr(pipe, 'custom_unpack_latents'): + del pipe.custom_unpack_latents + def set_still(pipe, enabled: bool = True): """Toggle sub-floor generation for single-frame output. The duration floor is lifted only diff --git a/pipelines/generic_transformer.py b/pipelines/generic_transformer.py index 801431e15..a33b1192c 100644 --- a/pipelines/generic_transformer.py +++ b/pipelines/generic_transformer.py @@ -24,7 +24,6 @@ def load_transformer( override_slot='primary', trust_remote_code=False, **kwargs): - """Load a DiT transformer from the base repo, or from a user-selected single file when the slot's UNET override dropdown is set. @@ -164,19 +163,19 @@ def load_transformer( **load_kwargs, ) - # 4. default loading from local file (also the fallback when an incompatible override is dropped above) # 5. default loading from diffusers repo (also the fallback when an incompatible override is dropped above) + # 4. default loading from diffusers repo (also the fallback when an incompatible override is dropped above) else: transformer = load_from_repo() # mark the dropdown selection as loaded so the slot's onchange callback, does not force a redundant full reload for an already-consumed override - if transformer is not None and override_name is not None and getattr(shared.opts, override_opt, None) == override_name: + if (transformer is not None) and (override_name is not None) and getattr(shared.opts, override_opt, None) == override_name: setattr(sd_unet, tracker_attr, override_name) sd_models.allow_post_quant = False # we already handled it if shared.opts.diffusers_offload_mode != 'none' and transformer is not None: sd_models.move_model(transformer, devices.cpu) - if transformer is not None and not hasattr(transformer, 'quantization_config'): # attach quantization_config + if (transformer is not None) and not hasattr(transformer, 'quantization_config'): # attach quantization_config if hasattr(transformer, 'config') and hasattr(transformer.config, 'quantization_config'): transformer.quantization_config = transformer.config.quantization_config elif (quant_type is not None) and (quant_args.get('quantization_config', None) is not None): diff --git a/pipelines/minimax/minimax_nunchaku.py b/pipelines/minimax/minimax_nunchaku.py new file mode 100644 index 000000000..d72de72f4 --- /dev/null +++ b/pipelines/minimax/minimax_nunchaku.py @@ -0,0 +1,20 @@ +import diffusers +from modules import shared +from modules.logger import log + + +def load_nunchaku(repo_id, load_config=None): + load_config = load_config or {} + + from modules.attention import hijack_kernels + hijack_kernels() + + cls_name = diffusers.MiniMaxH3Transformer3DModel + log.debug(f'Load model: transformer="{repo_id}" subfolder="calibrated-8x20" cls={cls_name.__name__} loader="nunchaku-lite" args={load_config}') + transformer = cls_name.from_pretrained( + repo_id, + subfolder="calibrated-8x20", + cache_dir=shared.opts.hfcache_dir, + **load_config, + ) + return transformer diff --git a/pipelines/model_minimax.py b/pipelines/model_minimax.py index 964f8629a..720d0313c 100644 --- a/pipelines/model_minimax.py +++ b/pipelines/model_minimax.py @@ -15,6 +15,14 @@ def load_minimax(checkpoint_info, diffusers_load_config = None, workflow: str | workflow = (workflow or getattr(checkpoint_info, 'subfolder', None) or 'fl2va').lower() # one repo holds both checkpoint partitions; reference entries select ref2va via the subfolder tag log.debug(f'Load model: type=MiniMaxH3 repo="{repo_id}" workflow={workflow} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype}') + loaded = {} + if 'nunchaku-lite' in repo_id.lower(): + from pipelines.minimax.minimax_nunchaku import load_nunchaku + transformer = load_nunchaku(repo_id, load_config=diffusers_load_config) + if transformer is not None: + loaded['transformer'] = transformer + repo_id = 'OzzyGT/MiniMax_H3_sdnq_dynamic_4bit' # nunchaku repo does not contain non-transformer modules + sd_models.warn_group_offload(min_vram=20) repo_cls = diffusers.MiniMaxH3ModularPipeline pipe = load_modular_pipe( @@ -24,6 +32,7 @@ def load_minimax(checkpoint_info, diffusers_load_config = None, workflow: str | offline_args=offline_args, base=True, load_config=diffusers_load_config, + loaded=loaded, ) if pipe is None: return None @@ -34,8 +43,9 @@ def load_minimax(checkpoint_info, diffusers_load_config = None, workflow: str | pipe.sdnext_supported_min_frames = int(pipe.min_duration * pipe.fps) # fresh pipes report the true floor; still mode gates per instance video_load.loaded_model = None # image-path load invalidates the video tab's name cache - # if hasattr(pipe, 'vae'): - # pipe.vae = pipe.vae.to(torch.float16) # minimax loads vae in float32 + if hasattr(pipe, 'vae'): + import torch + pipe.vae = pipe.vae.to(torch.float16) # minimax loads vae in float32 if hasattr(pipe, 'vae') and hasattr(pipe.vae, 'enable_tiling'): pipe.vae.enable_tiling()