diff --git a/CHANGELOG.md b/CHANGELOG.md index db39a3aba..33bf89bae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,15 +12,22 @@ includes *Normal*, *Edit*, and *Lite* (distilled) variants features lightweight SmolLM3-3B text encoder with efficient inference *note*: this is a [gated model](https://vladmandic.github.io/sdnext-docs/Gated/) + - [VIBE Image Edit](https://huggingface.co/iitolstykh/VIBE-Image-Edit) visual instruction based image editing model + combines *Sana1.5-1.6B* diffusion backbone with *Qwen3-VL-2B* multimodal conditioning - [Lumina-DiMOO](https://huggingface.co/Alpha-VLLM/Lumina-DiMOO) omni diffusion large language model for multimodal generation and understanding supports *text-to-image*, *image editing*, and *image understanding* in a unified discrete diffusion framework uses a VQ-VAE backed token space with any-to-any task coverage + - [Step1X-Edit v1.2](https://huggingface.co/stepfun-ai/Step1X-Edit-v1p2) image editing model using Step1X transformer + integrates Qwen2.5-VL text encoder and Flow Matching scheduler for high-quality in-context edits + enables multimodal understanding and conditional image refinement + category: *base* - **Caption & Prompt Enhance** - [Google Gemma 4] in *E2B* and *E4B* variants as well as *heretic* fine-tune - **Compute** - **ROCm** futher work on advanced configuration and tuning, thanks @resonantsky now covers both ROCm on Windows and Linux see *main interface -> scripts -> rocm advanced config* + - **SDNQ** add quant support to `nn.Embedding` type - **Features** - **nudenet** add `LlavaGuard` and `QwenGuard` as image safety evaulation models - installer auto-restart on upgrade diff --git a/TODO.md b/TODO.md index ae39d504c..674d1c900 100644 --- a/TODO.md +++ b/TODO.md @@ -44,24 +44,15 @@ TODO: Investigate which models are diffusers-compatible and prioritize! -### Image-Base +### Image - [Mugen](https://huggingface.co/CabalResearch/Mugen) - [NucleusMoe](https://github.com/huggingface/diffusers/pull/13317) -- [Liquid](https://github.com/FoundationVision/Liquid): Unified vision-language auto-regressive generation paradigm -- [nVidia Cosmos-Predict-2.5](https://huggingface.co/nvidia/Cosmos-Predict2.5-2B): Physics-aware world foundation model for consistent scene prediction -- [Liquid (unified multimodal generator)](https://github.com/FoundationVision/Liquid): Auto-regressive generation paradigm across vision and language - -### Image-Edit - +- [Liquid](https://github.com/FoundationVision/Liquid) +- [nVidia Cosmos-Predict-2.5](https://huggingface.co/nvidia/Cosmos-Predict2.5-2B) +- [Liquid (unified multimodal generator)](https://github.com/FoundationVision/Liquid) - [Tencent HY-WU](https://huggingface.co/tencent/HY-WU) - [JoyAI Image Edit](https://huggingface.co/jdopensource/JoyAI-Image-Edit) -- [Meituan LongCat-Image-Edit-Turbo](https://huggingface.co/meituan-longcat/LongCat-Image-Edit-Turbo):6B instruction-following image editing with high visual consistency -- [VIBE Image-Edit](https://huggingface.co/iitolstykh/VIBE-Image-Edit): (Sana+Qwen-VL)Fast visual instruction-based image editing framework -- [Step1X-Edit](https://github.com/stepfun-ai/Step1X-Edit):Multimodal image editing decoding MLLM tokens via DiT -- [Step1X-Edit-v1p2](https://huggingface.co/stepfun-ai/Step1X-Edit-v1p2) -- [OneReward](https://github.com/bytedance/OneReward):Reinforcement learning grounded generative reward model for image editing -- [ByteDance DreamO](https://huggingface.co/ByteDance/DreamO): image customization framework for IP adaptation and virtual try-on - [nVidia Cosmos-Transfer-2.5](https://github.com/huggingface/diffusers/pull/13066) ### Video @@ -93,6 +84,8 @@ TODO: Investigate which models are diffusers-compatible and prioritize! ### Other/Unsorted +- [OneReward](https://github.com/bytedance/OneReward) +- [ByteDance DreamO](https://huggingface.co/ByteDance/DreamO) - [DiffusionForcing](https://github.com/kwsong0113/diffusion-forcing-transformer): Full-sequence diffusion with autoregressive next-token prediction - [Self-Forcing](https://github.com/guandeh17/Self-Forcing): Framework for improving temporal consistency in long-horizon video generation - [SEVA](https://github.com/huggingface/diffusers/pull/11440): Stable Virtual Camera for novel view synthesis and 3D-consistent video @@ -108,18 +101,6 @@ TODO: Investigate which models are diffusers-compatible and prioritize! - [FlashFace](https://github.com/ali-vilab/FlashFace): High-fidelity human image customization and face swapping framework - [ReNO](https://github.com/ExplainableML/ReNO): Reward-based Noise Optimization to improve text-to-image quality during inference -### Not Planned - -- [LoRAdapter](https://github.com/CompVis/LoRAdapter): Not recently updated -- [SD3 UltraEdit](https://github.com/HaozheZhao/UltraEdit): Based on SD3 -- [PowerPaint](https://github.com/open-mmlab/PowerPaint): Based on SD15 -- [FreeCustom](https://github.com/aim-uofa/FreeCustom): Based on SD15 -- [AnyDoor](https://github.com/ali-vilab/AnyDoor): Based on SD21 -- [AnyText2](https://github.com/tyxsspa/AnyText2): Based on SD15 -- [DragonDiffusion](https://github.com/MC-E/DragonDiffusion): Based on SD15 -- [DenseDiffusion](https://github.com/naver-ai/DenseDiffusion): Based on SD15 -- [IC-Light](https://github.com/lllyasviel/IC-Light): Based on SD15 - ## Code TODO > npm run todo diff --git a/data/reference.json b/data/reference.json index d2a0508f0..52b1f2e7d 100644 --- a/data/reference.json +++ b/data/reference.json @@ -273,6 +273,26 @@ "date": "2025 December" }, + "StepFun Step1X-Edit v1.2": { + "path": "stepfun-ai/Step1X-Edit-v1p2", + "preview": "stepfun-ai--Step1X-Edit-v1p2.jpg", + "desc": "Multimodal image editing model using Step1X transformer architecture with Qwen2.5-VL text encoding, trained with Flow Matching scheduler for high-quality in-context image edits and refinements.", + "skip": true, + "extras": "sampler: Default", + "size": 8.58, + "date": "2025 October" + }, + + "VIBE Image Edit": { + "path": "iitolstykh/VIBE-Image-Edit", + "preview": "iitolstykh--VIBE-Image-Edit.jpg", + "desc": "VIBE is an open-source text-guided image editing model combining Sana1.5-1.6B diffusion backbone with Qwen3-VL multimodal conditioning for fast, instruction-based edits.", + "skip": true, + "extras": "sampler: Default, cfg_scale: 4.5, image_guidance_scale: 1.2, steps: 20", + "size": 9.72, + "date": "2025 December" + }, + "Ostris Flex.2 Preview": { "path": "ostris/Flex.2-preview", "preview": "ostris--Flex.2-preview.jpg", diff --git a/models/Reference/iitolstykh--VIBE-Image-Edit.jpg b/models/Reference/iitolstykh--VIBE-Image-Edit.jpg new file mode 100644 index 000000000..e69de29bb diff --git a/models/Reference/stepfun-ai--Step1X-Edit-v1p2.jpg b/models/Reference/stepfun-ai--Step1X-Edit-v1p2.jpg new file mode 100644 index 000000000..e69de29bb diff --git a/modules/modeldata.py b/modules/modeldata.py index be9fa821f..7e2ece774 100644 --- a/modules/modeldata.py +++ b/modules/modeldata.py @@ -68,6 +68,8 @@ def get_model_type(pipe): model_type = 'cogview4' elif "Sana" in name: model_type = 'sana' + elif 'VIBE' in name: + model_type = 'sana' elif "HiDream" in name: model_type = 'h1' elif "Cosmos2TextToImage" in name or "AnimaTextToImage" in name: @@ -96,6 +98,8 @@ def get_model_type(pipe): model_type = 'longcat' elif 'GlmImage' in name: model_type = 'glmimage' + elif 'Step1XEdit' in name: + model_type = 'step1x_edit' elif 'OvisImage' in name or 'Ovis-Image' in name: model_type = 'ovis' elif 'Wan' in name: diff --git a/modules/sd_detect.py b/modules/sd_detect.py index 7c2170986..bde8e5891 100644 --- a/modules/sd_detect.py +++ b/modules/sd_detect.py @@ -152,6 +152,10 @@ def guess_by_name(fn, current_guess): new_guess = 'GLM-Image' elif 'sdxs-1b' in fn.lower(): new_guess = 'SDXS' + elif 'step1x-edit' in fn.lower(): + new_guess = 'Step1X-Edit' + elif 'vibe-image-edit' in fn.lower(): + new_guess = 'VIBE' if debug_load: log.trace(f'Autodetect: method=name file="{fn}" previous="{current_guess}" current="{new_guess}"') return new_guess or current_guess diff --git a/modules/sd_models.py b/modules/sd_models.py index 0a72a8ac8..e9808663d 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -453,6 +453,14 @@ def load_diffuser_force(detected_model_type, checkpoint_info, diffusers_load_con from pipelines.model_bria import load_bria sd_model = load_bria(checkpoint_info, diffusers_load_config) allow_post_quant = False + elif model_type in ['Step1X-Edit']: + from pipelines.model_step1x_edit import load_step1x_edit + sd_model = load_step1x_edit(checkpoint_info, diffusers_load_config) + allow_post_quant = False + elif model_type in ['VIBE']: + from pipelines.model_vibe import load_vibe + sd_model = load_vibe(checkpoint_info, diffusers_load_config) + allow_post_quant = False elif model_type in ['Qwen']: from pipelines.model_qwen import load_qwen sd_model = load_qwen(checkpoint_info, diffusers_load_config) diff --git a/modules/shared_items.py b/modules/shared_items.py index ac14aef3c..a417d60bf 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -53,6 +53,7 @@ pipelines = { 'FLUX2 Klein': getattr(diffusers, 'Flux2KleinPipeline', None), 'LongCat': getattr(diffusers, 'LongCatImagePipeline', None), 'GLM-Image': getattr(diffusers, 'GlmImagePipeline', None), + 'VIBE': getattr(diffusers, 'VIBESanaEditingPipeline', getattr(diffusers, 'DiffusionPipeline', None)), # dynamically imported and redefined later 'Meissonic': getattr(diffusers, 'DiffusionPipeline', None), 'OmniGen2': getattr(diffusers, 'DiffusionPipeline', None), @@ -65,6 +66,7 @@ pipelines = { 'ChronoEdit': getattr(diffusers, 'DiffusionPipeline', None), 'Anima': getattr(diffusers, 'DiffusionPipeline', None), 'SDXS': getattr(diffusers, 'DiffusionPipeline', None), + 'Step1X-Edit': getattr(diffusers, 'DiffusionPipeline', None), } diff --git a/pipelines/model_step1x_edit.py b/pipelines/model_step1x_edit.py new file mode 100644 index 000000000..e15edadd3 --- /dev/null +++ b/pipelines/model_step1x_edit.py @@ -0,0 +1,49 @@ +import transformers +import diffusers +from modules import shared, devices, sd_models, model_quant, sd_hijack_te +from modules.logger import log +from pipelines import generic + + +def load_step1x_edit(checkpoint_info, diffusers_load_config=None): + if diffusers_load_config is None: + diffusers_load_config = {} + repo_id = sd_models.path_to_repo(checkpoint_info) + sd_models.hf_auth_check(checkpoint_info) + + load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config, allow_quant=False) + log.debug(f'Load model: type=Step1XEdit repo="{repo_id}" config={diffusers_load_config} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}') + + # Load text encoder (Qwen2.5-VL - available in transformers) + text_encoder = generic.load_text_encoder(repo_id, cls_name=transformers.Qwen2_5_VLForConditionalGeneration, load_config=diffusers_load_config) + + # Load processor for Qwen2.5-VL + processor = transformers.Qwen2_5_VLProcessor.from_pretrained(repo_id, cache_dir=shared.opts.hfcache_dir) + + # Step1XEditPipeline and Step1XEditTransformer2DModel are custom classes not in current diffusers + # Try direct pipeline class first, fall back to trust_remote_code + pipe_cls = getattr(diffusers, 'Step1XEditPipeline', None) + if pipe_cls is not None: + pipe = pipe_cls.from_pretrained( + repo_id, + cache_dir=shared.opts.diffusers_dir, + text_encoder=text_encoder, + processor=processor, + **load_args, + ) + else: + pipe = diffusers.DiffusionPipeline.from_pretrained( + repo_id, + cache_dir=shared.opts.diffusers_dir, + text_encoder=text_encoder, + processor=processor, + trust_remote_code=True, + **load_args, + ) + + del text_encoder + del processor + sd_hijack_te.init_hijack(pipe) + + devices.torch_gc(force=True, reason='load') + return pipe diff --git a/pipelines/model_vibe.py b/pipelines/model_vibe.py new file mode 100644 index 000000000..c6969839e --- /dev/null +++ b/pipelines/model_vibe.py @@ -0,0 +1,75 @@ +import diffusers +import transformers +from modules import shared, devices, sd_models, model_quant, sd_hijack_te, sd_hijack_vae +from modules.logger import log +from pipelines import generic + + +def load_vibe(checkpoint_info, diffusers_load_config=None): + if diffusers_load_config is None: + diffusers_load_config = {} + repo_id = sd_models.path_to_repo(checkpoint_info) + sd_models.hf_auth_check(checkpoint_info) + + load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config, allow_quant=False) + log.debug(f'Load model: type=VIBE repo="{repo_id}" config={diffusers_load_config} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}') + + pipe_cls = getattr(diffusers, 'VIBESanaEditingPipeline', None) + transformer_cls = getattr(diffusers, 'VIBESanaEditingModel', None) + text_encoder_cls = getattr(transformers, 'Qwen3VLForConditionalGeneration', None) + processor_cls = getattr(transformers, 'Qwen3VLProcessor', None) + + if pipe_cls is not None and transformer_cls is not None and text_encoder_cls is not None and processor_cls is not None: + transformer = generic.load_transformer( + repo_id, + cls_name=transformer_cls, + load_config=diffusers_load_config, + allow_quant=False, + ) + text_encoder = generic.load_text_encoder( + repo_id, + cls_name=text_encoder_cls, + load_config=diffusers_load_config, + allow_quant=False, + allow_shared=False, + ) + processor = processor_cls.from_pretrained(repo_id, subfolder='tokenizer', cache_dir=shared.opts.hfcache_dir) + + pipe = pipe_cls.from_pretrained( + repo_id, + cache_dir=shared.opts.diffusers_dir, + transformer=transformer, + text_encoder=text_encoder, + tokenizer=processor, + **load_args, + ) + + del transformer + del text_encoder + del processor + else: + try: + import vibe # pylint: disable=unused-import + except Exception as e: + raise RuntimeError('VIBE requires either native diffusers VIBESana classes or `vibe` package') from e + + pipe = diffusers.DiffusionPipeline.from_pretrained( + repo_id, + cache_dir=shared.opts.diffusers_dir, + trust_remote_code=True, + **load_args, + ) + + diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING['vibe-sana'] = pipe.__class__ + diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING['vibe-sana'] = pipe.__class__ + diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING['vibe-sana'] = pipe.__class__ + + pipe.task_args = { + 'output_type': 'np', + } + + sd_hijack_te.init_hijack(pipe) + sd_hijack_vae.init_hijack(pipe) + + devices.torch_gc(force=True, reason='load') + return pipe