diff --git a/CHANGELOG.md b/CHANGELOG.md index a4c209ac8..ead928686 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ ## Update for 2026-07-31 - **Models** + - [SeFi-Image](https://huggingface.co/SeFi-Image/SeFi-Image-5B-RL) in *Base*, *Turbo* (distilled) and *RL* (finetuned) variants + SeFi is an interesting model that separates generation into semantic and texture latent streams + and denoising semantic structure slightly ahead of texture details + SeFi comes in 1B, 2B and 5B variants - [Microsoft Mage-Flow](https://huggingface.co/mage-flow-community/Mage-Flow) in *Base* and *Turbo* (distilled) variants Mage-Flow is a 4B-scale generative stack for efficient text-to-image generation and instruction-based image editing *note*: Microsoft released and then unpublished the model, but we still have a mirror available for download diff --git a/data/reference-base.json b/data/reference-base.json index ff4c225ee..dba2f0ce1 100644 --- a/data/reference-base.json +++ b/data/reference-base.json @@ -926,5 +926,37 @@ "extras": "sampler: Default", "size": 16.19, "date": "2026 July" + }, + "SeFi-Image 1B Base": { + "path": "vladmandic/SeFi-Image-1B-Base", + "preview": "vladmandic--SeFi-Image-1B-Base.jpg", + "desc": "SeFi-Image is a text-to-image foundation model family built with Semantic-First Diffusion. It separates generation into semantic and texture latent streams, denoising semantic structure slightly ahead of texture details.", + "extras": "sampler: Default", + "size": 0, + "date": "2026 July" + }, + "SeFi-Image 2B Base": { + "path": "vladmandic/SeFi-Image-2B-Base", + "preview": "vladmandic--SeFi-Image-2B-Base.jpg", + "desc": "SeFi-Image is a text-to-image foundation model family built with Semantic-First Diffusion. It separates generation into semantic and texture latent streams, denoising semantic structure slightly ahead of texture details.", + "extras": "sampler: Default", + "size": 0, + "date": "2026 July" + }, + "SeFi-Image 5B Base": { + "path": "vladmandic/SeFi-Image-5B-Base", + "preview": "vladmandic--SeFi-Image-5B-Base.jpg", + "desc": "SeFi-Image is a text-to-image foundation model family built with Semantic-First Diffusion. It separates generation into semantic and texture latent streams, denoising semantic structure slightly ahead of texture details.", + "extras": "sampler: Default", + "size": 0, + "date": "2026 July" + }, + "SeFi-Image 5B RL": { + "path": "vladmandic/SeFi-Image-5B-RL", + "preview": "vladmandic--SeFi-Image-5B-RL.jpg", + "desc": "SeFi-Image is a text-to-image foundation model family built with Semantic-First Diffusion. It separates generation into semantic and texture latent streams, denoising semantic structure slightly ahead of texture details.", + "extras": "sampler: Default", + "size": 0, + "date": "2026 July" } } diff --git a/data/reference-distilled.json b/data/reference-distilled.json index 0c555b820..ceb3a5d32 100644 --- a/data/reference-distilled.json +++ b/data/reference-distilled.json @@ -233,5 +233,29 @@ "extras": "sampler: Default", "size": 16.19, "date": "2026 July" + }, + "SeFi-Image 1B Turbo": { + "path": "vladmandic/SeFi-Image-1B-Turbo", + "preview": "vladmandic--SeFi-Image-1B-Turbo.jpg", + "desc": "SeFi-Image is a text-to-image foundation model family built with Semantic-First Diffusion. It separates generation into semantic and texture latent streams, denoising semantic structure slightly ahead of texture details.", + "extras": "sampler: Default", + "size": 0, + "date": "2026 July" + }, + "SeFi-Image 2B Turbo": { + "path": "vladmandic/SeFi-Image-2B-Turbo", + "preview": "vladmandic--SeFi-Image-2B-Turbo.jpg", + "desc": "SeFi-Image is a text-to-image foundation model family built with Semantic-First Diffusion. It separates generation into semantic and texture latent streams, denoising semantic structure slightly ahead of texture details.", + "extras": "sampler: Default", + "size": 0, + "date": "2026 July" + }, + "SeFi-Image 5B Turbo": { + "path": "vladmandic/SeFi-Image-5B-Turbo", + "preview": "vladmandic--SeFi-Image-5B-Turbo.jpg", + "desc": "SeFi-Image is a text-to-image foundation model family built with Semantic-First Diffusion. It separates generation into semantic and texture latent streams, denoising semantic structure slightly ahead of texture details.", + "extras": "sampler: Default", + "size": 0, + "date": "2026 July" } } diff --git a/pipelines/model_anima.py b/pipelines/model_anima.py index 488b0074a..cbf10885c 100644 --- a/pipelines/model_anima.py +++ b/pipelines/model_anima.py @@ -58,48 +58,6 @@ def load_anima(checkpoint_info, diffusers_load_config=None): if repo_id is None or repo_id.lower() == 'none': return None - # load-or-download custom pipeline modules from repo - """ - import os - import sys - import huggingface_hub as hf - - if os.path.exists(os.path.join(repo_id, 'pipeline.py')): - pipeline_file = os.path.join(repo_id, 'pipeline.py') - else: - try: - if os.path.exists(repo_id): - from pipelines.generic_map import transformers_map - custom_id = transformers_map.get('AnimaTextToImagePipeline', repo_id) - else: - custom_id = repo_id - pipeline_file = hf.hf_hub_download(repo_id=custom_id, filename='pipeline.py', cache_dir=shared.opts.hfcache_dir) - except Exception as e: - log.error(f'Load model: type=Anima failed to download custom modules: {e}') - return None - if os.path.exists(os.path.join(repo_id, 'llm_adapter/modeling_llm_adapter.py')): - adapter_file = os.path.join(repo_id, 'llm_adapter/modeling_llm_adapter.py') - else: - try: - if os.path.exists(repo_id): - from pipelines.generic_map import transformers_map - custom_id = transformers_map.get('AnimaTextToImagePipeline', repo_id) - else: - custom_id = repo_id - adapter_file = hf.hf_hub_download(repo_id=custom_id, filename='llm_adapter/modeling_llm_adapter.py', cache_dir=shared.opts.hfcache_dir) - except Exception as e: - log.error(f'Load model: type=Anima failed to download custom modules: {e}') - return None - - # dynamically import custom classes and register in sys.modules so Diffusers' from_pretrained can resolve them via trust_remote_code - adapter_mod = _import_from_file('modeling_llm_adapter', adapter_file) - sys.modules['modeling_llm_adapter'] = adapter_mod - pipeline_mod = _import_from_file('pipeline', pipeline_file) - sys.modules['pipeline'] = pipeline_mod - AnimaTextToImagePipeline = pipeline_mod.AnimaTextToImagePipeline - AnimaLLMAdapter = adapter_mod.AnimaLLMAdapter - """ - import sys from pipelines.anima import modeling_llm_adapter sys.modules['modeling_llm_adapter'] = modeling_llm_adapter diff --git a/pipelines/model_boogu.py b/pipelines/model_boogu.py index fa62ac75c..e6c9dfd34 100644 --- a/pipelines/model_boogu.py +++ b/pipelines/model_boogu.py @@ -22,20 +22,20 @@ def load_boogu(checkpoint_info, diffusers_load_config=None): from pipelines.boogu import transformer_boogu, scheduling_flow_match_euler_discrete_time_shifting sys.modules['transformer_boogu'] = transformer_boogu # for loading custom code from HF repo sys.modules['scheduling_flow_match_euler_discrete_time_shifting'] = scheduling_flow_match_euler_discrete_time_shifting # for loading custom code from HF repo - scheduler = scheduling_flow_match_euler_discrete_time_shifting.FlowMatchEulerDiscreteScheduler.from_pretrained(repo_id, subfolder='scheduler', cache_dir=shared.opts.diffusers_dir) + generic.set_pipeline('Boogu', BooguImagePipeline) if repo_id is None or repo_id.lower() == 'none': return None mllm = generic.load_text_encoder(repo_id, cls_name=transformers.Qwen3VLForConditionalGeneration, load_config=diffusers_load_config, subfolder='mllm') transformer = generic.load_transformer(repo_id, cls_name=BooguImageTransformer2DModel, load_config=diffusers_load_config) + scheduler = scheduling_flow_match_euler_discrete_time_shifting.FlowMatchEulerDiscreteScheduler.from_pretrained(repo_id, subfolder='scheduler', cache_dir=shared.opts.diffusers_dir) if 'turbo' in repo_id.lower(): cls = BooguImageTurboPipeline else: cls = BooguImagePipeline - generic.set_pipeline('Boogu', cls) diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING['boogu'] = cls diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING['boogu'] = cls diff --git a/pipelines/model_mageflow.py b/pipelines/model_mageflow.py index cefa83324..e9ec72fe8 100644 --- a/pipelines/model_mageflow.py +++ b/pipelines/model_mageflow.py @@ -18,14 +18,15 @@ def load_mageflow(checkpoint_info, diffusers_load_config=None): from pipelines.mageflow import MageFlowPipeline, MageFlowTransformer2DModel log.debug(f'Load model: type=MageFlow repo="{repo_id}" config={diffusers_load_config} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={diffusers_load_config}') - transformer = generic.load_transformer(repo_id, cls_name=MageFlowTransformer2DModel, load_config=diffusers_load_config) - text_encoder = generic.load_text_encoder(repo_id, cls_name=transformers.Qwen3VLForConditionalGeneration, load_config=diffusers_load_config) - tokenizer = transformers.Qwen2Tokenizer.from_pretrained(repo_id, subfolder="text_encoder", cache_dir=shared.opts.diffusers_dir) + generic.set_pipeline('MageFlow', MageFlowPipeline) if repo_id is None or repo_id.lower() == 'none': return None - generic.set_pipeline('MageFlow', MageFlowPipeline) + transformer = generic.load_transformer(repo_id, cls_name=MageFlowTransformer2DModel, load_config=diffusers_load_config) + text_encoder = generic.load_text_encoder(repo_id, cls_name=transformers.Qwen3VLForConditionalGeneration, load_config=diffusers_load_config) + tokenizer = transformers.Qwen2Tokenizer.from_pretrained(repo_id, subfolder="text_encoder", cache_dir=shared.opts.diffusers_dir) + diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING['mageflow'] = MageFlowPipeline diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING['mageflow'] = MageFlowPipeline diff --git a/pipelines/model_prx.py b/pipelines/model_prx.py index 1fbaba01a..5a35e601d 100644 --- a/pipelines/model_prx.py +++ b/pipelines/model_prx.py @@ -20,6 +20,7 @@ def load_prx(checkpoint_info, diffusers_load_config=None): if repo_id is None or repo_id.lower() == 'none': return None + pipe = diffusers.PRXPipeline.from_pretrained( repo_id, transformer=transformer, diff --git a/pipelines/model_sefi.py b/pipelines/model_sefi.py index aa62b46db..e1cf8f237 100644 --- a/pipelines/model_sefi.py +++ b/pipelines/model_sefi.py @@ -18,6 +18,7 @@ def load_sefi(checkpoint_info, diffusers_load_config=None): transformer = generic.load_transformer(repo_id, cls_name=SeFiTransformer2DModel, load_config=diffusers_load_config) text_encoder = generic.load_text_encoder(repo_id, cls_name=transformers.Qwen3VLForConditionalGeneration, load_config=diffusers_load_config) + generic.set_pipeline('SeFi', SeFiPipeline) if repo_id is None or repo_id.lower() == 'none': return None @@ -31,6 +32,10 @@ def load_sefi(checkpoint_info, diffusers_load_config=None): diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["sefi"] = SeFiPipeline + pipe.task_args = { + "output_type": "np", + } + generic.load_vae_override(pipe, diffusers_load_config) del text_encoder diff --git a/pipelines/sefi/__init__.py b/pipelines/sefi/__init__.py index 282265fa0..efd4d82d2 100644 --- a/pipelines/sefi/__init__.py +++ b/pipelines/sefi/__init__.py @@ -1,3 +1,7 @@ from .transformer_sefi import SeFiTransformer2DModel from .pipeline_sefi import SeFiPipeline from .pipeline_output import SeFiPipelineOutput + +import diffusers +diffusers.SeFiTransformer2DModel = SeFiTransformer2DModel +diffusers.SeFiPipeline = SeFiPipeline diff --git a/pipelines/sefi/convert_sefi_to_diffusers.py b/pipelines/sefi/convert_sefi_to_diffusers.py index e55280a57..52d6c2565 100644 --- a/pipelines/sefi/convert_sefi_to_diffusers.py +++ b/pipelines/sefi/convert_sefi_to_diffusers.py @@ -24,7 +24,7 @@ from huggingface_hub import snapshot_download from safetensors.torch import load_file from diffusers import __version__ -from diffusers.models import SeFiTransformer2DModel +from transformer_sefi import SeFiTransformer2DModel SEFI_SCALE_PRESETS = { diff --git a/pipelines/sefi/pipeline_sefi.py b/pipelines/sefi/pipeline_sefi.py index 520d5bb34..120b4690c 100644 --- a/pipelines/sefi/pipeline_sefi.py +++ b/pipelines/sefi/pipeline_sefi.py @@ -633,7 +633,7 @@ class SeFiPipeline(DiffusionPipeline): self._current_timestep = base_sigmas_schedule[i] packed_latents = self._pack_latents(latents) - pred_cond = self.transformer( + noise_pred = self.transformer( hidden_states=packed_latents, timestep_sem=timesteps_sem_cur / 1000, timestep_tex=timesteps_tex_cur / 1000, @@ -643,7 +643,7 @@ class SeFiPipeline(DiffusionPipeline): joint_attention_kwargs=self.attention_kwargs, return_dict=False, )[0] - pred_cond = pred_cond[:, : packed_latents.size(1)] + pred_cond = noise_pred[:, : packed_latents.size(1)] pred_cond = self._unpack_latents_with_ids(pred_cond, latent_ids) if self.do_classifier_free_guidance: