add hidream-e1

Signed-off-by: Vladimir Mandic <mandic00@live.com>

.pylintrc

@@ -23,6 +23,7 @@ ignore-paths=/usr/lib/.*$,
              modules/k-diffusion,
              modules/flex2,
              modules/ldsr,
+             modules/hidream,
              modules/meissonic,
              modules/mod,
              modules/omnigen,

.ruff.toml

@@ -20,6 +20,7 @@ exclude = [
     "modules/meissonic",
     "modules/mod",
     "modules/omnigen",
+    "modules/hidream",
     "modules/pag",
     "modules/pixelsmith",
     "modules/postprocess/aurasr_arch.py",

CHANGELOG.md

@@ -1,10 +1,19 @@
 # Change Log for SD.Next
 
-## Update for 2025-04-28
+## Update for 2025-04-29
 
-- Prompt-Enhhance: add **Qwen3** 0.6B/1.7B/4B models  
-- Prompt-Enhhance: add thinking mode support (for models that have it)  
-- Docker: pre-install `ffmpeg`  
+- **Features**
+  - [HiDream-E1](https://huggingface.co/HiDream-ai/HiDream-E1-Full) natural language image-editing model built on HiDream-I1  
+    available via  *networks -> models -> reference*  
+    *note*: right now hidream-e1 is limited to 768x768 images, so you must force resize image before running it  
+  - Prompt-Enhhance: add **Qwen3** 0.6B/1.7B/4B models  
+  - Prompt-Enhhance: add thinking mode support (for models that have it)  
+- **Other**  
+  - FramePack: improve performance  
+  - Docker: pre-install `ffmpeg`  
+- **Fixes**  
+  - FramePack: correct dtype  
+  - NNCF: check dependencies and register quant type  
 
 ## Highlights for 2025-04-28
 

html/reference.json

@@ -364,6 +364,13 @@
     "skip": true,
     "extras": "sampler: Default"
   },
+  "HiDream-E1 Full": {
+    "path": "HiDream-ai/HiDream-E1-Full",
+    "desc": "HiDream-E1 is an image editing model built on HiDream-I1.",
+    "preview": "HiDream-ai--HiDream-I1-Fast.jpg",
+    "skip": true,
+    "extras": "sampler: Default"
+  },
 
   "Kwai Kolors": {
     "path": "Kwai-Kolors/Kolors-diffusers",

modules/model_flex.py

@@ -82,8 +82,8 @@ def load_flex(checkpoint_info, diffusers_load_config={}):
     )
     sd_hijack_te.init_hijack(pipe)
     diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["flex2"] = Flex2Pipeline
-    diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["fluxcfgzero"] = Flex2Pipeline
-    diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING["fluxcfgzero"] = Flex2Pipeline
+    diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["flex2"] = Flex2Pipeline
+    diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING["flex2"] = Flex2Pipeline
 
     del text_encoder_2
     del transformer

modules/model_hidream.py

@@ -42,6 +42,8 @@ def load_transformer(repo_id, diffusers_load_config={}):
 
 
 def load_text_encoders(repo_id, diffusers_load_config={}):
+    if repo_id == 'HiDream-ai/HiDream-E1-Full':
+        repo_id = 'HiDream-ai/HiDream-I1-Full' # use I1 for t5 and llm
     load_args, quant_args = model_quant.get_dit_args(diffusers_load_config, module='TE', device_map=True)
     shared.log.debug(f'Load model: type=HiDream te3="{repo_id}" quant="{model_quant.get_quant_type(quant_args)}" args={load_args}')
     text_encoder_3 = transformers.T5EncoderModel.from_pretrained(
@@ -92,7 +94,19 @@ def load_hidream(checkpoint_info, diffusers_load_config={}):
     load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config, module='Model')
     shared.log.debug(f'Load model: type=HiDream model="{checkpoint_info.name}" repo="{repo_id}" offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}')
 
-    pipe = diffusers.HiDreamImagePipeline.from_pretrained(
+    if 'I1' in repo_id:
+        cls = diffusers.HiDreamImagePipeline
+    elif 'E1' in repo_id:
+        from modules.hidream.pipeline_hidream_image_editing import HiDreamImageEditingPipeline
+        cls = HiDreamImageEditingPipeline
+        diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["hidream-e1"] = diffusers.HiDreamImagePipeline
+        diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["hidream-e1"] = HiDreamImageEditingPipeline
+        diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING["hidream-e1"] = HiDreamImageEditingPipeline
+    else:
+        shared.log.error(f'Load model: type=HiDream model="{checkpoint_info.name}" repo="{repo_id}" not recognized')
+        return False
+
+    pipe = cls.from_pretrained(
         repo_id,
         transformer=transformer,
         text_encoder_3=text_encoder_3,
@@ -101,6 +115,7 @@ def load_hidream(checkpoint_info, diffusers_load_config={}):
         cache_dir=shared.opts.diffusers_dir,
         **load_args,
     )
+
     sd_hijack_te.init_hijack(pipe)
     del text_encoder_3
     del text_encoder_4

modules/model_quant.py

@@ -263,6 +263,8 @@ def load_nncf(msg='', silent=False):
     if not installed('nncf'):
         install('nncf==2.16.0', quiet=True)
         log.warning('Quantization: nncf installed please restart')
+    install('jstyleson', quiet=True)
+    install('texttable', quiet=True)
     try:
         import nncf
         intel_nncf = nncf
@@ -350,6 +352,7 @@ def nncf_compress_model(model, op=None, sd_model=None, send_to_device=True, do_g
     num_bits = 8 if shared.opts.nncf_compress_weights_mode in {"INT8", "INT8_SYM", "INT8_ASYM"} else 4
     is_asym_mode = shared.opts.nncf_compress_weights_mode in {"INT8", "INT4", "INT8_ASYM", "INT4_ASYM"}
     model = apply_nncf_to_module(model, num_bits, is_asym_mode, quant_conv=shared.opts.nncf_quantize_conv_layers)
+    model.quantization_method = 'NNCF'
     if send_to_device:
         nncf_send_to_device(model, devices.device)
 

modules/processing_args.py

@@ -144,7 +144,7 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t
         'StableDiffusion' in model.__class__.__name__ or
         'StableCascade' in model.__class__.__name__ or
         'Flux' in model.__class__.__name__ or
-        'HiDreamImage' in model.__class__.__name__
+        'HiDreamImagePipeline' in model.__class__.__name__ # hidream-e1 has different embeds
     ):
         try:
             prompt_parser_diffusers.embedder = prompt_parser_diffusers.PromptEmbedder(prompts, negative_prompts, steps, clip_skip, p)
@@ -161,7 +161,7 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t
     if 'prompt' in possible:
         if 'OmniGen' in model.__class__.__name__:
             prompts = [p.replace('|image|', '<|image_1|>') for p in prompts]
-        if 'HiDreamImage' in model.__class__.__name__:
+        if 'HiDreamImage' in model.__class__.__name__  and prompt_parser_diffusers.embedder is not None:
             args['pooled_prompt_embeds'] = prompt_parser_diffusers.embedder('positive_pooleds')
             prompt_embeds = prompt_parser_diffusers.embedder('prompt_embeds')
             args['prompt_embeds_t5'] = prompt_embeds[0]
@@ -180,7 +180,7 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t
         else:
             args['prompt'] = prompts
     if 'negative_prompt' in possible:
-        if 'HiDreamImage' in model.__class__.__name__:
+        if 'HiDreamImage' in model.__class__.__name__ and prompt_parser_diffusers.embedder is not None:
             args['negative_pooled_prompt_embeds'] = prompt_parser_diffusers.embedder('negative_pooleds')
             negative_prompt_embeds = prompt_parser_diffusers.embedder('negative_prompt_embeds')
             args['negative_prompt_embeds_t5'] = negative_prompt_embeds[0]

modules/processing_helpers.py

@@ -458,6 +458,8 @@ def calculate_base_steps(p, use_denoise_start, use_refiner_start):
             steps = p.steps // (1 - p.refiner_start)
         elif 'Flex' in shared.sd_model.__class__.__name__:
             steps = p.steps
+        elif 'HiDreamImageEditingPipeline' in shared.sd_model.__class__.__name__:
+            steps = p.steps
         elif shared.sd_model_type == 'omnigen':
             steps = p.steps
         elif p.denoising_strength > 0:

modules/sd_detect.py

@@ -104,12 +104,14 @@ def detect_pipeline(f: str, op: str = 'model', warning=True, quiet=False):
                 index = shared.readfile(index, silent=True)
                 cls = index.get('_class_name', None)
                 if cls is not None:
-                    pipeline = getattr(diffusers, cls)
-                if 'Flux' in pipeline.__name__ and guess != 'FLEX':
+                    pipeline = getattr(diffusers, cls, None)
+                    if pipeline is None:
+                        pipeline = cls
+                if callable(pipeline) and 'Flux' in pipeline.__name__ and guess != 'FLEX':
                     guess = 'FLUX'
-                if 'StableDiffusion3' in pipeline.__name__:
+                if callable(pipeline) and 'StableDiffusion3' in pipeline.__name__:
                     guess = 'Stable Diffusion 3'
-                if 'Lumina2' in pipeline.__name__:
+                if callable(pipeline) and 'Lumina2' in pipeline.__name__:
                     guess = 'Lumina 2'
             # switch for specific variant
             if guess == 'Stable Diffusion' and 'inpaint' in f.lower():