diff --git a/html/reference.json b/html/reference.json
index 0a44f130d..ff29623ff 100644
--- a/html/reference.json
+++ b/html/reference.json
@@ -120,6 +120,15 @@
"preview": "stabilityai--stable-diffusion-3.jpg",
"extras": "sampler: Default, cfg_scale: 7.0"
},
+ "StabilityAI Stable Diffusion 3 Large": {
+ "path": "huggingface/stabilityai/stable-diffusion-3.5-large",
+ "skip": true,
+ "variant": "fp16",
+ "te3": null,
+ "desc": "Stable Diffusion 3 Medium is a Multimodal Diffusion Transformer (MMDiT) text-to-image model that features greatly improved performance in image quality, typography, complex prompt understanding, and resource-efficiency",
+ "preview": "stabilityai--stable-diffusion-3.jpg",
+ "extras": "sampler: Default, cfg_scale: 7.0"
+ },
"Black Forest Labs FLUX.1 Dev": {
"path": "black-forest-labs/FLUX.1-dev",
diff --git a/modules/model_sd3.py b/modules/model_sd3.py
index 5c0d76767..e842d15fc 100644
--- a/modules/model_sd3.py
+++ b/modules/model_sd3.py
@@ -4,18 +4,16 @@ import diffusers
import transformers
-def load_sd3(fn=None, cache_dir=None, config=None):
- from modules import devices, modelloader
- repo_id = 'stabilityai/stable-diffusion-3-medium-diffusers'
- model_id = 'stabilityai/stable-diffusion-3-medium-diffusers'
- dtype = torch.float16
+def load_sd3(checkpoint_info, cache_dir=None, config=None):
+ from modules import devices, modelloader, sd_models
+ repo_id = sd_models.path_to_repo(checkpoint_info.name)
+ # dtype = torch.float16
+ dtype = devices.dtype
kwargs = {}
- if fn is not None and fn.endswith('.safetensors') and os.path.exists(fn):
- model_id = fn
+ if checkpoint_info.path is not None and checkpoint_info.path.endswith('.safetensors') and os.path.exists(checkpoint_info.path):
loader = diffusers.StableDiffusion3Pipeline.from_single_file
- _diffusers_major, diffusers_minor, diffusers_micro = int(diffusers.__version__.split('.')[0]), int(diffusers.__version__.split('.')[1]), int(diffusers.__version__.split('.')[2]) # pylint: disable=use-maxsplit-arg
- fn_size = os.path.getsize(fn)
- if (diffusers_minor <= 29 and diffusers_micro < 1) or fn_size < 5e9: # te1/te2 do not get loaded correctly in diffusers 0.29.0 if model is without te1/te2
+ fn_size = os.path.getsize(checkpoint_info.path)
+ if fn_size < 5e9:
kwargs = {
'text_encoder': transformers.CLIPTextModelWithProjection.from_pretrained(
repo_id,
@@ -49,16 +47,14 @@ def load_sd3(fn=None, cache_dir=None, config=None):
kwargs = {}
else:
modelloader.hf_login()
- model_id = repo_id
loader = diffusers.StableDiffusion3Pipeline.from_pretrained
pipe = loader(
- model_id,
+ repo_id,
torch_dtype=dtype,
cache_dir=cache_dir,
config=config,
**kwargs,
)
- diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["stable-diffusion-3"] = diffusers.StableDiffusion3Pipeline
- diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["stable-diffusion-3"] = diffusers.StableDiffusion3Img2ImgPipeline
+ # pipe.transformer = pipe.transformer.to(devices.dtype) # diffusers loader leaves it as-is
devices.torch_gc()
return pipe
diff --git a/modules/processing_args.py b/modules/processing_args.py
index 7296fccb8..0e03ec1ba 100644
--- a/modules/processing_args.py
+++ b/modules/processing_args.py
@@ -126,6 +126,8 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2
timer.process.record('encode', reset=False)
if 'prompt' in possible:
+ if 'OmniGen' in model.__class__.__name__:
+ p.prompts = [p.replace('|image|', '<|image_1|>') for p in prompts]
if hasattr(model, 'text_encoder') and 'prompt_embeds' in possible and len(p.prompt_embeds) > 0 and p.prompt_embeds[0] is not None:
args['prompt_embeds'] = p.prompt_embeds[0]
if 'StableCascade' in model.__class__.__name__ and len(getattr(p, 'negative_pooleds', [])) > 0:
@@ -136,8 +138,6 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2
args['pooled_prompt_embeds'] = p.positive_pooleds[0]
elif 'Flux' in model.__class__.__name__ and len(getattr(p, 'positive_pooleds', [])) > 0:
args['pooled_prompt_embeds'] = p.positive_pooleds[0]
- if 'OmniGen' in model.__class__.__name__:
- args['prompt'] = [p.replace('|image|', '<|image_1|>') for p in prompts]
else:
args['prompt'] = prompts
if 'negative_prompt' in possible:
diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py
index 599ddf815..40a24465d 100644
--- a/modules/prompt_parser_diffusers.py
+++ b/modules/prompt_parser_diffusers.py
@@ -400,6 +400,8 @@ def get_weighted_text_embeddings(pipe, prompt: str = "", neg_prompt: str = "", c
pooled_prompt_embeds = []
negative_pooled_prompt_embeds = []
for i in range(len(embedding_providers)):
+ if i >= len(positives): # te may be missing/unloaded
+ break
t0 = time.time()
text = list(positives[i])
weights = list(positive_weights[i])
diff --git a/modules/sd_models.py b/modules/sd_models.py
index ab7eca5e6..dd38bdcec 100644
--- a/modules/sd_models.py
+++ b/modules/sd_models.py
@@ -1089,7 +1089,7 @@ def load_diffuser_force(model_type, checkpoint_info, diffusers_load_config, op='
from modules.model_sd3 import load_sd3
shared.log.debug(f'Load {op}: model="Stable Diffusion 3" variant=medium')
shared.opts.scheduler = 'Default'
- sd_model = load_sd3(cache_dir=shared.opts.diffusers_dir, config=diffusers_load_config.get('config', None))
+ sd_model = load_sd3(checkpoint_info, cache_dir=shared.opts.diffusers_dir, config=diffusers_load_config.get('config', None))
elif model_type in ['Meissonic']: # forced pipeline
from modules.model_meissonic import load_meissonic
sd_model = load_meissonic(checkpoint_info, diffusers_load_config)
@@ -1190,7 +1190,7 @@ def load_diffuser_file(model_type, pipeline, checkpoint_info, diffusers_load_con
diffusers_load_config['config'] = model_config
if model_type.startswith('Stable Diffusion 3'):
from modules.model_sd3 import load_sd3
- sd_model = load_sd3(fn=checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, config=diffusers_load_config.get('config', None))
+ sd_model = load_sd3(checkpoint_info=checkpoint_info, cache_dir=shared.opts.diffusers_dir, config=diffusers_load_config.get('config', None))
elif hasattr(pipeline, 'from_single_file'):
diffusers.loaders.single_file_utils.CHECKPOINT_KEY_NAMES["clip"] = "cond_stage_model.transformer.text_model.embeddings.position_embedding.weight" # patch for diffusers==0.28.0
diffusers_load_config['use_safetensors'] = True
@@ -1977,6 +1977,8 @@ def remove_token_merging(sd_model):
def path_to_repo(fn: str = ''):
+ if isinstance(fn, CheckpointInfo):
+ fn = fn.name
repo_id = fn.replace('\\', '/')
if 'models--' in repo_id:
repo_id = repo_id.split('models--')[-1]
diff --git a/modules/sd_samplers.py b/modules/sd_samplers.py
index 1034a6e5d..89b5a8a79 100644
--- a/modules/sd_samplers.py
+++ b/modules/sd_samplers.py
@@ -77,6 +77,10 @@ def create_sampler(name, model):
if 'Lumina' in model.__class__.__name__:
shared.log.warning(f'AlphaVLLM-Lumina: sampler="{name}" unsupported')
return None
+ if 'StableDiffusion3Pipeline' in model.__class__.__name__:
+ if sampler.name != 'Heun FlowMatch':
+ return None
+ return None
if 'AuraFlow' in model.__class__.__name__:
shared.log.warning(f'AuraFlow: sampler="{name}" unsupported')
return None