diff --git a/html/reference.json b/html/reference.json index 0a44f130d..ff29623ff 100644 --- a/html/reference.json +++ b/html/reference.json @@ -120,6 +120,15 @@ "preview": "stabilityai--stable-diffusion-3.jpg", "extras": "sampler: Default, cfg_scale: 7.0" }, + "StabilityAI Stable Diffusion 3 Large": { + "path": "huggingface/stabilityai/stable-diffusion-3.5-large", + "skip": true, + "variant": "fp16", + "te3": null, + "desc": "Stable Diffusion 3 Medium is a Multimodal Diffusion Transformer (MMDiT) text-to-image model that features greatly improved performance in image quality, typography, complex prompt understanding, and resource-efficiency", + "preview": "stabilityai--stable-diffusion-3.jpg", + "extras": "sampler: Default, cfg_scale: 7.0" + }, "Black Forest Labs FLUX.1 Dev": { "path": "black-forest-labs/FLUX.1-dev", diff --git a/modules/model_sd3.py b/modules/model_sd3.py index 5c0d76767..e842d15fc 100644 --- a/modules/model_sd3.py +++ b/modules/model_sd3.py @@ -4,18 +4,16 @@ import diffusers import transformers -def load_sd3(fn=None, cache_dir=None, config=None): - from modules import devices, modelloader - repo_id = 'stabilityai/stable-diffusion-3-medium-diffusers' - model_id = 'stabilityai/stable-diffusion-3-medium-diffusers' - dtype = torch.float16 +def load_sd3(checkpoint_info, cache_dir=None, config=None): + from modules import devices, modelloader, sd_models + repo_id = sd_models.path_to_repo(checkpoint_info.name) + # dtype = torch.float16 + dtype = devices.dtype kwargs = {} - if fn is not None and fn.endswith('.safetensors') and os.path.exists(fn): - model_id = fn + if checkpoint_info.path is not None and checkpoint_info.path.endswith('.safetensors') and os.path.exists(checkpoint_info.path): loader = diffusers.StableDiffusion3Pipeline.from_single_file - _diffusers_major, diffusers_minor, diffusers_micro = int(diffusers.__version__.split('.')[0]), int(diffusers.__version__.split('.')[1]), int(diffusers.__version__.split('.')[2]) # pylint: disable=use-maxsplit-arg - fn_size = os.path.getsize(fn) - if (diffusers_minor <= 29 and diffusers_micro < 1) or fn_size < 5e9: # te1/te2 do not get loaded correctly in diffusers 0.29.0 if model is without te1/te2 + fn_size = os.path.getsize(checkpoint_info.path) + if fn_size < 5e9: kwargs = { 'text_encoder': transformers.CLIPTextModelWithProjection.from_pretrained( repo_id, @@ -49,16 +47,14 @@ def load_sd3(fn=None, cache_dir=None, config=None): kwargs = {} else: modelloader.hf_login() - model_id = repo_id loader = diffusers.StableDiffusion3Pipeline.from_pretrained pipe = loader( - model_id, + repo_id, torch_dtype=dtype, cache_dir=cache_dir, config=config, **kwargs, ) - diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["stable-diffusion-3"] = diffusers.StableDiffusion3Pipeline - diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["stable-diffusion-3"] = diffusers.StableDiffusion3Img2ImgPipeline + # pipe.transformer = pipe.transformer.to(devices.dtype) # diffusers loader leaves it as-is devices.torch_gc() return pipe diff --git a/modules/processing_args.py b/modules/processing_args.py index 7296fccb8..0e03ec1ba 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -126,6 +126,8 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 timer.process.record('encode', reset=False) if 'prompt' in possible: + if 'OmniGen' in model.__class__.__name__: + p.prompts = [p.replace('|image|', '<|image_1|>') for p in prompts] if hasattr(model, 'text_encoder') and 'prompt_embeds' in possible and len(p.prompt_embeds) > 0 and p.prompt_embeds[0] is not None: args['prompt_embeds'] = p.prompt_embeds[0] if 'StableCascade' in model.__class__.__name__ and len(getattr(p, 'negative_pooleds', [])) > 0: @@ -136,8 +138,6 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 args['pooled_prompt_embeds'] = p.positive_pooleds[0] elif 'Flux' in model.__class__.__name__ and len(getattr(p, 'positive_pooleds', [])) > 0: args['pooled_prompt_embeds'] = p.positive_pooleds[0] - if 'OmniGen' in model.__class__.__name__: - args['prompt'] = [p.replace('|image|', '<|image_1|>') for p in prompts] else: args['prompt'] = prompts if 'negative_prompt' in possible: diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 599ddf815..40a24465d 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -400,6 +400,8 @@ def get_weighted_text_embeddings(pipe, prompt: str = "", neg_prompt: str = "", c pooled_prompt_embeds = [] negative_pooled_prompt_embeds = [] for i in range(len(embedding_providers)): + if i >= len(positives): # te may be missing/unloaded + break t0 = time.time() text = list(positives[i]) weights = list(positive_weights[i]) diff --git a/modules/sd_models.py b/modules/sd_models.py index ab7eca5e6..dd38bdcec 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1089,7 +1089,7 @@ def load_diffuser_force(model_type, checkpoint_info, diffusers_load_config, op=' from modules.model_sd3 import load_sd3 shared.log.debug(f'Load {op}: model="Stable Diffusion 3" variant=medium') shared.opts.scheduler = 'Default' - sd_model = load_sd3(cache_dir=shared.opts.diffusers_dir, config=diffusers_load_config.get('config', None)) + sd_model = load_sd3(checkpoint_info, cache_dir=shared.opts.diffusers_dir, config=diffusers_load_config.get('config', None)) elif model_type in ['Meissonic']: # forced pipeline from modules.model_meissonic import load_meissonic sd_model = load_meissonic(checkpoint_info, diffusers_load_config) @@ -1190,7 +1190,7 @@ def load_diffuser_file(model_type, pipeline, checkpoint_info, diffusers_load_con diffusers_load_config['config'] = model_config if model_type.startswith('Stable Diffusion 3'): from modules.model_sd3 import load_sd3 - sd_model = load_sd3(fn=checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, config=diffusers_load_config.get('config', None)) + sd_model = load_sd3(checkpoint_info=checkpoint_info, cache_dir=shared.opts.diffusers_dir, config=diffusers_load_config.get('config', None)) elif hasattr(pipeline, 'from_single_file'): diffusers.loaders.single_file_utils.CHECKPOINT_KEY_NAMES["clip"] = "cond_stage_model.transformer.text_model.embeddings.position_embedding.weight" # patch for diffusers==0.28.0 diffusers_load_config['use_safetensors'] = True @@ -1977,6 +1977,8 @@ def remove_token_merging(sd_model): def path_to_repo(fn: str = ''): + if isinstance(fn, CheckpointInfo): + fn = fn.name repo_id = fn.replace('\\', '/') if 'models--' in repo_id: repo_id = repo_id.split('models--')[-1] diff --git a/modules/sd_samplers.py b/modules/sd_samplers.py index 1034a6e5d..89b5a8a79 100644 --- a/modules/sd_samplers.py +++ b/modules/sd_samplers.py @@ -77,6 +77,10 @@ def create_sampler(name, model): if 'Lumina' in model.__class__.__name__: shared.log.warning(f'AlphaVLLM-Lumina: sampler="{name}" unsupported') return None + if 'StableDiffusion3Pipeline' in model.__class__.__name__: + if sampler.name != 'Heun FlowMatch': + return None + return None if 'AuraFlow' in model.__class__.__name__: shared.log.warning(f'AuraFlow: sampler="{name}" unsupported') return None