diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dc018e31..c71b6000e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ - New fine-tuned [CLiP-ViT-L]((https://huggingface.co/zer0int/CLIP-GmP-ViT-L-14)) 1st stage **text-encoders** used by SD15, SDXL, Flux.1, etc. brings additional details to your images - New models: + [Stable Diffusion 3.5 Large](https://huggingface.co/stabilityai/stable-diffusion-3.5-large) [OmniGen](https://arxiv.org/pdf/2409.11340) [CogView 3 Plus](https://huggingface.co/THUDM/CogView3-Plus-3B) [Meissonic](https://github.com/viiika/Meissonic) @@ -178,6 +179,12 @@ And there are also other goodies like multiple *XYZ grid* improvements, addition - Params used: prompt, steps, guidance scale for prompt guidance, refine guidance scale for image guidance Recommended: guidance=3.0, refine-guidance=1.6 +- [Stable Diffusion 3.5 Large](https://huggingface.co/stabilityai/stable-diffusion-3.5-large) + - New/improved variant of Stable Diffusion 3 + - Select from *networks -> models -> reference* + - Available in standard and turbo variations + - *Note*: Access to to both variations of SD3.5 model is gated, you must accept the conditions and use HF login + - [CogView 3 Plus](https://huggingface.co/THUDM/CogView3-Plus-3B) - Select from *networks -> models -> reference* - resolution width and height can be from 512px to 2048px and must be divisible by 32 diff --git a/html/reference.json b/html/reference.json index ff29623ff..8d26433e7 100644 --- a/html/reference.json +++ b/html/reference.json @@ -112,21 +112,27 @@ "extras": "sampler: Default, cfg_scale: 4.0, image_cfg_scale: 1.0" }, "StabilityAI Stable Diffusion 3 Medium": { - "path": "huggingface/stabilityai/stable-diffusion-3-medium-diffusers", + "path": "stabilityai/stable-diffusion-3-medium-diffusers", "skip": true, "variant": "fp16", - "te3": null, "desc": "Stable Diffusion 3 Medium is a Multimodal Diffusion Transformer (MMDiT) text-to-image model that features greatly improved performance in image quality, typography, complex prompt understanding, and resource-efficiency", "preview": "stabilityai--stable-diffusion-3.jpg", "extras": "sampler: Default, cfg_scale: 7.0" }, - "StabilityAI Stable Diffusion 3 Large": { - "path": "huggingface/stabilityai/stable-diffusion-3.5-large", + "StabilityAI Stable Diffusion 3.5 Large": { + "path": "stabilityai/stable-diffusion-3.5-large", "skip": true, "variant": "fp16", - "te3": null, "desc": "Stable Diffusion 3 Medium is a Multimodal Diffusion Transformer (MMDiT) text-to-image model that features greatly improved performance in image quality, typography, complex prompt understanding, and resource-efficiency", - "preview": "stabilityai--stable-diffusion-3.jpg", + "preview": "stabilityai--stable-diffusion-3_5.jpg", + "extras": "sampler: Default, cfg_scale: 7.0" + }, + "StabilityAI Stable Diffusion 3.5 Turbo": { + "path": "stabilityai/stable-diffusion-3.5-large-turbo", + "skip": true, + "variant": "fp16", + "desc": "Stable Diffusion 3 Medium is a Multimodal Diffusion Transformer (MMDiT) text-to-image model that features greatly improved performance in image quality, typography, complex prompt understanding, and resource-efficiency", + "preview": "stabilityai--stable-diffusion-3_5.jpg", "extras": "sampler: Default, cfg_scale: 7.0" }, diff --git a/models/Reference/stabilityai--stable-diffusion-3_5.jpg b/models/Reference/stabilityai--stable-diffusion-3_5.jpg new file mode 100644 index 000000000..cfc6be452 Binary files /dev/null and b/models/Reference/stabilityai--stable-diffusion-3_5.jpg differ diff --git a/modules/model_sd3.py b/modules/model_sd3.py index e842d15fc..abc013853 100644 --- a/modules/model_sd3.py +++ b/modules/model_sd3.py @@ -1,5 +1,4 @@ import os -import torch import diffusers import transformers @@ -7,7 +6,6 @@ import transformers def load_sd3(checkpoint_info, cache_dir=None, config=None): from modules import devices, modelloader, sd_models repo_id = sd_models.path_to_repo(checkpoint_info.name) - # dtype = torch.float16 dtype = devices.dtype kwargs = {} if checkpoint_info.path is not None and checkpoint_info.path.endswith('.safetensors') and os.path.exists(checkpoint_info.path): @@ -48,6 +46,7 @@ def load_sd3(checkpoint_info, cache_dir=None, config=None): else: modelloader.hf_login() loader = diffusers.StableDiffusion3Pipeline.from_pretrained + kwargs['variant'] = 'fp16' pipe = loader( repo_id, torch_dtype=dtype, @@ -55,6 +54,5 @@ def load_sd3(checkpoint_info, cache_dir=None, config=None): config=config, **kwargs, ) - # pipe.transformer = pipe.transformer.to(devices.dtype) # diffusers loader leaves it as-is devices.torch_gc() return pipe diff --git a/modules/modelloader.py b/modules/modelloader.py index 50eb0974b..f9b7a4497 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -196,6 +196,8 @@ def download_diffusers_model(hub_id: str, cache_dir: str = None, download_config return None from diffusers import DiffusionPipeline shared.state.begin('HuggingFace') + if hub_id.startswith('huggingface/'): + hub_id = hub_id.replace('huggingface/', '') if download_config is None: download_config = { "force_download": False, @@ -335,7 +337,7 @@ def get_reference_opts(name: str, quiet=False): # shared.log.error(f'Reference: model="{name}" not found') return {} if not quiet: - shared.log.debug(f'Reference: model="{name}" {model_opts.get("extras", None)}') + shared.log.debug(f'Reference: model="{name}" {model_opts}') return model_opts diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 40a24465d..a7c5c296c 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -127,7 +127,11 @@ def get_tokens(msg, prompt): if shared.sd_loaded and hasattr(shared.sd_model, 'tokenizer') and shared.sd_model.tokenizer is not None: if token_dict is None or token_type != shared.sd_model_type: token_type = shared.sd_model_type - fn = os.path.join(shared.sd_model.tokenizer.name_or_path, 'tokenizer', 'vocab.json') + fn = shared.sd_model.tokenizer.name_or_path + if fn.endswith('tokenizer'): + fn = os.path.join(shared.sd_model.tokenizer.name_or_path, 'vocab.json') + else: + fn = os.path.join(shared.sd_model.tokenizer.name_or_path, 'tokenizer', 'vocab.json') token_dict = shared.readfile(fn, silent=True) for k, v in shared.sd_model.tokenizer.added_tokens_decoder.items(): token_dict[str(v)] = k