diff --git a/CHANGELOG.md b/CHANGELOG.md index 639ccab3c..71e3e4f37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,22 +31,19 @@ To use and of the new models, simply select model from *Networks -> Reference* a FLUX.1 models are based on a hybrid architecture of multimodal and parallel diffusion transformer blocks, scaled to 12B parameters and builing on flow matching This is a very large model at ~32GB in size, its recommended to use a) offloading, b) quantization *Note*: [FLUX.1 Dev](https://huggingface.co/black-forest-labs/FLUX.1-dev) variant is a gated model, you need to accept the terms and conditions to use it - Use scheduler: default or euler flowmatch - Use of FLUX.1 LoRAs is supported - Use of TAESD for preview is supported For more information, see [Wiki](https://github.com/vladmandic/automatic/wiki/FLUX) SD.Next supports: - [FLUX.1 Dev](https://huggingface.co/black-forest-labs/FLUX.1-dev) and [FLUX.1 Schnell](https://huggingface.co/black-forest-labs/FLUX.1-schnell) original variations - additional [qint8](https://huggingface.co/Disty0/FLUX.1-dev-qint8) and [qint4](https://huggingface.co/Disty0/FLUX.1-dev-qint4) quantized variations - additional [nf4](https://huggingface.co/sayakpaul/flux.1-dev-nf4) quantized variation - [AuraFlow](https://huggingface.co/fal/AuraFlow) - AuraFlow is inspired by SD3 and is by far the largest text-to-image generation model that comes with an Apache 2.0 license + AuraFlow v0.1 is the fully open-sourced largest flow-based text-to-image generation model This is a very large model at 6.8B params and nearly 31GB in size, smaller variants are expected in the future - Use scheduler: default or euler flowmatch or heun flowmatch + Use scheduler: Default or Euler FlowMatch or Heun FlowMatch - [AlphaVLLM Lumina-Next-SFT](https://huggingface.co/Alpha-VLLM/Lumina-Next-SFT-diffusers) Lumina-Next-SFT is a Next-DiT model containing 2B parameters, enhanced through high-quality supervised fine-tuning (SFT) This model uses T5 XXL variation of text encoder (previous version of Lumina used Gemma 2B as text encoder) - Use scheduler: default or euler flowmatch or heun flowmatch + Use scheduler: Default or Euler FlowMatch or Heun FlowMatch - [Kwai Kolors](https://huggingface.co/Kwai-Kolors/Kolors) Kolors is a large-scale text-to-image generation model based on latent diffusion This is an SDXL style model that replaces standard CLiP-L and CLiP-G text encoders with a massive `chatglm3-6b` encoder supporting both English and Chinese prompting diff --git a/TODO.md b/TODO.md index b88ab3310..2729def18 100644 --- a/TODO.md +++ b/TODO.md @@ -5,7 +5,8 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma ## Future Candidates - cogvideo-x: -- animatediff-sdxl +- animatediff-sdxl: +- animatediff prompt-travel: - async lowvram: - fp8: - init latents: variations, img2img diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index 5e728032c..c84d677e0 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit 5e728032c054ce4344d96e327b9e389420711e21 +Subproject commit c84d677e0c2df4aabe556dc3b40d5fed024e4cc1 diff --git a/javascript/sdnext.css b/javascript/sdnext.css index f2d553106..6f6cfc200 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -277,7 +277,7 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt /* custom component */ .folder-selector textarea { height: 2em !important; padding: 6px !important; } .nvml { position: fixed; bottom: 10px; right: 10px; background: var(--background-fill-primary); border: 1px solid var(--button-primary-border-color); padding: 6px; color: var(--button-primary-text-color); - font-size: var(--text-xxs); z-index: 50; font-family: monospace; display: none; } + font-size: var(--text-xxs); z-index: 1000; font-family: monospace; display: none; } /* control */ #control_input_type { max-width: 18em } diff --git a/modules/control/units/controlnet.py b/modules/control/units/controlnet.py index 40f543d1b..ef53fb763 100644 --- a/modules/control/units/controlnet.py +++ b/modules/control/units/controlnet.py @@ -66,6 +66,9 @@ predefined_sdxl = { # 'StabilityAI Recolor R256': 'stabilityai/control-lora/control-LoRAs-rank256/control-lora-recolor-rank256.safetensors', # 'StabilityAI Sketch R256': 'stabilityai/control-lora/control-LoRAs-rank256/control-lora-sketch-rank256.safetensors', } +predefined_f1 = { + 'Shakker-Labs ControlNet Union': 'Shakker-Labs/FLUX.1-dev-ControlNet-Union-Pro', +} models = {} all_models = {} all_models.update(predefined_sd15) @@ -102,6 +105,8 @@ def list_models(refresh=False): models = ['None'] + list(predefined_sdxl) + sorted(find_models()) elif modules.shared.sd_model_type == 'sd': models = ['None'] + list(predefined_sd15) + sorted(find_models()) + elif modules.shared.sd_model_type == 'f1': + models = ['None'] + list(predefined_f1) + sorted(find_models()) else: log.warning(f'Control {what} model list failed: unknown model type') models = ['None'] + sorted(predefined_sd15) + sorted(predefined_sdxl) + sorted(find_models()) @@ -244,6 +249,8 @@ class ControlNetPipeline(): controlnet=controlnet, # can be a list ) sd_models.move_model(self.pipeline, pipeline.device) + elif detect.is_f1(pipeline): + log.warning('Control model pipeline: class=FluxPipeline unsupported model type') else: log.error(f'Control {what} pipeline: class={pipeline.__class__.__name__} unsupported model type') return diff --git a/modules/control/units/detect.py b/modules/control/units/detect.py index 79aad6cad..70a1c8000 100644 --- a/modules/control/units/detect.py +++ b/modules/control/units/detect.py @@ -15,3 +15,10 @@ def is_sdxl(model): if hasattr(model, '__name__'): return model.__name__ == p.StableDiffusionXLPipeline.__name__ or model.__name__ == p.StableDiffusionXLImg2ImgPipeline.__name__ or model.__name__ == p.StableDiffusionXLInpaintPipeline.__name__ return isinstance(model, p.StableDiffusionXLPipeline) or isinstance(model, p.StableDiffusionXLImg2ImgPipeline) or isinstance(model, p.StableDiffusionXLInpaintPipeline) + +def is_f1(model): + if model is None: + return False + if hasattr(model, '__name__'): + return model.__name__ == p.FluxPipeline.__name__ + return isinstance(model, p.FluxPipeline) diff --git a/wiki b/wiki index e3a7357b7..426ad4924 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit e3a7357b7fd588b4587600bec03149429b8e90eb +Subproject commit 426ad49241e46379021fe7e9ef1c47b32d66e471