diff --git a/CHANGELOG.md b/CHANGELOG.md index b27180ef8..866746320 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,23 +1,29 @@ # Change Log for SD.Next -## Update for 2025-11-23 +## Update for 2025-11-26 ### TBD Merge commit: `f903a36d9` -### Highlights for 2025-11-23 +### Highlights for 2025-11-26 New native [kanvas](https://vladmandic.github.io/sdnext-docs/Kanvas/) module for image manipulation that fully replaces img2img, inpaint and outpaint controls -And a first cloud model with **Google's Nano Banana** *2.5 Flash and 3.0 Pro* plus new **Photoroom PRX** model +New generation of **Flux.2** large image model and a first cloud model with **Google's Nano Banana** *2.5 Flash and 3.0 Pro* plus new **Photoroom PRX** model ![Screenshot](https://github.com/user-attachments/assets/54b25586-b611-4d70-a28f-ee3360944034) [ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) | [Sponsor](https://github.com/sponsors/vladmandic) -### Details for 2025-11-23 +### Details for 2025-11-26 - **Models** + - [Black Forest Labs FLUX.2 Dev](https://bfl.ai/blog/flux-2) and prequantized variation [SDNQ-SVD-Uint4](https://huggingface.co/Disty0/FLUX.2-dev-SDNQ-uint4-svd-r32) + **FLUX.2-Dev** is a brand new model from BFL and uses large 32B DiT together with Mistral 24B as text encoder + model is available for text, image and edit tasks and can optionally use control input as second input image + this is a very large model at ~100GB, so use of prequantized model at ~32GB is strongly advised + using prequant version and default offloading, model runs on GPUs with ~20GB + *note*: model is [gated](https://vladmandic.github.io/sdnext-docs/Gated/) - **Google Gemini Nano Banana** [2.5 Flash](https://blog.google/products/gemini/gemini-nano-banana-examples/) and [3.0 Pro](https://deepmind.google/models/gemini-image/pro/) first cloud-based model directly supported in SD.Next UI *note*: need to set `GOOGLE_API_KEY` environment variable with your key to use this model diff --git a/TODO.md b/TODO.md index 5a4c6519e..67dc72f18 100644 --- a/TODO.md +++ b/TODO.md @@ -6,13 +6,10 @@ ## Kanvas -- server-side mask handling vs ui mask handling -- implement different auto-masking options +- implement different llama remover ## Internal -- UI: New inpaint/outpaint interface - [Kanvas](https://github.com/vladmandic/kanvas) - Deploy: Create executable for SD.Next - Feature: Integrate natural language image search [ImageDB](https://github.com/vladmandic/imagedb) @@ -31,6 +28,7 @@ ## Features +- [Flux.2 TinyVAE](https://huggingface.co/fal/FLUX.2-Tiny-AutoEncoder) - [IPAdapter composition](https://huggingface.co/ostris/ip-composition-adapter) - [IPAdapter negative guidance](https://github.com/huggingface/diffusers/discussions/7167) - [MagCache](https://github.com/lllyasviel/FramePack/pull/673/files) diff --git a/html/reference.json b/html/reference.json index 6995682bf..ca0289d33 100644 --- a/html/reference.json +++ b/html/reference.json @@ -144,6 +144,15 @@ "size": 32.93, "date": "2025 July" }, + "Black Forest Labs FLUX.2 Dev": { + "path": "black-forest-labs/FLUX.2-dev", + "preview": "black-forest-labs--FLUX.2-dev.jpg", + "desc": "FLUX.2 generates high-quality images while maintaining character and style consistency across multiple reference images, following structured prompts, reading and writing complex text, adhering to brand guidelines, and reliably handling lighting, layouts, and logos.", + "skip": true, + "extras": "", + "size": 0, + "date": "2025 November" + }, "Tencent FLUX.1 Dev SRPO": { "path": "vladmandic/flux.1-dev-SRPO", "preview": "vladmandic--flux.1-dev-SRPO.jpg", @@ -959,6 +968,15 @@ "date": "2025 October", "extras": "" }, + "FLUX.2 Dev sdnq-svd-uint4": { + "path": "Disty0/FLUX.2-dev-SDNQ-uint4-svd-r32", + "preview": "black-forest-labs--FLUX.2-dev.jpg", + "desc": "Quantization of black-forest-labs/FLUX.2-dev using SDNQ: sdnq-svd 4-bit uint with svd rank 32", + "skip": true, + "extras": "", + "size": 0, + "date": "2025 November" + }, "Chroma1-HD sdnq-svd-uint4": { "path": "Disty0/Chroma1-HD-SDNQ-uint4-svd-r32", "preview": "Disty0--Chroma1-HD-SDNQ-uint4-svd-r32.jpg", diff --git a/installer.py b/installer.py index d75ab3f77..d725d118d 100644 --- a/installer.py +++ b/installer.py @@ -619,7 +619,7 @@ def check_diffusers(): t_start = time.time() if args.skip_all: return - sha = 'cd3bbe2910666880307b84729176203f5785ff7e' # diffusers commit hash + sha = 'c8656ed73c638e51fc2e777a5fd355d69fa5220f' # diffusers commit hash # if args.use_rocm or args.use_zluda or args.use_directml: # sha = '043ab2520f6a19fce78e6e060a68dbc947edb9f9' # lock diffusers versions for now pkg = pkg_resources.working_set.by_key.get('diffusers', None) diff --git a/models/Reference/black-forest-labs--FLUX.2-dev.jpg b/models/Reference/black-forest-labs--FLUX.2-dev.jpg new file mode 100644 index 000000000..9a65b92b3 Binary files /dev/null and b/models/Reference/black-forest-labs--FLUX.2-dev.jpg differ diff --git a/modules/modeldata.py b/modules/modeldata.py index 6b0fb89da..a7c14c02c 100644 --- a/modules/modeldata.py +++ b/modules/modeldata.py @@ -32,6 +32,8 @@ def get_model_type(pipe): model_type = 'auraflow' elif 'Chroma' in name: model_type = 'chroma' + elif "Flux2" in name: + model_type = 'f2' elif "Flux" in name or "Flex1" in name or "Flex2" in name: model_type = 'f1' elif "Lumina2" in name: diff --git a/modules/processing_args.py b/modules/processing_args.py index 74bfce1fd..372ac493e 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -127,6 +127,8 @@ def task_specific_kwargs(p, model): task_args['image'] = [Image.new('RGB', (p.width, p.height), (0, 0, 0))] # monkey-patch so qwen-image-edit pipeline does not error-out on t2i if ('QwenImageEditPlusPipeline' in model_cls) and (p.init_control is not None) and (len(p.init_control) > 0): task_args['image'] += p.init_control + if ('Flux2' in model_cls) and (p.init_control is not None) and (len(p.init_control) > 0): + task_args['image'] += p.init_control if ('LatentConsistencyModelPipeline' in model_cls) and (len(p.init_images) > 0): p.ops.append('lcm') init_latents = [processing_vae.vae_encode(image, model=shared.sd_model, vae_type=p.vae_type).squeeze(dim=0) for image in p.init_images] @@ -204,7 +206,7 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t if (prompt_attention != 'fixed') and ('Onnx' not in model.__class__.__name__) and ('prompt' not in p.task_args) and ( 'StableDiffusion' in model.__class__.__name__ or 'StableCascade' in model.__class__.__name__ or - 'Flux' in model.__class__.__name__ or + ('Flux' in model.__class__.__name__ and 'Flux2' not in model.__class__.__name__) or 'Chroma' in model.__class__.__name__ or 'HiDreamImagePipeline' in model.__class__.__name__ ): diff --git a/modules/processing_helpers.py b/modules/processing_helpers.py index b58f0594a..c8692d6ce 100644 --- a/modules/processing_helpers.py +++ b/modules/processing_helpers.py @@ -408,7 +408,7 @@ def calculate_base_steps(p, use_denoise_start, use_refiner_start): if len(getattr(p, 'timesteps', [])) > 0: return None cls = shared.sd_model.__class__.__name__ - if 'Flex' in cls or 'Kontext' in cls or 'Edit' in cls or 'Wan' in cls: + if 'Flex' in cls or 'Kontext' in cls or 'Edit' in cls or 'Wan' in cls or 'Flux2': steps = p.steps elif is_modular(): steps = p.steps diff --git a/modules/sd_detect.py b/modules/sd_detect.py index 97701cf8b..220cbe7c9 100644 --- a/modules/sd_detect.py +++ b/modules/sd_detect.py @@ -92,6 +92,8 @@ def guess_by_name(fn, current_guess): new_guess = 'HiDream' elif 'chroma' in fn.lower() and 'xl' not in fn.lower(): new_guess = 'Chroma' + elif 'flux.2' in fn.lower(): + new_guess = 'FLUX2' elif 'flux' in fn.lower() or 'flex.1' in fn.lower(): size = round(os.path.getsize(fn) / 1024 / 1024) if os.path.isfile(fn) else 0 if size > 11000 and size < 16000: diff --git a/modules/sd_models.py b/modules/sd_models.py index 0ee830b7c..d37f7b154 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -322,6 +322,10 @@ def load_diffuser_force(model_type, checkpoint_info, diffusers_load_config, op=' from pipelines.model_flux import load_flux sd_model = load_flux(checkpoint_info, diffusers_load_config) allow_post_quant = False + elif model_type in ['FLUX2']: + from pipelines.model_flux2 import load_flux2 + sd_model = load_flux2(checkpoint_info, diffusers_load_config) + allow_post_quant = False elif model_type in ['FLEX']: from pipelines.model_flex import load_flex sd_model = load_flex(checkpoint_info, diffusers_load_config) diff --git a/modules/sd_offload.py b/modules/sd_offload.py index fcc4317dc..e521cf8c2 100644 --- a/modules/sd_offload.py +++ b/modules/sd_offload.py @@ -14,7 +14,7 @@ from modules.timer import process as process_timer debug = os.environ.get('SD_MOVE_DEBUG', None) is not None verbose = os.environ.get('SD_MOVE_VERBOSE', None) is not None debug_move = log.trace if debug else lambda *args, **kwargs: None -offload_warn = ['sc', 'sd3', 'f1', 'h1', 'hunyuandit', 'auraflow', 'omnigen', 'omnigen2', 'cogview4', 'cosmos', 'chroma', 'x-omni', 'hunyuanimage', 'hunyuanimage3'] +offload_warn = ['sc', 'sd3', 'f1', 'f2', 'h1', 'hunyuandit', 'auraflow', 'omnigen', 'omnigen2', 'cogview4', 'cosmos', 'chroma', 'x-omni', 'hunyuanimage', 'hunyuanimage3'] offload_post = ['h1'] offload_hook_instance = None balanced_offload_exclude = ['CogView4Pipeline', 'MeissonicPipeline'] diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index a45f238d3..0cea700a6 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -9,7 +9,7 @@ from modules import shared, devices, processing, images, sd_vae_approx, sd_vae_t SamplerData = namedtuple('SamplerData', ['name', 'constructor', 'aliases', 'options']) approximation_indexes = { "Simple": 0, "Approximate": 1, "TAESD": 2, "Full VAE": 3 } -flow_models = ['f1', 'sd3', 'lumina', 'auraflow', 'sana', 'lumina2', 'cogview4', 'h1', 'cosmos', 'chroma', 'omnigen', 'omnigen2'] +flow_models = ['f1', 'f2', 'sd3', 'lumina', 'auraflow', 'sana', 'lumina2', 'cogview4', 'h1', 'cosmos', 'chroma', 'omnigen', 'omnigen2'] warned = False queue_lock = threading.Lock() diff --git a/modules/ui_common.py b/modules/ui_common.py index 46b595db2..71a062bfa 100644 --- a/modules/ui_common.py +++ b/modules/ui_common.py @@ -419,8 +419,8 @@ def update_token_counter(text): from modules import extra_networks prompt, _ = extra_networks.parse_prompt(text) if shared.sd_loaded and hasattr(shared.sd_model, 'tokenizer') and shared.sd_model.tokenizer is not None: - has_bos_token = shared.sd_model.tokenizer.bos_token_id is not None - has_eos_token = shared.sd_model.tokenizer.eos_token_id is not None + has_bos_token = hasattr(shared.sd_model.tokenizer, 'bos_token_id') and shared.sd_model.tokenizer.bos_token_id is not None + has_eos_token = hasattr(shared.sd_model.tokenizer, 'eos_token_id') and shared.sd_model.tokenizer.eos_token_id is not None ids = shared.sd_model.tokenizer(prompt) ids = getattr(ids, 'input_ids', []) token_count = len(ids) - int(has_bos_token) - int(has_eos_token) diff --git a/pipelines/model_flux2.py b/pipelines/model_flux2.py new file mode 100644 index 000000000..388ecd6f6 --- /dev/null +++ b/pipelines/model_flux2.py @@ -0,0 +1,39 @@ +import transformers +import diffusers +from modules import shared, devices, sd_models, model_quant, sd_hijack_te, sd_hijack_vae +from pipelines import generic + + +def load_flux2(checkpoint_info, diffusers_load_config=None): + if diffusers_load_config is None: + diffusers_load_config = {} + repo_id = sd_models.path_to_repo(checkpoint_info) + sd_models.hf_auth_check(checkpoint_info) + + load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config, allow_quant=False) + shared.log.debug(f'Load model: type=Flux2 repo="{repo_id}" config={diffusers_load_config} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}') + + transformer = generic.load_transformer(repo_id, cls_name=diffusers.Flux2Transformer2DModel, load_config=diffusers_load_config) + text_encoder = generic.load_text_encoder(repo_id, cls_name=transformers.Mistral3ForConditionalGeneration, load_config=diffusers_load_config) + + pipe = diffusers.Flux2Pipeline.from_pretrained( + repo_id, + transformer=transformer, + text_encoder=text_encoder, + cache_dir=shared.opts.diffusers_dir, + **load_args, + ) + pipe.task_args = { + 'output_type': 'np', + } + diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["flux2"] = diffusers.Flux2Pipeline + diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["flux2"] = diffusers.Flux2Pipeline + diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING["flux2"] = diffusers.Flux2Pipeline + + del text_encoder + del transformer + sd_hijack_te.init_hijack(pipe) + sd_hijack_vae.init_hijack(pipe) + + devices.torch_gc() + return pipe