mirror of
https://github.com/vladmandic/automatic
synced 2026-09-06 13:00:44 +02:00
feat: add FLUX.2 Klein model support
Add support for FLUX.2 Klein distilled models (4B and 9B variants): - Add pipeline loader for Flux2KleinPipeline - Add model detection for 'flux.2' + 'klein' patterns - Add pipeline mapping in shared_items - Add shared Qwen3ForCausalLM text encoder handling: - 4B variants use Z-Image-Turbo's Qwen3-8B - 9B variants use FLUX.2-klein-9B's Qwen3-14B - Add reference entries for distilled (4B, 9B) and base models - Update diffusers commit for Flux2KleinPipeline support
This commit is contained in:
@@ -161,5 +161,25 @@
|
||||
"preview": "Tencent-Hunyuan--HunyuanDiT-v1.1-Diffusers-Distilled.jpg",
|
||||
"tags": "distilled",
|
||||
"extras": "sampler: Default, cfg_scale: 2.0"
|
||||
},
|
||||
"Black Forest Labs FLUX.2 Klein 4B": {
|
||||
"path": "black-forest-labs/FLUX.2-klein-4B",
|
||||
"preview": "black-forest-labs--FLUX.2-klein-4B.jpg",
|
||||
"desc": "FLUX.2-klein-4B is a 4 billion parameter size-distilled version of FLUX.2-dev optimized for consumer GPUs. Achieves sub-second inference with 4 steps while fitting in ~13GB VRAM. Supports both text-to-image generation and multi-reference image editing. Apache 2.0 licensed.",
|
||||
"skip": true,
|
||||
"tags": "distilled",
|
||||
"extras": "sampler: Default, cfg_scale: 4.0, steps: 4",
|
||||
"size": 8.5,
|
||||
"date": "2025 January"
|
||||
},
|
||||
"Black Forest Labs FLUX.2 Klein 9B": {
|
||||
"path": "black-forest-labs/FLUX.2-klein-9B",
|
||||
"preview": "black-forest-labs--FLUX.2-klein-9B.jpg",
|
||||
"desc": "FLUX.2-klein-9B is a 9 billion parameter size-distilled version of FLUX.2-dev. Higher quality than 4B variant with sub-second inference using 4 steps. Requires ~29GB VRAM. Supports text-to-image and multi-reference editing. Non-commercial license.",
|
||||
"skip": true,
|
||||
"tags": "distilled",
|
||||
"extras": "sampler: Default, cfg_scale: 4.0, steps: 4",
|
||||
"size": 18.5,
|
||||
"date": "2025 January"
|
||||
}
|
||||
}
|
||||
@@ -124,6 +124,24 @@
|
||||
"size": 104.74,
|
||||
"date": "2025 November"
|
||||
},
|
||||
"Black Forest Labs FLUX.2 Klein Base 4B": {
|
||||
"path": "black-forest-labs/FLUX.2-klein-base-4B",
|
||||
"preview": "black-forest-labs--FLUX.2-klein-base-4B.jpg",
|
||||
"desc": "FLUX.2-klein-base-4B is the undistilled 4 billion parameter base model of FLUX.2-klein. Requires 50 inference steps for full quality but offers flexibility for fine-tuning. Fits in ~13GB VRAM. Supports text-to-image and multi-reference editing. Apache 2.0 licensed.",
|
||||
"skip": true,
|
||||
"extras": "sampler: Default, cfg_scale: 4.0, steps: 50",
|
||||
"size": 8.5,
|
||||
"date": "2025 January"
|
||||
},
|
||||
"Black Forest Labs FLUX.2 Klein Base 9B": {
|
||||
"path": "black-forest-labs/FLUX.2-klein-base-9B",
|
||||
"preview": "black-forest-labs--FLUX.2-klein-base-9B.jpg",
|
||||
"desc": "FLUX.2-klein-base-9B is the undistilled 9 billion parameter base model of FLUX.2-klein. Requires 50 inference steps for full quality but offers flexibility for fine-tuning. Requires ~29GB VRAM. Supports text-to-image and multi-reference editing. Non-commercial license.",
|
||||
"skip": true,
|
||||
"extras": "sampler: Default, cfg_scale: 4.0, steps: 50",
|
||||
"size": 18.5,
|
||||
"date": "2025 January"
|
||||
},
|
||||
|
||||
"Z-Image-Turbo": {
|
||||
"path": "Tongyi-MAI/Z-Image-Turbo",
|
||||
|
||||
+1
-1
@@ -648,7 +648,7 @@ def check_diffusers():
|
||||
t_start = time.time()
|
||||
if args.skip_all:
|
||||
return
|
||||
sha = '5efb81fa711863fdece9136ad10788440e658b40' # diffusers commit hash
|
||||
sha = '61f175660a8ac54f1470a74a810e6c38fb4795d5' # diffusers commit hash
|
||||
# if args.use_rocm or args.use_zluda or args.use_directml:
|
||||
# sha = '043ab2520f6a19fce78e6e060a68dbc947edb9f9' # lock diffusers versions for now
|
||||
pkg = pkg_resources.working_set.by_key.get('diffusers', None)
|
||||
|
||||
@@ -92,6 +92,8 @@ def guess_by_name(fn, current_guess):
|
||||
new_guess = 'HiDream'
|
||||
elif 'chroma' in fn.lower() and 'xl' not in fn.lower():
|
||||
new_guess = 'Chroma'
|
||||
elif 'flux.2' in fn.lower() and 'klein' in fn.lower():
|
||||
new_guess = 'FLUX2 Klein'
|
||||
elif 'flux.2' in fn.lower():
|
||||
new_guess = 'FLUX2'
|
||||
elif 'flux' in fn.lower() or 'flex.1' in fn.lower():
|
||||
|
||||
@@ -359,6 +359,10 @@ def load_diffuser_force(detected_model_type, checkpoint_info, diffusers_load_con
|
||||
from pipelines.model_flux2 import load_flux2
|
||||
sd_model = load_flux2(checkpoint_info, diffusers_load_config)
|
||||
allow_post_quant = False
|
||||
elif model_type in ['FLUX2 Klein']:
|
||||
from pipelines.model_flux2_klein import load_flux2_klein
|
||||
sd_model = load_flux2_klein(checkpoint_info, diffusers_load_config)
|
||||
allow_post_quant = False
|
||||
elif model_type in ['FLEX']:
|
||||
from pipelines.model_flex import load_flex
|
||||
sd_model = load_flex(checkpoint_info, diffusers_load_config)
|
||||
|
||||
@@ -48,6 +48,8 @@ pipelines = {
|
||||
'Qwen': getattr(diffusers, 'QwenImagePipeline', None),
|
||||
'HunyuanImage': getattr(diffusers, 'HunyuanImagePipeline', None),
|
||||
'Z-Image': getattr(diffusers, 'ZImagePipeline', None),
|
||||
'FLUX2': getattr(diffusers, 'Flux2Pipeline', None),
|
||||
'FLUX2 Klein': getattr(diffusers, 'Flux2KleinPipeline', None),
|
||||
'LongCat': getattr(diffusers, 'LongCatImagePipeline', None),
|
||||
'GLM-Image': getattr(diffusers, 'GlmImagePipeline', None),
|
||||
# dynamically imported and redefined later
|
||||
|
||||
@@ -200,6 +200,23 @@ def load_text_encoder(repo_id, cls_name, load_config=None, subfolder="text_encod
|
||||
**load_args,
|
||||
**quant_args,
|
||||
)
|
||||
# Qwen3ForCausalLM - shared text encoders by hidden_size:
|
||||
# - Z-Image, Klein-4B: Qwen3-4B (hidden_size=2560)
|
||||
# - Klein-9B: Qwen3-8B (hidden_size=4096)
|
||||
elif cls_name == transformers.Qwen3ForCausalLM and allow_shared and shared.opts.te_shared_t5:
|
||||
if '-9b' in repo_id.lower():
|
||||
shared_repo = 'black-forest-labs/FLUX.2-klein-9B' # 9B variants use Qwen3-8B
|
||||
else:
|
||||
shared_repo = 'Tongyi-MAI/Z-Image-Turbo' # 4B variants and Z-Image use Qwen3-4B
|
||||
subfolder = 'text_encoder'
|
||||
shared.log.debug(f'Load model: text_encoder="{shared_repo}" cls={cls_name.__name__} quant="{quant_type}" loader={_loader("transformers")} shared={shared.opts.te_shared_t5}')
|
||||
text_encoder = cls_name.from_pretrained(
|
||||
shared_repo,
|
||||
cache_dir=shared.opts.hfcache_dir,
|
||||
subfolder=subfolder,
|
||||
**load_args,
|
||||
**quant_args,
|
||||
)
|
||||
|
||||
# load from repo
|
||||
if text_encoder is None:
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
import transformers
|
||||
import diffusers
|
||||
from modules import shared, devices, sd_models, model_quant, sd_hijack_te, sd_hijack_vae
|
||||
from pipelines import generic
|
||||
|
||||
|
||||
def load_flux2_klein(checkpoint_info, diffusers_load_config=None):
|
||||
if diffusers_load_config is None:
|
||||
diffusers_load_config = {}
|
||||
repo_id = sd_models.path_to_repo(checkpoint_info)
|
||||
sd_models.hf_auth_check(checkpoint_info)
|
||||
|
||||
load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config, allow_quant=False)
|
||||
shared.log.debug(f'Load model: type=Flux2Klein repo="{repo_id}" config={diffusers_load_config} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}')
|
||||
|
||||
# Load transformer - Klein uses Flux2Transformer2DModel (same class as Flux2, different size)
|
||||
transformer = generic.load_transformer(repo_id, cls_name=diffusers.Flux2Transformer2DModel, load_config=diffusers_load_config)
|
||||
|
||||
# Load text encoder - Klein uses Qwen3ForCausalLM (8B), shared across all Klein variants
|
||||
text_encoder = generic.load_text_encoder(repo_id, cls_name=transformers.Qwen3ForCausalLM, load_config=diffusers_load_config)
|
||||
|
||||
pipe = diffusers.Flux2KleinPipeline.from_pretrained(
|
||||
repo_id,
|
||||
transformer=transformer,
|
||||
text_encoder=text_encoder,
|
||||
cache_dir=shared.opts.diffusers_dir,
|
||||
**load_args,
|
||||
)
|
||||
pipe.task_args = {
|
||||
'output_type': 'np',
|
||||
}
|
||||
diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["flux2klein"] = diffusers.Flux2KleinPipeline
|
||||
diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["flux2klein"] = diffusers.Flux2KleinPipeline
|
||||
diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING["flux2klein"] = diffusers.Flux2KleinPipeline
|
||||
|
||||
del text_encoder
|
||||
del transformer
|
||||
sd_hijack_te.init_hijack(pipe)
|
||||
sd_hijack_vae.init_hijack(pipe)
|
||||
|
||||
devices.torch_gc(force=True, reason='load')
|
||||
return pipe
|
||||
Reference in New Issue
Block a user