feat: add FLUX.2 Klein model support

Add support for FLUX.2 Klein distilled models (4B and 9B variants):

- Add pipeline loader for Flux2KleinPipeline
- Add model detection for 'flux.2' + 'klein' patterns
- Add pipeline mapping in shared_items
- Add shared Qwen3ForCausalLM text encoder handling:
  - 4B variants use Z-Image-Turbo's Qwen3-8B
  - 9B variants use FLUX.2-klein-9B's Qwen3-14B
- Add reference entries for distilled (4B, 9B) and base models
- Update diffusers commit for Flux2KleinPipeline support
This commit is contained in:
CalamitousFelicitousness
2026-01-16 01:16:06 +00:00
parent 6f86cc633f
commit fe99d3fe5d
8 changed files with 106 additions and 1 deletions
+20
View File
@@ -161,5 +161,25 @@
"preview": "Tencent-Hunyuan--HunyuanDiT-v1.1-Diffusers-Distilled.jpg",
"tags": "distilled",
"extras": "sampler: Default, cfg_scale: 2.0"
},
"Black Forest Labs FLUX.2 Klein 4B": {
"path": "black-forest-labs/FLUX.2-klein-4B",
"preview": "black-forest-labs--FLUX.2-klein-4B.jpg",
"desc": "FLUX.2-klein-4B is a 4 billion parameter size-distilled version of FLUX.2-dev optimized for consumer GPUs. Achieves sub-second inference with 4 steps while fitting in ~13GB VRAM. Supports both text-to-image generation and multi-reference image editing. Apache 2.0 licensed.",
"skip": true,
"tags": "distilled",
"extras": "sampler: Default, cfg_scale: 4.0, steps: 4",
"size": 8.5,
"date": "2025 January"
},
"Black Forest Labs FLUX.2 Klein 9B": {
"path": "black-forest-labs/FLUX.2-klein-9B",
"preview": "black-forest-labs--FLUX.2-klein-9B.jpg",
"desc": "FLUX.2-klein-9B is a 9 billion parameter size-distilled version of FLUX.2-dev. Higher quality than 4B variant with sub-second inference using 4 steps. Requires ~29GB VRAM. Supports text-to-image and multi-reference editing. Non-commercial license.",
"skip": true,
"tags": "distilled",
"extras": "sampler: Default, cfg_scale: 4.0, steps: 4",
"size": 18.5,
"date": "2025 January"
}
}
+18
View File
@@ -124,6 +124,24 @@
"size": 104.74,
"date": "2025 November"
},
"Black Forest Labs FLUX.2 Klein Base 4B": {
"path": "black-forest-labs/FLUX.2-klein-base-4B",
"preview": "black-forest-labs--FLUX.2-klein-base-4B.jpg",
"desc": "FLUX.2-klein-base-4B is the undistilled 4 billion parameter base model of FLUX.2-klein. Requires 50 inference steps for full quality but offers flexibility for fine-tuning. Fits in ~13GB VRAM. Supports text-to-image and multi-reference editing. Apache 2.0 licensed.",
"skip": true,
"extras": "sampler: Default, cfg_scale: 4.0, steps: 50",
"size": 8.5,
"date": "2025 January"
},
"Black Forest Labs FLUX.2 Klein Base 9B": {
"path": "black-forest-labs/FLUX.2-klein-base-9B",
"preview": "black-forest-labs--FLUX.2-klein-base-9B.jpg",
"desc": "FLUX.2-klein-base-9B is the undistilled 9 billion parameter base model of FLUX.2-klein. Requires 50 inference steps for full quality but offers flexibility for fine-tuning. Requires ~29GB VRAM. Supports text-to-image and multi-reference editing. Non-commercial license.",
"skip": true,
"extras": "sampler: Default, cfg_scale: 4.0, steps: 50",
"size": 18.5,
"date": "2025 January"
},
"Z-Image-Turbo": {
"path": "Tongyi-MAI/Z-Image-Turbo",
+1 -1
View File
@@ -648,7 +648,7 @@ def check_diffusers():
t_start = time.time()
if args.skip_all:
return
sha = '5efb81fa711863fdece9136ad10788440e658b40' # diffusers commit hash
sha = '61f175660a8ac54f1470a74a810e6c38fb4795d5' # diffusers commit hash
# if args.use_rocm or args.use_zluda or args.use_directml:
# sha = '043ab2520f6a19fce78e6e060a68dbc947edb9f9' # lock diffusers versions for now
pkg = pkg_resources.working_set.by_key.get('diffusers', None)
+2
View File
@@ -92,6 +92,8 @@ def guess_by_name(fn, current_guess):
new_guess = 'HiDream'
elif 'chroma' in fn.lower() and 'xl' not in fn.lower():
new_guess = 'Chroma'
elif 'flux.2' in fn.lower() and 'klein' in fn.lower():
new_guess = 'FLUX2 Klein'
elif 'flux.2' in fn.lower():
new_guess = 'FLUX2'
elif 'flux' in fn.lower() or 'flex.1' in fn.lower():
+4
View File
@@ -359,6 +359,10 @@ def load_diffuser_force(detected_model_type, checkpoint_info, diffusers_load_con
from pipelines.model_flux2 import load_flux2
sd_model = load_flux2(checkpoint_info, diffusers_load_config)
allow_post_quant = False
elif model_type in ['FLUX2 Klein']:
from pipelines.model_flux2_klein import load_flux2_klein
sd_model = load_flux2_klein(checkpoint_info, diffusers_load_config)
allow_post_quant = False
elif model_type in ['FLEX']:
from pipelines.model_flex import load_flex
sd_model = load_flex(checkpoint_info, diffusers_load_config)
+2
View File
@@ -48,6 +48,8 @@ pipelines = {
'Qwen': getattr(diffusers, 'QwenImagePipeline', None),
'HunyuanImage': getattr(diffusers, 'HunyuanImagePipeline', None),
'Z-Image': getattr(diffusers, 'ZImagePipeline', None),
'FLUX2': getattr(diffusers, 'Flux2Pipeline', None),
'FLUX2 Klein': getattr(diffusers, 'Flux2KleinPipeline', None),
'LongCat': getattr(diffusers, 'LongCatImagePipeline', None),
'GLM-Image': getattr(diffusers, 'GlmImagePipeline', None),
# dynamically imported and redefined later
+17
View File
@@ -200,6 +200,23 @@ def load_text_encoder(repo_id, cls_name, load_config=None, subfolder="text_encod
**load_args,
**quant_args,
)
# Qwen3ForCausalLM - shared text encoders by hidden_size:
# - Z-Image, Klein-4B: Qwen3-4B (hidden_size=2560)
# - Klein-9B: Qwen3-8B (hidden_size=4096)
elif cls_name == transformers.Qwen3ForCausalLM and allow_shared and shared.opts.te_shared_t5:
if '-9b' in repo_id.lower():
shared_repo = 'black-forest-labs/FLUX.2-klein-9B' # 9B variants use Qwen3-8B
else:
shared_repo = 'Tongyi-MAI/Z-Image-Turbo' # 4B variants and Z-Image use Qwen3-4B
subfolder = 'text_encoder'
shared.log.debug(f'Load model: text_encoder="{shared_repo}" cls={cls_name.__name__} quant="{quant_type}" loader={_loader("transformers")} shared={shared.opts.te_shared_t5}')
text_encoder = cls_name.from_pretrained(
shared_repo,
cache_dir=shared.opts.hfcache_dir,
subfolder=subfolder,
**load_args,
**quant_args,
)
# load from repo
if text_encoder is None:
+42
View File
@@ -0,0 +1,42 @@
import transformers
import diffusers
from modules import shared, devices, sd_models, model_quant, sd_hijack_te, sd_hijack_vae
from pipelines import generic
def load_flux2_klein(checkpoint_info, diffusers_load_config=None):
if diffusers_load_config is None:
diffusers_load_config = {}
repo_id = sd_models.path_to_repo(checkpoint_info)
sd_models.hf_auth_check(checkpoint_info)
load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config, allow_quant=False)
shared.log.debug(f'Load model: type=Flux2Klein repo="{repo_id}" config={diffusers_load_config} offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}')
# Load transformer - Klein uses Flux2Transformer2DModel (same class as Flux2, different size)
transformer = generic.load_transformer(repo_id, cls_name=diffusers.Flux2Transformer2DModel, load_config=diffusers_load_config)
# Load text encoder - Klein uses Qwen3ForCausalLM (8B), shared across all Klein variants
text_encoder = generic.load_text_encoder(repo_id, cls_name=transformers.Qwen3ForCausalLM, load_config=diffusers_load_config)
pipe = diffusers.Flux2KleinPipeline.from_pretrained(
repo_id,
transformer=transformer,
text_encoder=text_encoder,
cache_dir=shared.opts.diffusers_dir,
**load_args,
)
pipe.task_args = {
'output_type': 'np',
}
diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING["flux2klein"] = diffusers.Flux2KleinPipeline
diffusers.pipelines.auto_pipeline.AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["flux2klein"] = diffusers.Flux2KleinPipeline
diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING["flux2klein"] = diffusers.Flux2KleinPipeline
del text_encoder
del transformer
sd_hijack_te.init_hijack(pipe)
sd_hijack_vae.init_hijack(pipe)
devices.torch_gc(force=True, reason='load')
return pipe