diff --git a/CHANGELOG.md b/CHANGELOG.md index 32d906a7d..f0b8b9c52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,30 @@ # Change Log for SD.Next -## Update for 2025-03-14 +## Update for 2025-03-15 -- fix installer not starting when older version of rich is installed -- fix circular imports when debug flags are enabled -- fix cuda errors with directml -- fix memory stats not displaying the ram usage -- fix runpod memory limit reporting -- fix remote vae not being stored in metadata, thanks @iDeNoh -- add --upgrade to torch_command when using --use-nightly for ipex and rocm -- **ipex** +- **Models** + - [THUDM CogView 4 6B](https://huggingface.co/THUDM/CogView4-6B) + new foundation model for image generation based o T5-XXL text encoder and a flow-based diffusion transformer + fully supports offloading and on-the-fly quantization + simply select from *networks -> models -> reference* + - New [zer0int CLiP-L](https://huggingface.co/zer0int/CLIP-Registers-Gated_MLP-ViT-L-14) models: + download text encoders into folder set in settings -> system paths -> text encoders (default is `models/Text-encoder`) + load using *settings -> text encoder* + *tip*: add *sd_text_encoder* to your *settings -> user interface -> quicksettings* list to have it appear at the top of the ui +- **Wiki/Docs** + - updated [Models](https://github.com/vladmandic/sdnext/wiki/Models) info + - Updated SD3 +- **Other** + - add remote vae info to metadata, thanks @iDeNoh + - add quantization support to **CogView-3Plus** +- **Fixes** + - fix installer not starting when older version of `rich` is installed + - fix circular imports when debug flags are enabled + - fix cuda errors with *directml* + - fix memory stats not displaying the ram usage + - fix **RunPod** memory limit reporting +- **IPEX** + - add `--upgrade` to torch_command when using `--use-nightly` for *ipex* and *rocm* - add xpu to profiler - fix untyped_storage, torch.eye and torch.cuda.device ops - fix torch 2.7 compatibility diff --git a/html/reference.json b/html/reference.json index 4cc2edb28..7b6589215 100644 --- a/html/reference.json +++ b/html/reference.json @@ -376,9 +376,15 @@ "extras": "sampler: DPM++ 2M EDM" }, + "CogView 4": { + "path": "THUDM/CogView4-6B", + "desc": "An innovative cascaded framework that enhances the performance of text-to-image diffusion. CogView is the first model implementing relay diffusion in the realm of text-to-image generation, executing the task by first creating low-resolution images and subsequently applying relay-based super-resolution.", + "preview": "THUDM--CogView4-6B.jpg", + "skip": true + }, "CogView 3 Plus": { "path": "THUDM/CogView3-Plus-3B", - "desc": "This model is the DiT version of CogView3, a text-to-image generation model, supporting image generation from 512 to 2048px. Resolution: Width and height must meet the range from 512px to 2048px and must be divisible by 32.", + "desc": "An innovative cascaded framework that enhances the performance of text-to-image diffusion. CogView is the first model implementing relay diffusion in the realm of text-to-image generation, executing the task by first creating low-resolution images and subsequently applying relay-based super-resolution.", "preview": "THUDM--CogView3-Plus-3B.jpg", "skip": true }, diff --git a/models/Reference/Alpha-VLLM--Lumina-Image-2.0.jpg b/models/Reference/Alpha-VLLM--Lumina-Image-2.0.jpg old mode 100755 new mode 100644 diff --git a/models/Reference/THUDM--CogView4-6B.jpg b/models/Reference/THUDM--CogView4-6B.jpg new file mode 100644 index 000000000..5876b935b Binary files /dev/null and b/models/Reference/THUDM--CogView4-6B.jpg differ diff --git a/modules/model_cogview.py b/modules/model_cogview.py new file mode 100644 index 000000000..f76a95ed3 --- /dev/null +++ b/modules/model_cogview.py @@ -0,0 +1,95 @@ +import transformers +import diffusers +from modules import shared, devices, sd_models + + +def load_common(diffusers_load_config={}, module=None): + from modules import model_quant, modelloader + modelloader.hf_login() + + if 'torch_dtype' not in diffusers_load_config: + diffusers_load_config['torch_dtype'] = 'torch.float16' + if 'low_cpu_mem_usage' in diffusers_load_config: + del diffusers_load_config['low_cpu_mem_usage'] + if 'load_connected_pipeline' in diffusers_load_config: + del diffusers_load_config['load_connected_pipeline'] + if 'safety_checker' in diffusers_load_config: + del diffusers_load_config['safety_checker'] + if 'requires_safety_checker' in diffusers_load_config: + del diffusers_load_config['requires_safety_checker'] + + quant_args = {} + if not quant_args: + quant_args = model_quant.create_bnb_config(quant_args, module=module) + if not quant_args: + quant_args = model_quant.create_ao_config(quant_args, module=module) + if quant_args: + shared.log.debug(f'Load model: type=CogView quantization module="{module}" {quant_args}') + + return diffusers_load_config, quant_args + + +def load_cogview3(checkpoint_info, diffusers_load_config={}): + repo_id = sd_models.path_to_repo(checkpoint_info.name) + shared.log.debug(f'Load model: type=CogView3 model="{checkpoint_info.name}" repo="{repo_id}" offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype}') + + diffusers_load_config, quant_args = load_common(diffusers_load_config, module='Model') + transformer = diffusers.CogView3PlusTransformer2DModel.from_pretrained( + repo_id, + subfolder="transformer", + cache_dir=shared.opts.diffusers_dir, + **diffusers_load_config, + **quant_args, + ) + + diffusers_load_config, quant_args = load_common(diffusers_load_config, module='Text Encoder') + text_encoder = transformers.T5EncoderModel.from_pretrained( + repo_id, + subfolder="text_encoder", + cache_dir=shared.opts.diffusers_dir, + **diffusers_load_config, + **quant_args, + ) + + pipe = diffusers.CogView3PlusPipeline.from_pretrained( + repo_id, + text_encoder=text_encoder, + transformer=transformer, + cache_dir=shared.opts.diffusers_dir, + **diffusers_load_config, + ) + devices.torch_gc() + return pipe + + +def load_cogview4(checkpoint_info, diffusers_load_config={}): + repo_id = sd_models.path_to_repo(checkpoint_info.name) + shared.log.debug(f'Load model: type=CogView4 model="{checkpoint_info.name}" repo="{repo_id}" offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype}') + + diffusers_load_config, quant_args = load_common(diffusers_load_config, module='Model') + transformer = diffusers.CogView4Transformer2DModel.from_pretrained( + repo_id, + subfolder="transformer", + cache_dir=shared.opts.diffusers_dir, + **diffusers_load_config, + **quant_args, + ) + + diffusers_load_config, quant_args = load_common(diffusers_load_config, module='Text Encoder') + text_encoder = transformers.T5EncoderModel.from_pretrained( + repo_id, + subfolder="text_encoder", + cache_dir=shared.opts.diffusers_dir, + **diffusers_load_config, + **quant_args, + ) + + pipe = diffusers.CogView4Pipeline.from_pretrained( + repo_id, + text_encoder=text_encoder, + transformer=transformer, + cache_dir=shared.opts.diffusers_dir, + **diffusers_load_config, + ) + devices.torch_gc() + return pipe diff --git a/modules/model_flux.py b/modules/model_flux.py index bdb42037b..190007819 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -110,39 +110,6 @@ def load_flux_bnb(checkpoint_info, diffusers_load_config): # pylint: disable=unu return transformer, text_encoder_2 -""" -def quant_flux_bnb(checkpoint_info, transformer, text_encoder_2): - repo_id = sd_models.path_to_repo(checkpoint_info.name) - cache_dir=shared.opts.diffusers_dir - if len(shared.opts.bnb_quantization) > 0 and (transformer is None or text_encoder_2 is None): - from modules.model_quant import load_bnb - load_bnb('Load model: type=FLUX') - try: - bnb_config = diffusers.BitsAndBytesConfig( - load_in_8bit=shared.opts.bnb_quantization_type in ['fp8'], - load_in_4bit=shared.opts.bnb_quantization_type in ['nf4', 'fp4'], - bnb_4bit_quant_storage=shared.opts.bnb_quantization_storage, - bnb_4bit_quant_type=shared.opts.bnb_quantization_type, - bnb_4bit_compute_dtype=devices.dtype - ) - if ('Model' in shared.opts.bnb_quantization) and (transformer is None): - transformer = diffusers.FluxTransformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, quantization_config=bnb_config, torch_dtype=devices.dtype) - shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') - if ('Text Encoder' in shared.opts.bnb_quantization) and (text_encoder_2 is None): - if repo_id == 'sayakpaul/flux.1-dev-nf4': - repo_id = 'black-forest-labs/FLUX.1-dev' # workaround since sayakpaul model is missing model_index.json - text_encoder_2 = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder_2", cache_dir=cache_dir, quantization_config=bnb_config, torch_dtype=devices.dtype) - shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') - except Exception as e: - shared.log.error(f"Load model: type=FLUX failed quantize using BnB: {e}") - transformer, text_encoder_2 = None, None - if debug: - from modules import errors - errors.display(e, 'FLUX:') - return transformer, text_encoder_2 -""" - - def load_quants(kwargs, repo_id, cache_dir, allow_quant): try: if not allow_quant: diff --git a/modules/model_quant.py b/modules/model_quant.py index 23cc37aec..127254343 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -30,10 +30,10 @@ def get_quant(name): return 'none' -def create_bnb_config(kwargs = None, allow_bnb: bool = True): +def create_bnb_config(kwargs = None, allow_bnb: bool = True, module: str = 'Model'): from modules import shared, devices if len(shared.opts.bnb_quantization) > 0 and allow_bnb: - if 'Model' in shared.opts.bnb_quantization: + if 'Model' in shared.opts.bnb_quantization or (module is not None and module in shared.opts.bnb_quantization): load_bnb() if bnb is None: return kwargs @@ -53,10 +53,10 @@ def create_bnb_config(kwargs = None, allow_bnb: bool = True): return kwargs -def create_ao_config(kwargs = None, allow_ao: bool = True): +def create_ao_config(kwargs = None, allow_ao: bool = True, module: str = 'Model'): from modules import shared if len(shared.opts.torchao_quantization) > 0 and shared.opts.torchao_quantization_mode == 'pre' and allow_ao: - if 'Model' in shared.opts.torchao_quantization: + if 'Model' in shared.opts.torchao_quantization or (module is not None and module in shared.opts.torchao_quantization): load_torchao() if ao is None: return kwargs diff --git a/modules/modeldata.py b/modules/modeldata.py index 0105771e5..012372b3b 100644 --- a/modules/modeldata.py +++ b/modules/modeldata.py @@ -37,6 +37,10 @@ def get_model_type(pipe): model_type = 'lumina' elif "OmniGen" in name: model_type = 'omnigen' + elif "CogView3" in name: + model_type = 'cogview3' + elif "CogView4" in name: + model_type = 'cogview4' elif "CogVideo" in name: model_type = 'cogvideox' elif "Sana" in name: diff --git a/modules/sd_detect.py b/modules/sd_detect.py index f011fd77a..642043f29 100644 --- a/modules/sd_detect.py +++ b/modules/sd_detect.py @@ -75,8 +75,10 @@ def detect_pipeline(f: str, op: str = 'model', warning=True, quiet=False): guess = 'Kolors' if 'auraflow' in f.lower(): guess = 'AuraFlow' - if 'cogview' in f.lower(): - guess = 'CogView' + if 'cogview3' in f.lower(): + guess = 'CogView3' + if 'cogview4' in f.lower(): + guess = 'CogView4' if 'meissonic' in f.lower(): guess = 'Meissonic' pipeline = 'custom' diff --git a/modules/sd_models.py b/modules/sd_models.py index bb5b672f7..0d662f9df 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -295,9 +295,13 @@ def load_diffuser_force(model_type, checkpoint_info, diffusers_load_config, op=' sd_model = load_lumina2(checkpoint_info, diffusers_load_config) elif model_type in ['Stable Diffusion 3']: from modules.model_sd3 import load_sd3 - shared.log.debug(f'Load {op}: model="Stable Diffusion 3"') - shared.opts.scheduler = 'Default' sd_model = load_sd3(checkpoint_info, cache_dir=shared.opts.diffusers_dir, config=diffusers_load_config.get('config', None)) + elif model_type in ['CogView3']: # forced pipeline + from modules.model_cogview import load_cogview3 + sd_model = load_cogview3(checkpoint_info, diffusers_load_config) + elif model_type in ['CogView4']: # forced pipeline + from modules.model_cogview import load_cogview4 + sd_model = load_cogview4(checkpoint_info, diffusers_load_config) elif model_type in ['Meissonic']: # forced pipeline from modules.model_meissonic import load_meissonic sd_model = load_meissonic(checkpoint_info, diffusers_load_config) diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index 0b50c5c6c..5eb4a5f94 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -9,7 +9,7 @@ from modules import shared, devices, processing, images, sd_vae_approx, sd_vae_t SamplerData = namedtuple('SamplerData', ['name', 'constructor', 'aliases', 'options']) approximation_indexes = { "Simple": 0, "Approximate": 1, "TAESD": 2, "Full VAE": 3 } -flow_models = ['f1', 'sd3', 'lumina', 'auraflow', 'sana', 'lumina2'] +flow_models = ['f1', 'sd3', 'lumina', 'auraflow', 'sana', 'lumina2', 'cogview4'] warned = False queue_lock = threading.Lock() diff --git a/modules/shared_items.py b/modules/shared_items.py index 5c1e3aebb..7b9940a45 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -89,7 +89,8 @@ def get_pipelines(): 'SegMoE': getattr(diffusers, 'StableDiffusionPipeline', None), # dynamically redefined and loaded in sd_models.load_diffuser 'Kolors': getattr(diffusers, 'KolorsPipeline', None), 'AuraFlow': getattr(diffusers, 'AuraFlowPipeline', None), - 'CogView': getattr(diffusers, 'CogView3PlusPipeline', None), + 'CogView3': getattr(diffusers, 'CogView3PlusPipeline', None), + 'CogView4': getattr(diffusers, 'CogView4Pipeline', None), 'Stable Cascade': getattr(diffusers, 'StableCascadeCombinedPipeline', None), 'PixArt-Sigma': getattr(diffusers, 'PixArtSigmaPipeline', None), 'HunyuanDiT': getattr(diffusers, 'HunyuanDiTPipeline', None), diff --git a/wiki b/wiki index 910dc3083..cba8d182b 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 910dc3083c087eb806f548f05b6cf86ce4666268 +Subproject commit cba8d182b3cb18aeb4519d7c211d14cf9532021c