diff --git a/CHANGELOG.md b/CHANGELOG.md index bfb581ef4..edf341fdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,16 +1,17 @@ # Change Log for SD.Next -## Update for 2025-10-24 +## Update for 2025-10-25 - **Models** - [Tencent HunyuanImage 2.1](https://huggingface.co/tencent/HunyuanImage-2.1) in *full*, *distilled* and *refiner* variants HunyuanImage-2.1 is a large (51GB) T2I model capable of natively generating 2K images and uses Qwen2.5 + T5 text-encoders and 32x VAE - - networks reference section is now split into actual **Reference** models plus: - **Distilled** which shows distilled variants of base models and **Community** which shows community highlights - - Add SDNQ-SVD **pre-quantized** models to distilled models: *FLUX.1-Dev, Chroma1-HD, NoobAI-XL* - *note*: these models are pre-quantized to minimal resource usage while maintaining high quality - if you're low on vram and don't want to explore all of quantization options, these are a good starting point! - - Add additional community models: *CenKreChro, WAI-Illustrious, NoobAI, Pony-Realism* +- **Reference** networks section is now split into actual *Base* models plus: + - **Quantized**: pre-quantized variants of the base models using SDNQ-SVD quantization for optimal quality and smallest possible resource usage + examples: *FLUX.1-Dev/Krea/Kontext/Schnell, Qwen-Image/Edit/2509, Chroma1-HD, WAN-2.2-A44B, etc.* + - **Distilled**: distilled variants of base models + examples: *Turbo, Lightning, Lite, SRPO, Distill, Pruning, etc.* + - **Community**: community highlights + examples: *Tempest, Juggernaut, Illustrious, Pony, NoobAI, etc.* - **Features** - **offline mode**: enable in *settings -> hugginface* enables fully offline mode where previously downloaded models can be used as-is @@ -19,7 +20,7 @@ - switch to `torch==2.9` for *ipex, rocm and openvino* - switch to `rocm==7.0` for nightlies - **Quantization** - - improved SDNQ SVD and low-bit matmul performance + - improved **SDNQ SVD** and low-bit matmul performance - **Other** - change default **schedulers** for sdxl - warn on `python==3.9` end-of-life and `python==3.10` not actively supported @@ -27,6 +28,7 @@ - enhance `--optional` flag to pre-install optional packages - add `[lora]` to recognized filename patterns - add **Apple DepthPro** controlnet processor, thanks @nolbert82 + - when using **shared-t5** *(default)*, it will load standard or pre-quant depending on model - **Fixes** - startup error with `--profile` enabled if using `--skip` - restore orig init image for each batch sequence @@ -40,6 +42,8 @@ - lora auto-detect low/high stage if not specified - lora disable fuse on partially applied network - fix networks display with extended characters, thanks @awsr + - installer handle different `opencv` package variants + - fix using pre-quantized shared-t5 ## Update for 2025-10-18 diff --git a/TODO.md b/TODO.md index 6471ccd81..226f295a3 100644 --- a/TODO.md +++ b/TODO.md @@ -35,7 +35,6 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - [Ovi](https://github.com/character-ai/Ovi) - [Bytedance Lynx](https://github.com/bytedance/lynx) -- [HunyuanImage](https://huggingface.co/tencent/HunyuanImage-2.1) - [Phantom HuMo](https://github.com/Phantom-video/Phantom) - [Lumina-DiMOO](https://huggingface.co/Alpha-VLLM/Lumina-DiMOO) - [Wan2.2-Animate-14B](https://huggingface.co/Wan-AI/Wan2.2-Animate-14B) diff --git a/cli/hf-search.py b/cli/hf-search.py index f69c215eb..9ee696602 100755 --- a/cli/hf-search.py +++ b/cli/hf-search.py @@ -7,12 +7,12 @@ from rich import print # pylint: disable=redefined-builtin if __name__ == "__main__": sys.argv.pop(0) keyword = sys.argv[0] if len(sys.argv) > 0 else '' + hf.logging.set_verbosity_info() hf_api = hf.HfApi() - model_filter = hf.ModelFilter( - model_name=keyword, - # task='text-to-image', - library=['diffusers'], - ) - res = hf_api.list_models(filter=model_filter, full=True, limit=50, sort="downloads", direction=-1) - models = [{ 'name': m.id, 'downloads': m.downloads, 'mtime': m.lastModified, 'url': f'https://huggingface.co/{m.id}', 'pipeline': m.pipeline_tag, 'tags': m.tags } for m in res] - print(models) + res = hf_api.list_models(model_name=keyword, full=True, limit=100, sort="downloads", direction=-1) + res = sorted(res, key=lambda x: x.id) + for m in res: + meta = hf_api.model_info(m.id, files_metadata=True) + m.files = [f.rfilename for f in meta.siblings if f.rfilename.endswith('.bin') or f.rfilename.endswith('.safetensors')] + m.size = sum([f.size for f in meta.siblings]) / 1024 / 1024 / 1024 # in GB + print({ 'name': m.id, 'files': len(m.files), 'size': m.size, 'downloads': m.downloads, 'mtime': m.lastModified, 'url': f'https://huggingface.co/{m.id}', 'pipeline': m.pipeline_tag }) diff --git a/configs/sdxl/scheduler/scheduler_config.json b/configs/sdxl/scheduler/scheduler_config.json index 4b974f7ca..a55b71ccb 100644 --- a/configs/sdxl/scheduler/scheduler_config.json +++ b/configs/sdxl/scheduler/scheduler_config.json @@ -2,7 +2,7 @@ "_class_name": "EulerAncestralDiscreteScheduler", "_diffusers_version": "0.35.1", "beta_end": 0.012, - "beta_schedule": "linear", + "beta_schedule": "scaled_linear", "beta_start": 0.00085, "clip_sample": false, "interpolation_type": "linear", diff --git a/html/reference.json b/html/reference.json index b5113b612..2f7f83b1e 100644 --- a/html/reference.json +++ b/html/reference.json @@ -68,7 +68,7 @@ "tags": "distilled", "date": "2024 February" }, - "StabilityAI Stable Diffusion 3 Medium": { + "StabilityAI Stable Diffusion 3.0 Medium": { "path": "stabilityai/stable-diffusion-3-medium-diffusers", "skip": true, "variant": "fp16", @@ -806,28 +806,6 @@ "date": "2024 October" }, - "ShuttleAI Shuttle 3.0 Diffusion": { - "path": "shuttleai/shuttle-3-diffusion", - "desc": "Shuttle uses Flux.1 Schnell as its base. It can produce images similar to Flux Dev or Pro in just 4 steps, and it is licensed under Apache 2. The model was partially de-distilled during training. When used beyond 10 steps, it enters refiner mode enhancing image details without altering the composition", - "preview": "shuttleai--shuttle-3-diffusion.jpg", - "tags": "community", - "skip": true - }, - "ShuttleAI Shuttle 3.1 Aesthetic": { - "path": "shuttleai/shuttle-3.1-aesthetic", - "desc": "Shuttle uses Flux.1 Schnell as its base. It can produce images similar to Flux Dev or Pro in just 4 steps, and it is licensed under Apache 2. The model was partially de-distilled during training. When used beyond 10 steps, it enters refiner mode enhancing image details without altering the composition", - "preview": "shuttleai--shuttle-3_1-aestetic.jpg", - "tags": "community", - "skip": true - }, - "ShuttleAI Shuttle Jaguar": { - "path": "shuttleai/shuttle-jaguar", - "desc": "Shuttle uses Flux.1 Schnell as its base. It can produce images similar to Flux Dev or Pro in just 4 steps, and it is licensed under Apache 2. The model was partially de-distilled during training. When used beyond 10 steps, it enters refiner mode enhancing image details without altering the composition", - "preview": "shuttleai--shuttle-jaguar.jpg", - "tags": "community", - "skip": true - }, - "Bria 3.2": { "path": "briaai/BRIA-3.2", "desc": "Bria 3.2 is the next-generation commercial-ready text-to-image model. With just 4 billion parameters, it provides exceptional aesthetics and text rendering, evaluated to provide on par results to leading open-source models, and outperforming other licensed models.", @@ -931,40 +909,127 @@ "FLUX.1-Dev sdnq-svd-uint4": { "path": "Disty0/FLUX.1-dev-SDNQ-uint4-svd-r32", "preview": "Disty0--FLUX.1-dev-SDNQ-uint4-svd-r32.jpg", - "desc": "4 bit (UINT4 with SVD rank 32) quantization of black-forest-labs/FLUX.1-dev using SDNQ.", + "desc": "Quantization of black-forest-labs/FLUX.1-dev using SDNQ: sdnq-svd 4-bit uint with svd rank 32", "skip": true, - "tags": "distilled", - "size": 6.8, + "tags": "quantized", + "size": 12.60, + "date": "2025 October", + "extras": "" + }, + "FLUX.1-Schnell sdnq-svd-uint4": { + "path": "Disty0/FLUX.1-schnell-SDNQ-uint4-svd-r32", + "preview": "Disty0--FLUX.1-dev-SDNQ-uint4-svd-r32.jpg", + "desc": "Quantization of black-forest-labs/FLUX.1-schnell using SDNQ: sdnq-svd 4-bit uint with svd rank 32", + "skip": true, + "tags": "quantized", + "size": 12.60, + "date": "2025 October", + "extras": "" + }, + "FLUX.1-Dev Krea sdnq-svd-uint4": { + "path": "Disty0/FLUX.1-Krea-dev-SDNQ-uint4-svd-r32", + "preview": "Disty0--FLUX.1-dev-SDNQ-uint4-svd-r32.jpg", + "desc": "Quantization of black-forest-labs/FLUX.1-Krea-dev using SDNQ: sdnq-svd 4-bit uint with svd rank 32", + "skip": true, + "tags": "quantized", + "size": 12.60, + "date": "2025 October", + "extras": "" + }, + "FLUX.1-Dev Kontext sdnq-svd-uint4": { + "path": "Disty0/FLUX.1-Kontext-dev-SDNQ-uint4-svd-r32", + "preview": "Disty0--FLUX.1-dev-SDNQ-uint4-svd-r32.jpg", + "desc": "Quantization of black-forest-labs/FLUX.1-Kontext-dev using SDNQ: sdnq-svd 4-bit uint with svd rank 32", + "skip": true, + "tags": "quantized", + "size": 12.60, "date": "2025 October", "extras": "" }, "Chroma1-HD sdnq-svd-uint4": { "path": "Disty0/Chroma1-HD-SDNQ-uint4-svd-r32", "preview": "Disty0--Chroma1-HD-SDNQ-uint4-svd-r32.jpg", - "desc": "4 bit (UINT4 with SVD rank 32) quantization of lodestones/Chroma1-HD using SDNQ.", + "desc": "Quantization of lodestones/Chroma1-HD using SDNQ: sdnq-svd 4-bit uint with svd rank 32", "skip": true, - "tags": "distilled", - "size": 5.4, + "tags": "quantized", + "size": 11.89, "date": "2025 October", "extras": "" }, - "NoobAI-XL v1.1 sdnq-svd-uint4": { + "Wan-AI Wan2.2 A14B T2I sdnq-svd-uint4": { + "path": "Disty0/Wan2.2-T2V-A14B-SDNQ-uint4-svd-r32", + "preview": "Wan-AI--Wan2.2-T2V-A14B-Diffusers.jpg", + "desc": "Quantization of black-forest-labs/FLUX.1-dev using SDNQ: sdnq-svd 4-bit uint with svd rank 32", + "skip": true, + "tags": "quantized", + "date": "2025 October", + "size": 23.54, + "extras": "" + }, + "Wan-AI Wan2.2 A14B I2I sdnq-svd-uint4": { + "path": "Disty0/Wan2.2-I2V-A14B-SDNQ-uint4-svd-r32", + "preview": "Wan-AI--Wan2.2-T2V-A14B-Diffusers.jpg", + "desc": "Quantization of Laxhar/noobai-XL-1.1 using SDNQ: sdnq-svd 4-bit uint with svd rank 128", + "skip": true, + "tags": "quantized", + "date": "2025 October", + "size": 23.55, + "extras": "" + }, + "Qwen-Image sdnq-svd-uint4": { + "path": "Disty0/Qwen-Image-SDNQ-uint4-svd-r32", + "preview": "Qwen--Qwen-Image.jpg", + "desc": "Quantization of Qwen/Qwen-Image using SDNQ: sdnq-svd 4-bit uint with svd rank 32", + "skip": true, + "tags": "quantized", + "date": "2025 October", + "size": 16.09, + "extras": "" + }, + "Qwen-Image-Edit sdnq-svd-uint4": { + "path": "Disty0/Qwen-Image-Edit-SDNQ-uint4-svd-r32", + "preview": "Qwen--Qwen-Image-Edit.jpg", + "desc": "Quantization of Qwen/Qwen-Image-Edit using SDNQ: sdnq-svd 4-bit uint with svd rank 32", + "skip": true, + "tags": "quantized", + "date": "2025 October", + "size": 16.10, + "extras": "" + }, + "Qwen-Image-Edit-2509 sdnq-svd-uint4": { + "path": "Disty0/Qwen-Image-Edit-2509-SDNQ-uint4-svd-r32", + "preview": "Qwen--Qwen-Image-Edit-2509.jpg", + "desc": "Quantization of Qwen/Qwen-Image-Edit-2509 using SDNQ: sdnq-svd 4-bit uint with svd rank 32", + "skip": true, + "tags": "quantized", + "date": "2025 October", + "size": 16.10, + "extras": "" + }, + "Tempest-by-Vlad XL sdnq-svd-uint4": { + "path": "vladmandic/tempestByVlad_baseV01-SDNQ-uint4-svd", + "preview": "tempestByVlad_baseV01.jpg", + "desc": "Quantization of vladmandic/tempestByVlad_baseV01 using SDNQ: sdnq-svd 4-bit uint with svd rank 128", + "tags": "quantized", + "size": 3.37, + "date": "2025 October", + "extras": "" + }, + "NoobAI-XL v1.1 epsilon sdnq-svd-uint4": { "path": "Disty0/NoobAI-XL-v1.1-SDNQ-uint4-svd-r128", "preview": "Disty0--NoobAI-XL-v1.1-SDNQ-uint4-svd-r128.jpg", - "desc": "4 bit (UINT4 with SVD rank 128) quantization of Laxhar/noobai-XL-1.1 using SDNQ.", - "skip": true, - "tags": "distilled", - "size": 5.4, + "desc": "Quantization of Laxhar/noobai-XL-1.1 using SDNQ: sdnq-svd 4-bit uint with svd rank 128", + "tags": "quantized", + "size": 3.37, "date": "2025 October", "extras": "" }, "NoobAI-XL v1.0 v-pred sdnq-svd-uint4": { "path": "Disty0/NoobAI-XL-Vpred-v1.0-SDNQ-uint4-svd-r128", "preview": "Disty0--NoobAI-XL-Vpred-v1.0-SDNQ-uint4-svd-r128.jpg", - "desc": "4 bit (UINT4 with SVD rank 128) quantization of Laxhar/noobai-XL-Vpred-1.0 using SDNQ.", - "skip": true, - "tags": "distilled", - "size": 5.4, + "desc": "Quantization of Laxhar/noobai-XL-Vpred-1.0 using SDNQ: sdnq-svd 4-bit uint with svd rank 128", + "tags": "quantized", + "size": 3.37, "date": "2025 October", "extras": "" }, @@ -1015,15 +1080,6 @@ "tags": "community", "extras": "width: 512, height: 512, sampler: DEIS, steps: 20, cfg_scale: 6.0" }, - "Tiwaz CenKreChro": { - "path": "Tiwaz/CenKreChro", - "preview": "Tiwaz--CenKreChro.jpg", - "skip": true, - "desc": "Based Centerfold Flux 5, trying to merge in Chroma and Krea.", - "extras": "", - "tags": "community", - "date": "2025 September" - }, "WAI Illustrious XL v15": { "path": "waiIllustriousSDXL_v150.safetensors@https://civitai.com/api/download/models/2167369", "preview": "waiIllustriousSDXL_v150.jpg", @@ -1068,5 +1124,36 @@ "size": 6.94, "date": "2025 May", "extras": "" + }, + "Tiwaz CenKreChro": { + "path": "Tiwaz/CenKreChro", + "preview": "Tiwaz--CenKreChro.jpg", + "skip": true, + "desc": "Based Centerfold Flux 5, trying to merge in Chroma and Krea.", + "extras": "", + "tags": "community", + "date": "2025 September" + }, + "ShuttleAI Shuttle 3.0 Diffusion": { + "path": "shuttleai/shuttle-3-diffusion", + "desc": "Shuttle uses Flux.1 Schnell as its base. It can produce images similar to Flux Dev or Pro in just 4 steps, and it is licensed under Apache 2. The model was partially de-distilled during training. When used beyond 10 steps, it enters refiner mode enhancing image details without altering the composition", + "preview": "shuttleai--shuttle-3-diffusion.jpg", + "tags": "community", + "skip": true + }, + "ShuttleAI Shuttle 3.1 Aesthetic": { + "path": "shuttleai/shuttle-3.1-aesthetic", + "desc": "Shuttle uses Flux.1 Schnell as its base. It can produce images similar to Flux Dev or Pro in just 4 steps, and it is licensed under Apache 2. The model was partially de-distilled during training. When used beyond 10 steps, it enters refiner mode enhancing image details without altering the composition", + "preview": "shuttleai--shuttle-3_1-aestetic.jpg", + "tags": "community", + "skip": true + }, + "ShuttleAI Shuttle Jaguar": { + "path": "shuttleai/shuttle-jaguar", + "desc": "Shuttle uses Flux.1 Schnell as its base. It can produce images similar to Flux Dev or Pro in just 4 steps, and it is licensed under Apache 2. The model was partially de-distilled during training. When used beyond 10 steps, it enters refiner mode enhancing image details without altering the composition", + "preview": "shuttleai--shuttle-jaguar.jpg", + "tags": "community", + "skip": true } + } diff --git a/installer.py b/installer.py index a06d04868..bcf6f37ba 100644 --- a/installer.py +++ b/installer.py @@ -610,9 +610,6 @@ def check_diffusers(): t_start = time.time() if args.skip_all: return - if args.skip_git: - install('diffusers') - return sha = '7536f647e4144c7acaf9e140893ff7edb85bf9a3' # diffusers commit hash # if args.use_rocm or args.use_zluda or args.use_directml: # sha = '043ab2520f6a19fce78e6e060a68dbc947edb9f9' # lock diffusers versions for now @@ -625,6 +622,8 @@ def check_diffusers(): else: log.info(f'Diffusers update: current={pkg.version} hash={cur} target={sha}') pip('uninstall --yes diffusers', ignore=True, quiet=True, uv=False) + if args.skip_git: + log.warning('Git: marked as not available but required for diffusers installation') pip(f'install --upgrade git+https://github.com/huggingface/diffusers@{sha}', ignore=False, quiet=True, uv=False) global diffusers_commit # pylint: disable=global-statement diffusers_commit = sha @@ -1261,6 +1260,13 @@ def install_pydantic(): reload('pydantic', '1.10.21') +def install_opencv(): + install('opencv-python==4.12.0.88', ignore=True, quiet=True) + install('opencv-python-headless==4.12.0.88', ignore=True, quiet=True) + install('opencv-contrib-python==4.12.0.88', ignore=True, quiet=True) + install('opencv-contrib-python-headless==4.12.0.88', ignore=True, quiet=True) + + def install_insightface(): install('git+https://github.com/deepinsight/insightface@29b6cd65aa0e9ae3b6602de3c52e9d8949c8ee86#subdirectory=python-package', 'insightface') # insightface==0.7.3 with patches if args.new: @@ -1329,6 +1335,7 @@ def install_requirements(): if not installed(line, quiet=True): _res = install(line) install_pydantic() + install_opencv() if args.profile: pr.disable() print_profile(pr, 'Requirements') diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index d7affa804..aa9b71246 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -153,6 +153,10 @@ async function filterExtraNetworksForTab(searchTerm) { cards.forEach((elem) => elem.style.display = elem.dataset.tags .toLowerCase() .includes('community') ? '' : 'none'); + } else if (searchTerm === 'quantized/') { + cards.forEach((elem) => elem.style.display = elem.dataset.tags + .toLowerCase() + .includes('quantized') ? '' : 'none'); } else if (searchTerm === 'local/') { cards.forEach((elem) => elem.style.display = elem.dataset.name .toLowerCase() diff --git a/javascript/settings.js b/javascript/settings.js index f6d133a19..f82f0a692 100644 --- a/javascript/settings.js +++ b/javascript/settings.js @@ -170,7 +170,7 @@ async function initModels() { if (en.classList.contains('hide')) gradioApp().getElementById('txt2img_extra_networks_btn').click(); const repeat = setInterval(() => { const buttons = Array.from(gradioApp().querySelectorAll('#txt2img_model_subdirs > button')) || []; - const reference = buttons.find((b) => (b.innerText === 'Reference') || (b.innerText === 'Distilled') || (b.innerText === 'Community')); + const reference = buttons.find((b) => (b.innerText === 'Reference') || (b.innerText === 'Distilled') || (b.innerText === 'Community') || (b.innerText === 'Quantized')); if (reference) { clearInterval(repeat); reference.click(); diff --git a/launch.py b/launch.py index 40e1946fc..57a1930af 100755 --- a/launch.py +++ b/launch.py @@ -297,10 +297,9 @@ def main(): installer.log.info(f'Installer time: {init_summary()}') get_custom_args() + uv, instance = start_server(immediate=True, server=None) if installer.restart_required: installer.log.warning('Restart is recommended due to packages updates...') - - uv, instance = start_server(immediate=True, server=None) t_server = time.time() t_monitor = time.time() while True: diff --git a/modules/images_namegen.py b/modules/images_namegen.py index 66886cc9e..083d3c6b1 100644 --- a/modules/images_namegen.py +++ b/modules/images_namegen.py @@ -72,16 +72,16 @@ class FilenameGenerator: self.p = p if seed is not None and int(seed) > 0: self.seed = seed - elif p is not None and hasattr(p, 'all_seeds'): - self.seed = p.all_seeds[0] - elif p is not None and hasattr(p, 'seeds'): - self.seed = p.seeds[0] + elif p is not None and getattr(p, 'all_seeds', None) is not None and len(p.all_seeds) > 0: + self.seed = p.all_seeds[0] if int(p.all_seeds[0]) > 0 else 0 + elif p is not None and getattr(p, 'seeds', None) is not None and len(p.seeds) > 0: + self.seed = p.seeds[0] if int(p.seeds[0]) > 0 else 0 else: - self.seed = p.seed if p is not None else 0 + self.seed = p.seed if p is not None and getattr(p, 'seed', 0) > 0 else 0 if prompt is not None: self.prompt = prompt else: - self.prompt = p.prompt if p is not None else '' + self.prompt = p.prompt if p is not None and getattr(p, 'prompt', '') != '' else '' if isinstance(self.prompt, list): self.prompt = ' '.join(self.prompt) self.image = image @@ -169,7 +169,7 @@ class FilenameGenerator: def sanitize(self, filename): # starting reference: invalid_chars = ( - "#<>/\"'`" # ASCII quote and backtick + "#<>\"'`" # ASCII quote and backtick "’‚‛\u2018\u2019\u201B" # smart single quotes and variants # noqa: RUF001 "\u02BB" # modifier letter turned comma "\u201C\u201D\u201F" # smart double quotes and variants diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index 750439f0c..ead010828 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -14,14 +14,14 @@ processor = None model = None loaded: str = None quant_args = None -vlm_default = "Alibaba Qwen 2.5 VL 4B" +vlm_default = "Alibaba Qwen 2.5 VL 3B" vlm_models = { "Google Gemma 3 4B": "google/gemma-3-4b-it", "Google Gemma 3n E2B": "google/gemma-3n-E2B-it", # 1.5GB "Google Gemma 3n E4B": "google/gemma-3n-E4B-it", # 1.5GB "Alibaba Qwen 2.0 VL 2B": "Qwen/Qwen2-VL-2B-Instruct", "Alibaba Qwen 2.5 Omni 3B": "Qwen/Qwen2.5-Omni-3B", - "Alibaba Qwen 2.5 VL 4B": "Qwen/Qwen2.5-VL-3B-Instruct", + "Alibaba Qwen 2.5 VL 3B": "Qwen/Qwen2.5-VL-3B-Instruct", "Alibaba Qwen 3 VL 2B": "Qwen/Qwen3-VL-2B-Instruct", "Alibaba Qwen 3 VL 2B Thinking": "Qwen/Qwen3-VL-2B-Thinking", "Alibaba Qwen 3 VL 4B": "Qwen/Qwen3-VL-4B-Instruct", diff --git a/modules/loader.py b/modules/loader.py index d540fbac9..43824fc70 100644 --- a/modules/loader.py +++ b/modules/loader.py @@ -145,6 +145,7 @@ except Exception as e: sys.exit(1) import huggingface_hub # pylint: disable=W0611,C0411 +logging.getLogger("huggingface_hub.file_download").setLevel(logging.ERROR) timer.startup.record("hfhub") try: @@ -155,6 +156,9 @@ from PIL import Image # pylint: disable=W0611,C0411 timer.startup.record("pillow") +import cv2 # pylint: disable=W0611,C0411 +timer.startup.record("cv2") + class _tqdm_cls(): def __call__(self, *args, **kwargs): bar_format = 'Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + '{desc}' + '\x1b[0m' @@ -185,7 +189,7 @@ def get_packages(): try: import math cores = os.cpu_count() - affinity = len(os.sched_getaffinity(0)) + affinity = len(os.sched_getaffinity(0)) # pylint: disable=no-member threads = torch.get_num_threads() if threads < (affinity / 2): torch.set_num_threads(math.floor(affinity / 2)) @@ -225,4 +229,4 @@ class VersionString(str): # support both string and tuple for version check torch.__version__ = VersionString(torch.__version__) errors.log.info(f'Torch: torch=={torch.__version__} torchvision=={torchvision.__version__}') -errors.log.info(f'Packages: diffusers=={diffusers.__version__} transformers=={transformers.__version__} accelerate=={accelerate.__version__} gradio=={gradio.__version__} pydantic=={pydantic.__version__} numpy=={np.__version__}') +errors.log.info(f'Packages: diffusers=={diffusers.__version__} transformers=={transformers.__version__} accelerate=={accelerate.__version__} gradio=={gradio.__version__} pydantic=={pydantic.__version__} numpy=={np.__version__} cv2=={cv2.__version__}') diff --git a/modules/ltx/ltx_ui.py b/modules/ltx/ltx_ui.py index 6ac23bf86..ade98132a 100644 --- a/modules/ltx/ltx_ui.py +++ b/modules/ltx/ltx_ui.py @@ -14,7 +14,7 @@ def create_ui(prompt, negative, styles, overrides, init_image, init_strength, la with gr.Row(): generate = gr.Button('Generate', elem_id="ltx_generate_btn", variant='primary', visible=False) with gr.Row(): - ltx_models = [m.name for m in models['LTX Video']] + ltx_models = [m.name for m in models['LTX Video']] if 'LTX Video' in models else ['None'] model = gr.Dropdown(label='LTX model', choices=ltx_models, value=ltx_models[0]) with gr.Accordion(open=False, label="Condition", elem_id='ltx_condition_accordion'): with gr.Tabs(): diff --git a/modules/modelloader.py b/modules/modelloader.py index 645a7cc51..76456c3d7 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -79,7 +79,7 @@ def download_diffusers_model(hub_id: str, cache_dir: str = None, download_config download_config["mirror"] = mirror if custom_pipeline is not None and len(custom_pipeline) > 0: download_config["custom_pipeline"] = custom_pipeline - shared.log.debug(f'Diffusers downloading: id="{hub_id}" args={download_config}') + shared.log.debug(f'HF download: id="{hub_id}" args={download_config}') token = token or shared.opts.huggingface_token if token is not None and len(token) > 2: hf_login(token) @@ -94,20 +94,20 @@ def download_diffusers_model(hub_id: str, cache_dir: str = None, download_config except Exception as e: err = e ok = False - debug(f'Diffusers download error: id="{hub_id}" {e}') + debug(f'HF download error: id="{hub_id}" {e}') if not ok and 'Repository Not Found' not in str(err): try: download_config.pop('load_connected_pipeline', None) download_config.pop('variant', None) pipeline_dir = hf.snapshot_download(hub_id, **download_config) except Exception as e: - debug(f'Diffusers download error: id="{hub_id}" {e}') + debug(f'HF download error: id="{hub_id}" {e}') if 'gated' in str(e): - shared.log.error(f'Diffusers download error: id="{hub_id}" model access requires login') + shared.log.error(f'HF download error: id="{hub_id}" model access requires login') shared.state.end(jobid) return None if pipeline_dir is None: - shared.log.error(f'Diffusers download error: id="{hub_id}" {err}') + shared.log.error(f'HF download error: id="{hub_id}" {err}') shared.state.end(jobid) return None try: diff --git a/modules/sd_detect.py b/modules/sd_detect.py index 0b2d3d19c..c074aa3ee 100644 --- a/modules/sd_detect.py +++ b/modules/sd_detect.py @@ -179,7 +179,7 @@ def detect_pipeline(f: str, op: str = 'model'): pipeline = None if guess == 'Autodetect': try: - guess = 'Stable Diffusion XL' if 'XL' in f.upper() else 'Stable Diffusion' # set default guess + guess = 'Stable Diffusion XL' if ('XL' in f.upper() or 'SDNQ' in f.upper()) else 'Stable Diffusion' # set default guess guess = guess_by_size(f, guess) guess = guess_by_name(f, guess) guess, pipeline = guess_by_diffusers(f, guess) diff --git a/modules/sd_models.py b/modules/sd_models.py index ba465188a..b5b28a104 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -425,6 +425,10 @@ def load_diffuser_folder(model_type, pipeline, checkpoint_info, diffusers_load_c try: #0 - using detected model type and pipeline if (model_type is not None) and (pipeline is not None): + if ('sdnq' in model_type.lower()) or ('sdnq' in checkpoint_info.path.lower()): + from modules import sdnq # pylint: disable=unused-import # register to diffusers and transformers + global allow_post_quant # pylint: disable=global-statement + allow_post_quant = False sd_model = pipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) sd_model.model_type = sd_model.__class__.__name__ except Exception as e: @@ -466,7 +470,7 @@ def load_diffuser_folder(model_type, pipeline, checkpoint_info, diffusers_load_c try: # 3 - try basic pipeline just in case if err2 is not None: - sd_model = diffusers.StableDiffusionPipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) + sd_model = diffusers.StableDiffusionXLPipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config) sd_model.model_type = sd_model.__class__.__name__ except Exception as e: err3 = e # ignore last error @@ -576,8 +580,8 @@ def set_overrides(sd_model, checkpoint_info, model_type): and model_type.startswith("Stable Diffusion") and model_type != "Stable Diffusion 3" ): # SDXL and SD 1.5 scheduler_config = sd_model.scheduler.config - scheduler_config['beta_schedule'] = 'linear' - scheduler_config['timestep_spacing'] = 'trailing' + # scheduler_config['beta_schedule'] = 'scaled_linear' + # scheduler_config['timestep_spacing'] = 'trailing' sd_model.scheduler = diffusers.EulerAncestralDiscreteScheduler.from_config(scheduler_config) if 'bigaspv25' in checkpoint_info_name or ('flow' in checkpoint_info_name and 'flower' not in checkpoint_info_name): scheduler_config = sd_model.scheduler.config @@ -712,16 +716,16 @@ def load_diffuser(checkpoint_info=None, op='model', revision=None): # pylint: di allow_post_quant = False model_type = model_type.replace(' SDNQ', '') - # load from hf folder-style - if sd_model is None: - if os.path.isdir(checkpoint_info.path) or checkpoint_info.type == 'huggingface' or checkpoint_info.type == 'transformer': - sd_model = load_diffuser_folder(model_type, pipeline, checkpoint_info, diffusers_load_config, op) - # load from single-file if sd_model is None: if os.path.isfile(checkpoint_info.path) and checkpoint_info.path.lower().endswith('.safetensors'): sd_model = load_diffuser_file(model_type, pipeline, checkpoint_info, diffusers_load_config, op) + # load from hf folder-style + if sd_model is None: + if os.path.isdir(checkpoint_info.path) or (checkpoint_info.type == 'huggingface') or (checkpoint_info.type == 'transformer') or (checkpoint_info.type == 'reference'): + sd_model = load_diffuser_folder(model_type, pipeline, checkpoint_info, diffusers_load_config, op) + if sd_model is None: shared.log.error(f'Load {op}: name="{checkpoint_info.name if checkpoint_info is not None else None}" not loaded') return @@ -1147,18 +1151,19 @@ def set_diffusers_attention(pipe, quiet:bool=False): def add_noise_pred_to_diffusers_callback(pipe): if not hasattr(pipe, "_callback_tensor_inputs"): return pipe - if pipe.__class__.__name__.startswith("StableDiffusion"): - pipe._callback_tensor_inputs.append("noise_pred") # pylint: disable=protected-access - elif pipe.__class__.__name__.startswith("StableCascade"): + if pipe.__class__.__name__.startswith("StableCascade") and ("predicted_image_embedding" not in pipe._callback_tensor_inputs): # pylint: disable=protected-access pipe.prior_pipe._callback_tensor_inputs.append("predicted_image_embedding") # pylint: disable=protected-access - elif hasattr(pipe, "scheduler") and "flow" in pipe.scheduler.__class__.__name__.lower(): - pipe._callback_tensor_inputs.append("noise_pred") # pylint: disable=protected-access - elif hasattr(pipe, "scheduler") and hasattr(pipe.scheduler, "config") and getattr(pipe.scheduler.config, "prediction_type", "none") == "flow_prediction": - pipe._callback_tensor_inputs.append("noise_pred") # pylint: disable=protected-access - elif hasattr(pipe, "default_scheduler") and "flow" in pipe.default_scheduler.__class__.__name__.lower(): - pipe._callback_tensor_inputs.append("noise_pred") # pylint: disable=protected-access - elif hasattr(pipe, "default_scheduler") and hasattr(pipe.default_scheduler, "config") and getattr(pipe.default_scheduler.config, "prediction_type", "none") == "flow_prediction": - pipe._callback_tensor_inputs.append("noise_pred") # pylint: disable=protected-access + elif "noise_pred" not in pipe._callback_tensor_inputs: # pylint: disable=protected-access + if pipe.__class__.__name__.startswith("StableDiffusion"): + pipe._callback_tensor_inputs.append("noise_pred") # pylint: disable=protected-access + elif hasattr(pipe, "scheduler") and "flow" in pipe.scheduler.__class__.__name__.lower(): + pipe._callback_tensor_inputs.append("noise_pred") # pylint: disable=protected-access + elif hasattr(pipe, "scheduler") and hasattr(pipe.scheduler, "config") and (getattr(pipe.scheduler.config, "prediction_type", "none") == "flow_prediction"): + pipe._callback_tensor_inputs.append("noise_pred") # pylint: disable=protected-access + elif hasattr(pipe, "default_scheduler") and ("flow" in pipe.default_scheduler.__class__.__name__.lower()): + pipe._callback_tensor_inputs.append("noise_pred") # pylint: disable=protected-access + elif hasattr(pipe, "default_scheduler") and hasattr(pipe.default_scheduler, "config") and (getattr(pipe.default_scheduler.config, "prediction_type", "none") == "flow_prediction"): + pipe._callback_tensor_inputs.append("noise_pred") # pylint: disable=protected-access return pipe diff --git a/modules/sdnq/common.py b/modules/sdnq/common.py index c5ff8ab9a..c328cd76b 100644 --- a/modules/sdnq/common.py +++ b/modules/sdnq/common.py @@ -95,6 +95,10 @@ module_skip_keys_dict = { ["transformer_blocks.0.img_mod.1.weight", ".time_text_embed", ".txt_in", ".img_in", ".proj_out", ".norm_out", "pos_embed"], {} ], + "WanTransformer3DModel": [ + ["scale_shift_table", ".rope", ".patch_embedding", ".condition_embedder", ".proj_out", ".norm_out", "pos_embed"], + {} + ], "NaDiT": [ [".emb_in", ".txt_in", ".vid_in", ".emb_scale", ".vid_out", ".vid_out_norm", ".vid_out_ada"], {} diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index 50341f36f..6f3246236 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -273,6 +273,7 @@ class ExtraNetworksPage: subdirs['Local'] = 1 subdirs['Reference'] = 1 subdirs['Distilled'] = 1 + subdirs['Quantized'] = 1 subdirs['Community'] = 1 subdirs[diffusers_base] = 1 if self.name == 'style' and shared.opts.extra_networks_styles: @@ -289,13 +290,15 @@ class ExtraNetworksPage: subdirs.move_to_end('Reference', last=True) if 'Distilled' in subdirs: subdirs.move_to_end('Distilled', last=True) + if 'Quantized' in subdirs: + subdirs.move_to_end('Quantized', last=True) if 'Community' in subdirs: subdirs.move_to_end('Community', last=True) subdirs_html = '' for subdir in subdirs: if len(subdir) == 0: continue - if subdir in ['All', 'Local', 'Diffusers', 'Reference', 'Distilled', 'Community']: + if subdir in ['All', 'Local', 'Diffusers', 'Reference', 'Distilled', 'Quantized', 'Community']: style = 'network-reference' else: style = 'network-folder' @@ -544,6 +547,7 @@ def register_pages(): if shared.opts.diffusers_enable_embed: from modules.ui_extra_networks_textual_inversion import ExtraNetworksPageTextualInversion register_page(ExtraNetworksPageTextualInversion()) + from modules.video_models.models_def import models # pylint: disable=unused-import def get_pages(title=None): diff --git a/modules/ui_extra_networks_checkpoints.py b/modules/ui_extra_networks_checkpoints.py index cb21fe608..2fe4db376 100644 --- a/modules/ui_extra_networks_checkpoints.py +++ b/modules/ui_extra_networks_checkpoints.py @@ -4,6 +4,7 @@ import json import concurrent from datetime import datetime from modules import shared, ui_extra_networks, sd_models, modelstats, paths +from modules.json_helpers import readfile version_map = { @@ -36,8 +37,10 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): return any(model.endswith(url) for model in existing) if not shared.opts.sd_checkpoint_autodownload or not shared.opts.extra_network_reference_enable: + shared.log.debug(f'Networks: type="reference" autodownload={shared.opts.sd_checkpoint_autodownload} enable={shared.opts.extra_network_reference_enable}') return [] - count = { 'total': 0, 'ready': 0, 'hidden': 0, 'experimental': 0 } + count = { 'total': 0, 'ready': 0, 'hidden': 0, 'experimental': 0, 'base': 0 } + shared.reference_models = readfile(os.path.join('html', 'reference.json')) for k, v in shared.reference_models.items(): count['total'] += 1 url = v['path'] @@ -64,12 +67,22 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): path = f'{v.get("path", "")}+{v.get("subfolder", "")}' else: path = f'{v.get("path", "")}' + ready = reference_downloaded(url) if not ready and shared.opts.offline_mode: count['hidden'] += 1 continue if ready: count['ready'] += 1 + + tag = v.get('tags', '') + if tag in count: + count[tag] += 1 + elif tag != '': + count[tag] = 1 + else: + count['base'] += 1 + yield { "type": 'Model', "name": name, @@ -85,9 +98,9 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage): "metadata": {}, "description": v.get('desc', ''), "version": "ready" if ready else "download", - "tags": v.get('tags', ''), + "tags": tag, } - shared.log.debug(f'Networks: type="reference" items={count["total"]} ready={count["ready"]} hidden={count["hidden"]} experimental={count["experimental"]}') + shared.log.debug(f'Networks: type="reference" items={count}') def create_item(self, name): record = None diff --git a/modules/ui_settings.py b/modules/ui_settings.py index 673e45941..b6803aaf1 100644 --- a/modules/ui_settings.py +++ b/modules/ui_settings.py @@ -385,11 +385,19 @@ def create_quicksettings(interfaces): def reference_submit(model): if '@' not in model: # diffusers loaded = modelloader.load_reference(model) - return model if loaded else shared.opts.sd_model_checkpoint + if loaded: + shared.opts.sd_model_checkpoint = model + sd_models.reload_model_weights(force=True) + return model + return shared.opts.sd_model_checkpoint else: # civitai model, url = model.split('@') loaded = modelloader.load_civitai(model, url) - return loaded if loaded is not None else shared.opts.sd_model_checkpoint + if loaded is not None: + shared.opts.sd_model_checkpoint = loaded.title + sd_models.reload_model_weights(force=True) + return loaded + return shared.opts.sd_model_checkpoint button_set_reference = gr.Button('Change reference', elem_id='change_reference', visible=False) button_set_reference.click( diff --git a/modules/video_models/models_def.py b/modules/video_models/models_def.py index f3b64b40c..0b97935d0 100644 --- a/modules/video_models/models_def.py +++ b/modules/video_models/models_def.py @@ -1,6 +1,8 @@ from dataclasses import dataclass +import time import diffusers import transformers +from installer import log @dataclass @@ -27,346 +29,354 @@ class Model(): return f'name="{self.name}" url="{self.url}" repo="{self.repo}" repo_cls="{self.repo_cls}" dit="{self.dit}" dit_cls="{self.dit_cls}" dit_folder="{self.dit_folder}" te="{self.te}" te_cls="{self.te_cls}" te_folder="{self.te_folder}" te_hijack={self.te_hijack} vae_hijack={self.vae_hijack} vae_remote={self.vae_remote}' -models = { - 'None': [], - 'Hunyuan Video': [ - Model(name='None'), - Model(name='Hunyuan Video T2V', - url='https://huggingface.co/tencent/HunyuanVideo', - vae_remote=True, - repo='hunyuanvideo-community/HunyuanVideo', - repo_cls=diffusers.HunyuanVideoPipeline, - te_cls=transformers.LlamaModel, - dit_cls=diffusers.HunyuanVideoTransformer3DModel), - Model(name='Hunyuan Video I2V', # https://github.com/huggingface/diffusers/pull/10983 - url='https://huggingface.co/tencent/HunyuanVideo-I2V', - vae_remote=True, - repo='hunyuanvideo-community/HunyuanVideo-I2V', - repo_cls=diffusers.HunyuanVideoImageToVideoPipeline, - te_cls=transformers.LlavaForConditionalGeneration, - dit_cls=diffusers.HunyuanVideoTransformer3DModel), - Model(name='SkyReels Hunyuan T2V', # https://github.com/huggingface/diffusers/pull/10837 - url='https://huggingface.co/Skywork/SkyReels-V1-Hunyuan-T2V', - vae_remote=True, - repo='hunyuanvideo-community/HunyuanVideo', - repo_cls=diffusers.HunyuanVideoPipeline, - te_cls=transformers.LlamaModel, - dit='Skywork/SkyReels-V1-Hunyuan-T2V', - dit_folder=None, - dit_cls=diffusers.HunyuanVideoTransformer3DModel), - Model(name='SkyReels Hunyuan I2V', # https://github.com/huggingface/diffusers/pull/10837 - url='https://huggingface.co/Skywork/SkyReels-V1-Hunyuan-I2V', - vae_remote=True, - repo='hunyuanvideo-community/HunyuanVideo', - repo_cls=diffusers.HunyuanSkyreelsImageToVideoPipeline, - te_cls=transformers.LlamaModel, - dit='Skywork/SkyReels-V1-Hunyuan-I2V', - dit_folder=None, - dit_cls=diffusers.HunyuanVideoTransformer3DModel), - Model(name='Fast Hunyuan T2V', # https://github.com/hao-ai-lab/FastVideo/blob/8a77cf22c9b9e7f931f42bc4b35d21fd91d24e45/fastvideo/models/hunyuan/inference.py#L213 - url='https://huggingface.co/FastVideo/FastHunyuan', - vae_remote=True, - repo='hunyuanvideo-community/HunyuanVideo', - repo_cls=diffusers.HunyuanVideoPipeline, - te_cls=transformers.LlamaModel, - dit='FastVideo/FastHunyuan-diffusers', - dit_cls=diffusers.HunyuanVideoTransformer3DModel), - ], - 'LTX Video': [ - Model(name='None'), - Model(name='LTXVideo 0.9.8 13B', - url='https://huggingface.co/Lightricks/LTX-Video-0.9.8-13B-distilled', - repo='Lightricks/LTX-Video-0.9.8-13B-distilled', - repo_cls=diffusers.LTXConditionPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.LTXVideoTransformer3DModel), - Model(name='LTXVideo 0.9.7 13B', - url='https://huggingface.co/Lightricks/LTX-Video-0.9.7-dev', - repo='a-r-r-o-w/LTX-Video-0.9.7-diffusers', - repo_cls=diffusers.LTXConditionPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.LTXVideoTransformer3DModel), - Model(name='LTXVideo 0.9.6 2B T2V', - url='https://huggingface.co/Lightricks/LTX-Video', - repo='Lightricks/LTX-Video', - repo_cls=diffusers.LTXConditionPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.LTXVideoTransformer3DModel), - Model(name='LTXVideo 0.9.6 2B I2V', - url='https://huggingface.co/Lightricks/LTX-Video', - repo='Lightricks/LTX-Video', - repo_cls=diffusers.LTXConditionPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.LTXVideoTransformer3DModel), - Model(name='LTXVideo 0.9.6 2B T2V Distilled', - url='https://huggingface.co/Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25', - repo='Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25', - repo_cls=diffusers.LTXConditionPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.LTXVideoTransformer3DModel), - Model(name='LTXVideo 0.9.6 2B I2V Distilled', - url='https://huggingface.co/Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25', - repo='Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25', - repo_cls=diffusers.LTXConditionPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.LTXVideoTransformer3DModel), - Model(name='LTXVideo 0.9.5 T2V', # https://github.com/huggingface/diffusers/pull/10968 - url='https://huggingface.co/Lightricks/LTX-Video-0.9.5', - repo='Lightricks/LTX-Video-0.9.5', - repo_cls=diffusers.LTXConditionPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.LTXVideoTransformer3DModel), - Model(name='LTXVideo 0.9.5 I2V', - url='https://huggingface.co/Lightricks/LTX-Video-0.9.5', - repo='Lightricks/LTX-Video-0.9.5', - repo_cls=diffusers.LTXConditionPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.LTXVideoTransformer3DModel), - Model(name='LTXVideo 0.9.1 T2V', - url='https://huggingface.co/a-r-r-o-w/LTX-Video-0.9.1-diffusers', - repo='a-r-r-o-w/LTX-Video-0.9.1-diffusers', - repo_cls=diffusers.LTXPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.LTXVideoTransformer3DModel), - Model(name='LTXVideo 0.9.1 I2V', - url='https://huggingface.co/a-r-r-o-w/LTX-Video-0.9.1-diffusers', - repo='a-r-r-o-w/LTX-Video-0.9.1-diffusers', - repo_cls=diffusers.LTXImageToVideoPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.LTXVideoTransformer3DModel), - Model(name='LTXVideo 0.9.0 T2V', - url='https://huggingface.co/a-r-r-o-w/LTX-Video-diffusers', - repo='a-r-r-o-w/LTX-Video-diffusers', - repo_cls=diffusers.LTXPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.LTXVideoTransformer3DModel), - Model(name='LTXVideo 0.9.0 I2V', - url='https://huggingface.co/a-r-r-o-w/LTX-Video-diffusers', - repo='a-r-r-o-w/LTX-Video-diffusers', - repo_cls=diffusers.LTXImageToVideoPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.LTXVideoTransformer3DModel), - ], - 'WAN Video': [ - Model(name='None'), - Model(name='WAN 2.2 5B T2V', - url='https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B-Diffusers', - repo='Wan-AI/Wan2.2-TI2V-5B-Diffusers', - repo_cls=diffusers.WanPipeline, - te_cls=transformers.UMT5EncoderModel, - dit_cls=diffusers.WanTransformer3DModel), - Model(name='WAN 2.2 5B I2V', - url='https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B-Diffusers', - repo='Wan-AI/Wan2.2-TI2V-5B-Diffusers', - repo_cls=diffusers.WanImageToVideoPipeline, - te_cls=transformers.UMT5EncoderModel, - dit_cls=diffusers.WanTransformer3DModel), - Model(name='WAN 2.2 A14B T2V', - url='https://huggingface.co/Wan-AI/Wan2.2-T2V-A14B-Diffusers', - repo='Wan-AI/Wan2.2-T2V-A14B-Diffusers', - repo_cls=diffusers.WanPipeline, - te_cls=transformers.UMT5EncoderModel, - dit_cls=diffusers.WanTransformer3DModel, - dit_folder=("transformer", "transformer_2")), - Model(name='WAN 2.2 A14B I2V', - url='https://huggingface.co/Wan-AI/Wan2.2-I2V-A14B-Diffusers', - repo='Wan-AI/Wan2.2-I2V-A14B-Diffusers', - repo_cls=diffusers.WanImageToVideoPipeline, - te_cls=transformers.UMT5EncoderModel, - dit_cls=diffusers.WanTransformer3DModel, - dit_folder=("transformer", "transformer_2")), - Model(name='WAN 2.2 14B VACE', - url='https://huggingface.co/linoyts/Wan2.2-VACE-Fun-14B-diffusers', - repo='linoyts/Wan2.2-VACE-Fun-14B-diffusers', - repo_cls=diffusers.WanVACEPipeline, - te_cls=transformers.UMT5EncoderModel, - dit_cls=diffusers.WanVACETransformer3DModel, - dit_folder=("transformer", "transformer_2")), - Model(name='WAN 2.1 1.3B T2V', - url='https://huggingface.co/Wan-AI/Wan2.1-T2V-1.3B-Diffusers', - repo='Wan-AI/Wan2.1-T2V-1.3B-Diffusers', - repo_cls=diffusers.WanPipeline, - te_cls=transformers.UMT5EncoderModel, - dit_cls=diffusers.WanTransformer3DModel), - Model(name='WAN 2.1 14B T2V', - url='https://huggingface.co/Wan-AI/Wan2.1-T2V-14B-Diffusers', - repo='Wan-AI/Wan2.1-T2V-14B-Diffusers', - repo_cls=diffusers.WanPipeline, - te_cls=transformers.UMT5EncoderModel, - dit_cls=diffusers.WanTransformer3DModel), - Model(name='WAN 2.1 14B I2V 480p', - url='https://huggingface.co/Wan-AI/Wan2.1-I2V-14B-480P-Diffusers', - repo='Wan-AI/Wan2.1-I2V-14B-480P-Diffusers', - repo_cls=diffusers.WanImageToVideoPipeline, - te_cls=transformers.UMT5EncoderModel, - dit_cls=diffusers.WanTransformer3DModel), - Model(name='WAN 2.1 14B I2V 720p', - url='https://huggingface.co/Wan-AI/Wan2.1-I2V-14B-720P-Diffusers', - repo='Wan-AI/Wan2.1-I2V-14B-720P-Diffusers', - repo_cls=diffusers.WanImageToVideoPipeline, - te_cls=transformers.UMT5EncoderModel, - dit_cls=diffusers.WanTransformer3DModel), - Model(name='WAN 2.1 14B FLF2V 720p', - url='https://huggingface.co/Wan-AI/Wan2.1-FLF2V-14B-720P', - repo='Wan-AI/Wan2.1-FLF2V-14B-720P-diffusers', - repo_cls=diffusers.WanImageToVideoPipeline, - te_cls=transformers.UMT5EncoderModel, - dit_cls=diffusers.WanTransformer3DModel), - Model(name='WAN 2.1 VACE 1.3B', - url='https://huggingface.co/Wan-AI/Wan2.1-VACE-1.3B-diffusers', - repo='Wan-AI/Wan2.1-VACE-1.3B-diffusers', - repo_cls=diffusers.WanVACEPipeline, - te_cls=transformers.UMT5EncoderModel, - dit_cls=diffusers.WanVACETransformer3DModel), - Model(name='WAN 2.1 VACE 14B', - url='https://huggingface.co/Wan-AI/Wan2.1-VACE-14B-diffusers', - repo='Wan-AI/Wan2.1-VACE-14B-diffusers', - repo_cls=diffusers.WanVACEPipeline, - te_cls=transformers.UMT5EncoderModel, - dit_cls=diffusers.WanVACETransformer3DModel), - ], - 'SkyReels V2': [ - Model(name='None'), - Model(name='SkyReels-V2 T2V-DF 1.3B-540P', - url='https://huggingface.co/Skywork/SkyReels-V2-DF-1.3B-540P-Diffusers', - repo='Skywork/SkyReels-V2-DF-1.3B-540P-Diffusers', - repo_cls=diffusers.SkyReelsV2DiffusionForcingPipeline, - repo_revision='refs/pr/1', - te_cls=transformers.UMT5EncoderModel, - dit_cls=diffusers.SkyReelsV2Transformer3DModel), - Model(name='SkyReels-V2 T2V-DF 14B-720P', - url='https://huggingface.co/Skywork/SkyReels-V2-DF-14B-720P-Diffusers', - repo='Skywork/SkyReels-V2-DF-14B-720P-Diffusers', - repo_cls=diffusers.SkyReelsV2DiffusionForcingPipeline, - repo_revision='refs/pr/1', - te_cls=transformers.UMT5EncoderModel, - dit_cls=diffusers.SkyReelsV2Transformer3DModel), - Model(name='SkyReels-V2 I2V-DF 14B-720P', - url='https://huggingface.co/Skywork/SkyReels-V2-DF-14B-720P-Diffusers', - repo='Skywork/SkyReels-V2-DF-14B-720P-Diffusers', - repo_cls=diffusers.SkyReelsV2DiffusionForcingImageToVideoPipeline, - repo_revision='refs/pr/1', - te_cls=transformers.UMT5EncoderModel, - dit_cls=diffusers.SkyReelsV2Transformer3DModel), - Model(name='SkyReels-V2 T2V 14B-720P', - url='https://huggingface.co/Skywork/SkyReels-V2-T2V-14B-720P-Diffusers', - repo='Skywork/SkyReels-V2-T2V-14B-720P-Diffusers', - repo_cls=diffusers.SkyReelsV2Pipeline, - repo_revision='refs/pr/1', - te_cls=transformers.UMT5EncoderModel, - dit_cls=diffusers.SkyReelsV2Transformer3DModel), - Model(name='SkyReels-V2 I2V 14B-720P', - url='https://huggingface.co/Skywork/SkyReels-V2-I2V-14B-720P-Diffusers', - repo='Skywork/SkyReels-V2-I2V-14B-720P-Diffusers', - repo_cls=diffusers.SkyReelsV2ImageToVideoPipeline, - repo_revision='refs/pr/1', - te_cls=transformers.UMT5EncoderModel, - dit_cls=diffusers.SkyReelsV2Transformer3DModel), - ], - 'Mochi Video': [ - Model(name='None'), - Model(name='Mochi 1 T2V', - url='https://huggingface.co/genmo/mochi-1-preview', - repo='genmo/mochi-1-preview', - repo_cls=diffusers.MochiPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.MochiTransformer3DModel), - ], - 'Latte Video': [ - Model(name='None'), - Model(name='Latte 1 T2V', - url='https://huggingface.co/maxin-cn/Latte-1', - repo='maxin-cn/Latte-1', - repo_cls=diffusers.LattePipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.LatteTransformer3DModel), - ], - 'Allegro Video': [ - Model(name='None'), - Model(name='Allegro T2V', - url='https://huggingface.co/rhymes-ai/Allegro', - repo='rhymes-ai/Allegro', - repo_cls=diffusers.AllegroPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.AllegroTransformer3DModel), - ], - 'Cog Video': [ - Model(name='None'), - Model(name='CogVideoX 1.0 2B T2V', - url='https://huggingface.co/THUDM/CogVideoX-2b', - repo='THUDM/CogVideoX-2b', - repo_cls=diffusers.CogVideoXPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.CogVideoXTransformer3DModel), - Model(name='CogVideoX 1.0 5B T2V', - url='https://huggingface.co/THUDM/CogVideoX-5b', - repo='THUDM/CogVideoX-5b', - repo_cls=diffusers.CogVideoXPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.CogVideoXTransformer3DModel), - Model(name='CogVideoX 1.0 5B I2V', - url='https://huggingface.co/THUDM/CogVideoX-5b-I2V', - repo='THUDM/CogVideoX-5b-I2V', - repo_cls=diffusers.CogVideoXImageToVideoPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.CogVideoXTransformer3DModel), - Model(name='CogVideoX 1.5 5B T2V', - url='https://huggingface.co/THUDM/CogVideoX1.5-5B', - repo='THUDM/CogVideoX1.5-5B', - repo_cls=diffusers.CogVideoXPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.CogVideoXTransformer3DModel), - Model(name='CogVideoX 1.5 5B I2V', - url='https://huggingface.co/THUDM/CogVideoX1.5-5B-I2V', - repo='THUDM/CogVideoX1.5-5B-I2V', - repo_cls=diffusers.CogVideoXImageToVideoPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.CogVideoXTransformer3DModel), - Model(name='Index Anisora 1.0 5B I2V', - url='https://huggingface.co/Disty0/Index-anisora-5B-diffusers', - repo='Disty0/Index-anisora-5B-diffusers', - repo_cls=diffusers.CogVideoXImageToVideoPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.CogVideoXTransformer3DModel), - Model(name='Index Anisora 1.0 5B RL I2V', - url='https://huggingface.co/Disty0/Index-anisora-5B_RL-diffusers', - repo='Disty0/Index-anisora-5B_RL-diffusers', - repo_cls=diffusers.CogVideoXImageToVideoPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.CogVideoXTransformer3DModel), - ], - 'nVidia Cosmos': [ - Model(name='nvidia Cosmos Predict2 2B I2V', - url='https://huggingface.co/nvidia/Cosmos-Predict2-2B-Text2Image', - repo='nvidia/Cosmos-Predict2-2B-Video2World', - repo_cls=diffusers.Cosmos2VideoToWorldPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.CosmosTransformer3DModel), - Model(name='nvidia Cosmos Predict2 2B I2V', - url='https://huggingface.co/nvidia/Cosmos-Predict2-2B-Text2Image', - repo='nvidia/Cosmos-Predict2-2B-Video2World', - repo_cls=diffusers.Cosmos2VideoToWorldPipeline, - te_cls=transformers.T5EncoderModel, - dit_cls=diffusers.CosmosTransformer3DModel), - ], - 'Kandinsky': [ - Model(name='Kandinsky 5.0 Lite SFT T2V', - url='https://huggingface.co/ai-forever/Kandinsky-5.0-T2V-Lite-sft-5s-Diffusers', - repo='ai-forever/Kandinsky-5.0-T2V-Lite-sft-5s-Diffusers', - repo_cls=diffusers.Kandinsky5T2VPipeline, - te_cls=transformers.Qwen2_5_VLForConditionalGeneration, - dit_cls=diffusers.Kandinsky5Transformer3DModel), - Model(name='Kandinsky 5.0 Lite CFG-distilled T2V', - url='https://huggingface.co/ai-forever/Kandinsky-5.0-T2V-Lite-nocfg-5s-Diffusers', - repo='ai-forever/Kandinsky-5.0-T2V-Lite-nocfg-5s-Diffusers', - repo_cls=diffusers.Kandinsky5T2VPipeline, - te_cls=transformers.Qwen2_5_VLForConditionalGeneration, - dit_cls=diffusers.Kandinsky5Transformer3DModel), - Model(name='Kandinsky 5.0 Lite Steps-distilled T2V', - url='https://huggingface.co/ai-forever/Kandinsky-5.0-T2V-Lite-distilled16steps-5s-Diffusers', - repo='ai-forever/Kandinsky-5.0-T2V-Lite-distilled16steps-5s-Diffusers', - repo_cls=diffusers.Kandinsky5T2VPipeline, - te_cls=transformers.Qwen2_5_VLForConditionalGeneration, - dit_cls=diffusers.Kandinsky5Transformer3DModel), - ], -} +try: + t0 = time.time() + models = { + 'None': [], + 'Hunyuan Video': [ + Model(name='None'), + Model(name='Hunyuan Video T2V', + url='https://huggingface.co/tencent/HunyuanVideo', + vae_remote=True, + repo='hunyuanvideo-community/HunyuanVideo', + repo_cls=diffusers.HunyuanVideoPipeline, + te_cls=transformers.LlamaModel, + dit_cls=diffusers.HunyuanVideoTransformer3DModel), + Model(name='Hunyuan Video I2V', # https://github.com/huggingface/diffusers/pull/10983 + url='https://huggingface.co/tencent/HunyuanVideo-I2V', + vae_remote=True, + repo='hunyuanvideo-community/HunyuanVideo-I2V', + repo_cls=diffusers.HunyuanVideoImageToVideoPipeline, + te_cls=transformers.LlavaForConditionalGeneration, + dit_cls=diffusers.HunyuanVideoTransformer3DModel), + Model(name='SkyReels Hunyuan T2V', # https://github.com/huggingface/diffusers/pull/10837 + url='https://huggingface.co/Skywork/SkyReels-V1-Hunyuan-T2V', + vae_remote=True, + repo='hunyuanvideo-community/HunyuanVideo', + repo_cls=diffusers.HunyuanVideoPipeline, + te_cls=transformers.LlamaModel, + dit='Skywork/SkyReels-V1-Hunyuan-T2V', + dit_folder=None, + dit_cls=diffusers.HunyuanVideoTransformer3DModel), + Model(name='SkyReels Hunyuan I2V', # https://github.com/huggingface/diffusers/pull/10837 + url='https://huggingface.co/Skywork/SkyReels-V1-Hunyuan-I2V', + vae_remote=True, + repo='hunyuanvideo-community/HunyuanVideo', + repo_cls=diffusers.HunyuanSkyreelsImageToVideoPipeline, + te_cls=transformers.LlamaModel, + dit='Skywork/SkyReels-V1-Hunyuan-I2V', + dit_folder=None, + dit_cls=diffusers.HunyuanVideoTransformer3DModel), + Model(name='Fast Hunyuan T2V', # https://github.com/hao-ai-lab/FastVideo/blob/8a77cf22c9b9e7f931f42bc4b35d21fd91d24e45/fastvideo/models/hunyuan/inference.py#L213 + url='https://huggingface.co/FastVideo/FastHunyuan', + vae_remote=True, + repo='hunyuanvideo-community/HunyuanVideo', + repo_cls=diffusers.HunyuanVideoPipeline, + te_cls=transformers.LlamaModel, + dit='FastVideo/FastHunyuan-diffusers', + dit_cls=diffusers.HunyuanVideoTransformer3DModel), + ], + 'LTX Video': [ + Model(name='None'), + Model(name='LTXVideo 0.9.8 13B', + url='https://huggingface.co/Lightricks/LTX-Video-0.9.8-13B-distilled', + repo='Lightricks/LTX-Video-0.9.8-13B-distilled', + repo_cls=diffusers.LTXConditionPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LTXVideoTransformer3DModel), + Model(name='LTXVideo 0.9.7 13B', + url='https://huggingface.co/Lightricks/LTX-Video-0.9.7-dev', + repo='a-r-r-o-w/LTX-Video-0.9.7-diffusers', + repo_cls=diffusers.LTXConditionPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LTXVideoTransformer3DModel), + Model(name='LTXVideo 0.9.6 2B T2V', + url='https://huggingface.co/Lightricks/LTX-Video', + repo='Lightricks/LTX-Video', + repo_cls=diffusers.LTXConditionPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LTXVideoTransformer3DModel), + Model(name='LTXVideo 0.9.6 2B I2V', + url='https://huggingface.co/Lightricks/LTX-Video', + repo='Lightricks/LTX-Video', + repo_cls=diffusers.LTXConditionPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LTXVideoTransformer3DModel), + Model(name='LTXVideo 0.9.6 2B T2V Distilled', + url='https://huggingface.co/Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25', + repo='Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25', + repo_cls=diffusers.LTXConditionPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LTXVideoTransformer3DModel), + Model(name='LTXVideo 0.9.6 2B I2V Distilled', + url='https://huggingface.co/Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25', + repo='Lightricks/LTX-Video-2B-0.9.6-Distilled-04-25', + repo_cls=diffusers.LTXConditionPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LTXVideoTransformer3DModel), + Model(name='LTXVideo 0.9.5 T2V', # https://github.com/huggingface/diffusers/pull/10968 + url='https://huggingface.co/Lightricks/LTX-Video-0.9.5', + repo='Lightricks/LTX-Video-0.9.5', + repo_cls=diffusers.LTXConditionPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LTXVideoTransformer3DModel), + Model(name='LTXVideo 0.9.5 I2V', + url='https://huggingface.co/Lightricks/LTX-Video-0.9.5', + repo='Lightricks/LTX-Video-0.9.5', + repo_cls=diffusers.LTXConditionPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LTXVideoTransformer3DModel), + Model(name='LTXVideo 0.9.1 T2V', + url='https://huggingface.co/a-r-r-o-w/LTX-Video-0.9.1-diffusers', + repo='a-r-r-o-w/LTX-Video-0.9.1-diffusers', + repo_cls=diffusers.LTXPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LTXVideoTransformer3DModel), + Model(name='LTXVideo 0.9.1 I2V', + url='https://huggingface.co/a-r-r-o-w/LTX-Video-0.9.1-diffusers', + repo='a-r-r-o-w/LTX-Video-0.9.1-diffusers', + repo_cls=diffusers.LTXImageToVideoPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LTXVideoTransformer3DModel), + Model(name='LTXVideo 0.9.0 T2V', + url='https://huggingface.co/a-r-r-o-w/LTX-Video-diffusers', + repo='a-r-r-o-w/LTX-Video-diffusers', + repo_cls=diffusers.LTXPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LTXVideoTransformer3DModel), + Model(name='LTXVideo 0.9.0 I2V', + url='https://huggingface.co/a-r-r-o-w/LTX-Video-diffusers', + repo='a-r-r-o-w/LTX-Video-diffusers', + repo_cls=diffusers.LTXImageToVideoPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LTXVideoTransformer3DModel), + ], + 'WAN Video': [ + Model(name='None'), + Model(name='WAN 2.2 5B T2V', + url='https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B-Diffusers', + repo='Wan-AI/Wan2.2-TI2V-5B-Diffusers', + repo_cls=diffusers.WanPipeline, + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.WanTransformer3DModel), + Model(name='WAN 2.2 5B I2V', + url='https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B-Diffusers', + repo='Wan-AI/Wan2.2-TI2V-5B-Diffusers', + repo_cls=diffusers.WanImageToVideoPipeline, + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.WanTransformer3DModel), + Model(name='WAN 2.2 A14B T2V', + url='https://huggingface.co/Wan-AI/Wan2.2-T2V-A14B-Diffusers', + repo='Wan-AI/Wan2.2-T2V-A14B-Diffusers', + repo_cls=diffusers.WanPipeline, + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.WanTransformer3DModel, + dit_folder=("transformer", "transformer_2")), + Model(name='WAN 2.2 A14B I2V', + url='https://huggingface.co/Wan-AI/Wan2.2-I2V-A14B-Diffusers', + repo='Wan-AI/Wan2.2-I2V-A14B-Diffusers', + repo_cls=diffusers.WanImageToVideoPipeline, + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.WanTransformer3DModel, + dit_folder=("transformer", "transformer_2")), + Model(name='WAN 2.2 14B VACE', + url='https://huggingface.co/linoyts/Wan2.2-VACE-Fun-14B-diffusers', + repo='linoyts/Wan2.2-VACE-Fun-14B-diffusers', + repo_cls=diffusers.WanVACEPipeline, + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.WanVACETransformer3DModel, + dit_folder=("transformer", "transformer_2")), + Model(name='WAN 2.1 1.3B T2V', + url='https://huggingface.co/Wan-AI/Wan2.1-T2V-1.3B-Diffusers', + repo='Wan-AI/Wan2.1-T2V-1.3B-Diffusers', + repo_cls=diffusers.WanPipeline, + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.WanTransformer3DModel), + Model(name='WAN 2.1 14B T2V', + url='https://huggingface.co/Wan-AI/Wan2.1-T2V-14B-Diffusers', + repo='Wan-AI/Wan2.1-T2V-14B-Diffusers', + repo_cls=diffusers.WanPipeline, + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.WanTransformer3DModel), + Model(name='WAN 2.1 14B I2V 480p', + url='https://huggingface.co/Wan-AI/Wan2.1-I2V-14B-480P-Diffusers', + repo='Wan-AI/Wan2.1-I2V-14B-480P-Diffusers', + repo_cls=diffusers.WanImageToVideoPipeline, + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.WanTransformer3DModel), + Model(name='WAN 2.1 14B I2V 720p', + url='https://huggingface.co/Wan-AI/Wan2.1-I2V-14B-720P-Diffusers', + repo='Wan-AI/Wan2.1-I2V-14B-720P-Diffusers', + repo_cls=diffusers.WanImageToVideoPipeline, + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.WanTransformer3DModel), + Model(name='WAN 2.1 14B FLF2V 720p', + url='https://huggingface.co/Wan-AI/Wan2.1-FLF2V-14B-720P', + repo='Wan-AI/Wan2.1-FLF2V-14B-720P-diffusers', + repo_cls=diffusers.WanImageToVideoPipeline, + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.WanTransformer3DModel), + Model(name='WAN 2.1 VACE 1.3B', + url='https://huggingface.co/Wan-AI/Wan2.1-VACE-1.3B-diffusers', + repo='Wan-AI/Wan2.1-VACE-1.3B-diffusers', + repo_cls=diffusers.WanVACEPipeline, + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.WanVACETransformer3DModel), + Model(name='WAN 2.1 VACE 14B', + url='https://huggingface.co/Wan-AI/Wan2.1-VACE-14B-diffusers', + repo='Wan-AI/Wan2.1-VACE-14B-diffusers', + repo_cls=diffusers.WanVACEPipeline, + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.WanVACETransformer3DModel), + ], + 'SkyReels V2': [ + Model(name='None'), + Model(name='SkyReels-V2 T2V-DF 1.3B-540P', + url='https://huggingface.co/Skywork/SkyReels-V2-DF-1.3B-540P-Diffusers', + repo='Skywork/SkyReels-V2-DF-1.3B-540P-Diffusers', + repo_cls=diffusers.SkyReelsV2DiffusionForcingPipeline, + repo_revision='refs/pr/1', + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.SkyReelsV2Transformer3DModel), + Model(name='SkyReels-V2 T2V-DF 14B-720P', + url='https://huggingface.co/Skywork/SkyReels-V2-DF-14B-720P-Diffusers', + repo='Skywork/SkyReels-V2-DF-14B-720P-Diffusers', + repo_cls=diffusers.SkyReelsV2DiffusionForcingPipeline, + repo_revision='refs/pr/1', + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.SkyReelsV2Transformer3DModel), + Model(name='SkyReels-V2 I2V-DF 14B-720P', + url='https://huggingface.co/Skywork/SkyReels-V2-DF-14B-720P-Diffusers', + repo='Skywork/SkyReels-V2-DF-14B-720P-Diffusers', + repo_cls=diffusers.SkyReelsV2DiffusionForcingImageToVideoPipeline, + repo_revision='refs/pr/1', + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.SkyReelsV2Transformer3DModel), + Model(name='SkyReels-V2 T2V 14B-720P', + url='https://huggingface.co/Skywork/SkyReels-V2-T2V-14B-720P-Diffusers', + repo='Skywork/SkyReels-V2-T2V-14B-720P-Diffusers', + repo_cls=diffusers.SkyReelsV2Pipeline, + repo_revision='refs/pr/1', + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.SkyReelsV2Transformer3DModel), + Model(name='SkyReels-V2 I2V 14B-720P', + url='https://huggingface.co/Skywork/SkyReels-V2-I2V-14B-720P-Diffusers', + repo='Skywork/SkyReels-V2-I2V-14B-720P-Diffusers', + repo_cls=diffusers.SkyReelsV2ImageToVideoPipeline, + repo_revision='refs/pr/1', + te_cls=transformers.UMT5EncoderModel, + dit_cls=diffusers.SkyReelsV2Transformer3DModel), + ], + 'Mochi Video': [ + Model(name='None'), + Model(name='Mochi 1 T2V', + url='https://huggingface.co/genmo/mochi-1-preview', + repo='genmo/mochi-1-preview', + repo_cls=diffusers.MochiPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.MochiTransformer3DModel), + ], + 'Latte Video': [ + Model(name='None'), + Model(name='Latte 1 T2V', + url='https://huggingface.co/maxin-cn/Latte-1', + repo='maxin-cn/Latte-1', + repo_cls=diffusers.LattePipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.LatteTransformer3DModel), + ], + 'Allegro Video': [ + Model(name='None'), + Model(name='Allegro T2V', + url='https://huggingface.co/rhymes-ai/Allegro', + repo='rhymes-ai/Allegro', + repo_cls=diffusers.AllegroPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.AllegroTransformer3DModel), + ], + 'Cog Video': [ + Model(name='None'), + Model(name='CogVideoX 1.0 2B T2V', + url='https://huggingface.co/THUDM/CogVideoX-2b', + repo='THUDM/CogVideoX-2b', + repo_cls=diffusers.CogVideoXPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.CogVideoXTransformer3DModel), + Model(name='CogVideoX 1.0 5B T2V', + url='https://huggingface.co/THUDM/CogVideoX-5b', + repo='THUDM/CogVideoX-5b', + repo_cls=diffusers.CogVideoXPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.CogVideoXTransformer3DModel), + Model(name='CogVideoX 1.0 5B I2V', + url='https://huggingface.co/THUDM/CogVideoX-5b-I2V', + repo='THUDM/CogVideoX-5b-I2V', + repo_cls=diffusers.CogVideoXImageToVideoPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.CogVideoXTransformer3DModel), + Model(name='CogVideoX 1.5 5B T2V', + url='https://huggingface.co/THUDM/CogVideoX1.5-5B', + repo='THUDM/CogVideoX1.5-5B', + repo_cls=diffusers.CogVideoXPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.CogVideoXTransformer3DModel), + Model(name='CogVideoX 1.5 5B I2V', + url='https://huggingface.co/THUDM/CogVideoX1.5-5B-I2V', + repo='THUDM/CogVideoX1.5-5B-I2V', + repo_cls=diffusers.CogVideoXImageToVideoPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.CogVideoXTransformer3DModel), + Model(name='Index Anisora 1.0 5B I2V', + url='https://huggingface.co/Disty0/Index-anisora-5B-diffusers', + repo='Disty0/Index-anisora-5B-diffusers', + repo_cls=diffusers.CogVideoXImageToVideoPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.CogVideoXTransformer3DModel), + Model(name='Index Anisora 1.0 5B RL I2V', + url='https://huggingface.co/Disty0/Index-anisora-5B_RL-diffusers', + repo='Disty0/Index-anisora-5B_RL-diffusers', + repo_cls=diffusers.CogVideoXImageToVideoPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.CogVideoXTransformer3DModel), + ], + 'nVidia Cosmos': [ + Model(name='nvidia Cosmos Predict2 2B I2V', + url='https://huggingface.co/nvidia/Cosmos-Predict2-2B-Text2Image', + repo='nvidia/Cosmos-Predict2-2B-Video2World', + repo_cls=diffusers.Cosmos2VideoToWorldPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.CosmosTransformer3DModel), + Model(name='nvidia Cosmos Predict2 2B I2V', + url='https://huggingface.co/nvidia/Cosmos-Predict2-2B-Text2Image', + repo='nvidia/Cosmos-Predict2-2B-Video2World', + repo_cls=diffusers.Cosmos2VideoToWorldPipeline, + te_cls=transformers.T5EncoderModel, + dit_cls=diffusers.CosmosTransformer3DModel), + ], + 'Kandinsky': [ + Model(name='Kandinsky 5.0 Lite SFT T2V', + url='https://huggingface.co/ai-forever/Kandinsky-5.0-T2V-Lite-sft-5s-Diffusers', + repo='ai-forever/Kandinsky-5.0-T2V-Lite-sft-5s-Diffusers', + repo_cls=diffusers.Kandinsky5T2VPipeline, + te_cls=transformers.Qwen2_5_VLForConditionalGeneration, + dit_cls=diffusers.Kandinsky5Transformer3DModel), + Model(name='Kandinsky 5.0 Lite CFG-distilled T2V', + url='https://huggingface.co/ai-forever/Kandinsky-5.0-T2V-Lite-nocfg-5s-Diffusers', + repo='ai-forever/Kandinsky-5.0-T2V-Lite-nocfg-5s-Diffusers', + repo_cls=diffusers.Kandinsky5T2VPipeline, + te_cls=transformers.Qwen2_5_VLForConditionalGeneration, + dit_cls=diffusers.Kandinsky5Transformer3DModel), + Model(name='Kandinsky 5.0 Lite Steps-distilled T2V', + url='https://huggingface.co/ai-forever/Kandinsky-5.0-T2V-Lite-distilled16steps-5s-Diffusers', + repo='ai-forever/Kandinsky-5.0-T2V-Lite-distilled16steps-5s-Diffusers', + repo_cls=diffusers.Kandinsky5T2VPipeline, + te_cls=transformers.Qwen2_5_VLForConditionalGeneration, + dit_cls=diffusers.Kandinsky5Transformer3DModel), + ], + } + t1 = time.time() + total = sum([len(models[model]) for model in models.keys()]) + log.info(f'Networks: type="video" engines={len(models)} models={total} time={t1 - t0:.2f}') +except Exception as e: + models = {} + log.error(f'Networks: type="video" {e}') diff --git a/pipelines/generic.py b/pipelines/generic.py index e78ac7b8b..430ee34bc 100644 --- a/pipelines/generic.py +++ b/pipelines/generic.py @@ -12,6 +12,8 @@ def load_transformer(repo_id, cls_name, load_config={}, subfolder="transformer", transformer = None jobid = shared.state.begin('Load DiT') try: + if 'sdnq-' in repo_id.lower(): + from modules import sdnq # pylint: disable=unused-import # register to diffusers and transformers load_args, quant_args = model_quant.get_dit_args(load_config, module='Model', device_map=True, allow_quant=allow_quant, modules_to_not_convert=modules_to_not_convert, modules_dtype_dict=modules_dtype_dict) quant_type = model_quant.get_quant_type(quant_args) dtype = dtype or devices.dtype @@ -49,6 +51,8 @@ def load_transformer(repo_id, cls_name, load_config={}, subfolder="transformer", ) else: shared.log.debug(f'Load model: transformer="{repo_id}" cls={cls_name.__name__} subfolder={subfolder} quant="{quant_type}" args={load_args}') + if 'sdnq-' in repo_id.lower(): + quant_args = {} if dtype is not None: load_args['torch_dtype'] = dtype if subfolder is not None: @@ -81,6 +85,8 @@ def load_text_encoder(repo_id, cls_name, load_config={}, subfolder="text_encoder text_encoder = None jobid = shared.state.begin('Load TE') try: + if 'sdnq-' in repo_id.lower(): + from modules import sdnq # pylint: disable=unused-import # register to diffusers and transformers load_args, quant_args = model_quant.get_dit_args(load_config, module='TE', device_map=True, allow_quant=allow_quant, modules_to_not_convert=modules_to_not_convert, modules_dtype_dict=modules_dtype_dict) quant_type = model_quant.get_quant_type(quant_args) dtype = dtype or devices.dtype @@ -112,27 +118,31 @@ def load_text_encoder(repo_id, cls_name, load_config={}, subfolder="text_encoder text_encoder = model_quant.do_post_load_quant(text_encoder, allow=quant_type is not None) # load from local file safetensors elif local_file is not None and local_file.lower().endswith('.safetensors'): - shared.log.debug(f'Load model: text_encoder="{local_file}" cls={cls_name.__name__} quant="{quant_type}"') + shared.log.debug(f'Load model: text_encoder="{local_file}" cls={cls_name.__name__} quant="{quant_type}" args={load_args}') from modules import model_te text_encoder = model_te.load_t5(local_file) text_encoder = model_quant.do_post_load_quant(text_encoder, allow=quant_type is not None) # use shared t5 if possible elif cls_name == transformers.T5EncoderModel and allow_shared: - with open(os.path.join('configs', 'flux', 'text_encoder_2', 'config.json'), encoding='utf8') as f: - load_args['config'] = transformers.T5Config(**json.load(f)) if model_quant.check_nunchaku('TE'): import nunchaku repo_id = 'nunchaku-tech/nunchaku-t5/awq-int4-flux.1-t5xxl.safetensors' cls_name = nunchaku.NunchakuT5EncoderModel - shared.log.debug(f'Load model: text_encoder="{repo_id}" cls={cls_name.__name__} quant="SVDQuant"') + shared.log.debug(f'Load model: text_encoder="{repo_id}" cls={cls_name.__name__} quant="SVDQuant" args={load_args}') text_encoder = nunchaku.NunchakuT5EncoderModel.from_pretrained( repo_id, torch_dtype=dtype, ) text_encoder.quantization_method = 'SVDQuant' elif shared.opts.te_shared_t5: - repo_id = 'Disty0/t5-xxl' - shared.log.debug(f'Load model: text_encoder="{repo_id}" cls={cls_name.__name__} quant="{quant_type}" shared={shared.opts.te_shared_t5}') + if 'sdnq-uint4-svd' in repo_id.lower(): + repo_id = 'Disty0/FLUX.1-dev-SDNQ-uint4-svd-r32' + load_args['subfolder'] = 'text_encoder_2' + else: + repo_id = 'Disty0/t5-xxl' + with open(os.path.join('configs', 'flux', 'text_encoder_2', 'config.json'), encoding='utf8') as f: + load_args['config'] = transformers.T5Config(**json.load(f)) + shared.log.debug(f'Load model: text_encoder="{repo_id}" cls={cls_name.__name__} quant="{quant_type}" shared={shared.opts.te_shared_t5} args={load_args}') if dtype is not None: load_args['torch_dtype'] = dtype text_encoder = cls_name.from_pretrained( @@ -145,7 +155,7 @@ def load_text_encoder(repo_id, cls_name, load_config={}, subfolder="text_encoder if shared.opts.te_shared_t5: repo_id = 'hunyuanvideo-community/HunyuanImage-2.1-Diffusers' subfolder = 'text_encoder' - shared.log.debug(f'Load model: text_encoder="{repo_id}" cls={cls_name.__name__} quant="{quant_type}" shared={shared.opts.te_shared_t5}') + shared.log.debug(f'Load model: text_encoder="{repo_id}" cls={cls_name.__name__} quant="{quant_type}" shared={shared.opts.te_shared_t5} args={load_args}') if dtype is not None: load_args['torch_dtype'] = dtype text_encoder = cls_name.from_pretrained( @@ -158,7 +168,7 @@ def load_text_encoder(repo_id, cls_name, load_config={}, subfolder="text_encoder # load from repo if text_encoder is None: - shared.log.debug(f'Load model: text_encoder="{repo_id}" cls={cls_name.__name__} quant="{quant_type}" shared={shared.opts.te_shared_t5}') + shared.log.debug(f'Load model: text_encoder="{repo_id}" cls={cls_name.__name__} quant="{quant_type}" shared={shared.opts.te_shared_t5} args={load_args}') if dtype is not None: load_args['torch_dtype'] = dtype if subfolder is not None: diff --git a/requirements.txt b/requirements.txt index 66803df33..e29873aef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,6 +28,7 @@ fasteners orjson sqlalchemy invisible-watermark +PyWavelets pi-heif # versioned @@ -41,17 +42,15 @@ torchsde==0.2.6 antlr4-python3-runtime==4.9.3 requests==2.32.4 tqdm==4.67.1 -accelerate==1.10.1 -opencv-contrib-python-headless==4.11.0.86 +accelerate==1.11.0 einops==0.8.1 -huggingface_hub==0.35.3 +huggingface_hub==0.36.0 numexpr==2.11.0 numpy==2.1.2 pandas==2.3.1 numba==0.61.2 protobuf==4.25.3 pytorch_lightning==2.5.5 -PyWavelets==1.9.0 urllib3==1.26.19 Pillow==10.4.0 timm==1.0.16