Merge pull request #3940 from vladmandic/dev

merge dev

CHANGELOG.md

@@ -1,6 +1,6 @@
 # Change Log for SD.Next
 
-## Update for 2025-05-15
+## Update for 2025-05-17
 
 *Curious how your system is performing?*  
 Run a built-in benchmark and compare to over 15k unique results world-wide: [Benchmark data](https://vladmandic.github.io/sd-extension-system-info/pages/benchmark.html)!  
@@ -15,6 +15,11 @@ Btw, last few releases have been smaller, but more regular so do check posts abo
 
 - **Wiki**  
   - Updates for: *Quantization, NNCF, WSL, ZLUDA, ROCm*  
+- **Models**  
+  - [Index AniSora v1 5B](https://huggingface.co/IndexTeam/Index-anisora) I2V  
+    Based on CogVideoX architecture, trained as animated video generation model: This Project presenting Bilibili's gift to the anime world!  
+  - [Index AniSora v1 RL 5B](https://github.com/bilibili/Index-anisora?tab=readme-ov-file#anisorav10_rl) I2V  
+    RL-optimized AniSoraV1.0 for enhanced anime-style output  
 - **Compute**  
   - ZLUDA: update to `zluda==3.9.5` with `torch==2.7.0`  
     *Note*: delete `.zluda` folder so that newest zluda will be installed if you are using the latest AMD Adrenaline driver  
@@ -27,6 +32,9 @@ Btw, last few releases have been smaller, but more regular so do check posts abo
   - Pydantic: update to api types  
   - UI defaults: match correct prompt components  
   - NNCF with ControlNet  
+  - NNCF with CogVideo
+  - IPEX with CogVideo  
+  - JXL image format metadata handling  
 
 ## Update for 2025-05-12
 

configs/Dockerfile.cuda

@@ -2,7 +2,7 @@
 # docs: <https://github.com/vladmandic/sdnext/wiki/Docker>
 
 # base image
-FROM pytorch/pytorch:2.6.0-cuda12.6-cudnn9-runtime
+FROM pytorch/pytorch:2.7.0-cuda12.8-cudnn9-runtime
 
 # metadata
 LABEL org.opencontainers.image.vendor="SD.Next"
@@ -13,7 +13,7 @@ LABEL org.opencontainers.image.source="https://github.com/vladmandic/sdnext/"
 LABEL org.opencontainers.image.licenses="AGPL-3.0"
 LABEL org.opencontainers.image.title="SD.Next"
 LABEL org.opencontainers.image.description="SD.Next: Advanced Implementation of Stable Diffusion and other Diffusion-based generative image models"
-LABEL org.opencontainers.image.base.name="https://hub.docker.com/pytorch/pytorch:2.6.0-cuda12.6-cudnn9-runtime"
+LABEL org.opencontainers.image.base.name="https://hub.docker.com/pytorch/pytorch:2.7.0-cuda12.8-cudnn9-runtime"
 LABEL org.opencontainers.image.version="latest"
 
 # minimum install

installer.py

@@ -794,7 +794,7 @@ def install_torch_addons():
     if opts.get('torchao_quantization', False):
         install('torchao==0.10.0', 'torchao')
     if opts.get('samples_format', 'jpg') == 'jxl' or opts.get('grid_format', 'jpg') == 'jxl':
-        install('pillow-jxl-plugin==1.3.2', 'pillow-jxl-plugin')
+        install('pillow-jxl-plugin==1.3.3', 'pillow-jxl-plugin')
     if not args.experimental:
         uninstall('wandb', quiet=True)
     ts('addons', t_start)
@@ -1164,7 +1164,7 @@ def install_optional():
     install('basicsr')
     install('gfpgan')
     install('clean-fid')
-    install('pillow-jxl-plugin==1.3.2', ignore=True)
+    install('pillow-jxl-plugin==1.3.3', ignore=True)
     install('optimum-quanto==0.2.7', ignore=True)
     install('torchao==0.10.0', ignore=True)
     install('bitsandbytes==0.45.5', ignore=True)

modules/model_quant_nncf.py

@@ -228,6 +228,12 @@ class NNCFQuantizer(DiffusersQuantizer):
         layer, tensor_name = get_module_from_name(model, param_name)
         layer._parameters[tensor_name] = torch.nn.Parameter(param_value).to(device=target_device) # pylint: disable=protected-access
 
+        # nncf_padding_value somehow ends up in the meta device with cogvideo even if we don't use init_empty_weights
+        # set it to the default value if it is in the meta device:
+        if layer.__class__.__name__ == "NNCFConv2d" and hasattr(layer, "get_padding_value_ref") and hasattr(layer, "_set_padding_value"):
+            if layer.get_padding_value_ref().device == torch.device("meta"):
+                layer._set_padding_value(torch.zeros([1]))
+
         split_param_name = param_name.split(".")
         if param_name not in self.modules_to_not_convert and not any(param in split_param_name for param in self.modules_to_not_convert):
             layer = nncf_compress_layer(

modules/ui_video.py

@@ -71,6 +71,8 @@ def run_video(*args):
         return video_run.generate(*args)
     elif selected and 'Latte' in selected.name:
         return video_run.generate(*args)
+    elif selected and 'anisora' in selected.name.lower():
+        return video_run.generate(*args)
     return video_utils.queue_err(f'model not found: engine="{engine}" model="{model}"')
 
 

modules/video_models/models_def.py

@@ -224,5 +224,17 @@ models = {
               repo_cls=diffusers.CogVideoXImageToVideoPipeline,
               te_cls=transformers.T5EncoderModel,
               dit_cls=diffusers.CogVideoXTransformer3DModel),
+        Model(name='Index Anisora 1.0 5B I2V',
+              url='https://huggingface.co/Disty0/Index-anisora-5B-diffusers',
+              repo='Disty0/Index-anisora-5B-diffusers',
+              repo_cls=diffusers.CogVideoXImageToVideoPipeline,
+              te_cls=transformers.T5EncoderModel,
+              dit_cls=diffusers.CogVideoXTransformer3DModel),
+        Model(name='Index Anisora 1.0 5B RL I2V',
+              url='https://huggingface.co/Disty0/Index-anisora-5B_RL-diffusers',
+              repo='Disty0/Index-anisora-5B_RL-diffusers',
+              repo_cls=diffusers.CogVideoXImageToVideoPipeline,
+              te_cls=transformers.T5EncoderModel,
+              dit_cls=diffusers.CogVideoXTransformer3DModel),
     ],
 }