update changelog

CHANGELOG.md

@@ -10,6 +10,8 @@ Major refactor of [FLUX.1](https://blackforestlabs.ai/announcing-black-forest-la
 - Since both *Optimum-Quanto* and *BitsAndBytes* libraries are limited in their platform support matrix,  
   try enabling **NNCF** for quantization/compression on-the-fly!
 
+And support for [CogVideoX](https://huggingface.co/THUDM/CogVideoX-5b) text to video!
+
 Oh, as a sidenote, and also new auto **HDR** image create for SD and SDXL ;)
 
 ### Details
@@ -44,6 +46,10 @@ Oh, as a sidenote, and also new auto **HDR** image create for SD and SDXL ;)
   enable via *settings -> compute -> fused projections*  
 
 **Other improvements:**
+- [CogVideoX](https://huggingface.co/THUDM/CogVideoX-5b)  
+  - support for both **2B** and **5B** variations  
+  - simply select in scripts -> cogvideox  
+  - as with any video modules, includes additional frame interpolation using RIFE  
 - **HDR** high-dynamic-range image create for SD and SDXL  
   create hdr images from in multiple exposures by latent-space modifications during generation  
   use via *scripts -> hdr*  

modules/api/control.py

@@ -103,6 +103,7 @@ class APIControl():
                 args['ip_adapter_scales'].append(ipadapter.scale)
                 args['ip_adapter_starts'].append(ipadapter.start)
                 args['ip_adapter_ends'].append(ipadapter.end)
+                args['ip_adapter_crops'].append(ipadapter.end)
                 args['ip_adapter_images'].append([helpers.decode_base64_to_image(x) for x in ipadapter.images])
                 if ipadapter.masks:
                     args['ip_adapter_masks'].append([helpers.decode_base64_to_image(x) for x in ipadapter.masks])

modules/api/models.py

@@ -149,12 +149,13 @@ class ItemEmbedding(BaseModel):
     vectors: int = Field(title="Vectors", description="The number of vectors in the embedding")
 
 class ItemIPAdapter(BaseModel):
-    adapter: str = Field(title="Adapter", default="Base", description="")
-    images: List[str] = Field(title="Image", default=[], description="")
-    masks: Optional[List[str]] = Field(title="Mask", default=[], description="")
-    scale: float = Field(title="Scale", default=0.5, ge=0, le=1, description="")
-    start: float = Field(title="Start", default=0.0, ge=0, le=1, description="")
-    end: float = Field(title="End", default=1.0, gt=0, le=1, description="")
+    adapter: str = Field(title="Adapter", default="Base", description="IP adapter name")
+    images: List[str] = Field(title="Image", default=[], description="IP adapter input images")
+    masks: Optional[List[str]] = Field(title="Mask", default=[], description="IP adapter mask images")
+    scale: float = Field(title="Scale", default=0.5, ge=0, le=1, description="IP adapter scale")
+    start: float = Field(title="Start", default=0.0, ge=0, le=1, description="IP adapter start step")
+    end: float = Field(title="End", default=1.0, gt=0, le=1, description="IP adapter end step")
+    crop: bool = Field(title="Crop", default=False, description="IP adapter crop face from input")
 
 class ItemFace(BaseModel):
     mode: str = Field(title="Mode", default="FaceID", description="The mode to use (available values: FaceID, FaceSwap, PhotoMaker, InstantID).")

scripts/cogvideo.py

@@ -3,6 +3,7 @@ models: https://huggingface.co/THUDM/CogVideoX-2b https://huggingface.co/THUDM/C
 source: https://github.com/THUDM/CogVideo
 quanto: https://gist.github.com/a-r-r-o-w/31be62828b00a9292821b85c1017effa
 torchao: https://gist.github.com/a-r-r-o-w/4d9732d17412888c885480c6521a9897
+venhancer: https://github.com/THUDM/CogVideo/blob/dcb82ae30b454ab898aeced0633172d75dbd55b8/tools/venhancer/README.md
 """
 import os
 import time
@@ -96,6 +97,18 @@ class Script(scripts.Script):
         shared.sd_model.vae.enable_slicing()
         shared.sd_model.vae.enable_tiling()
 
+    def prepare(self, p, video):
+        import imageio # TODO dont use imageio
+        from torchvision import transforms
+        reader = imageio.get_reader(video, "ffmpeg")
+        frames = [transforms.ToTensor()(frame) for frame in reader]
+        frames = [transforms.Resize((p.height, p.width))(frame) for frame in frames]
+        frames = frames[:p.frames] # TODO drop interim frames instead of cropping the list
+        reader.close()
+        tensor = torch.stack(frames).to(devices.device).permute(1, 0, 2, 3).unsqueeze(0).to(devices.dtype)
+        encoded = shared.sd_model.vae.encode(tensor)[0].sample()
+        return encoded
+
     def generate(self, p: processing.StableDiffusionProcessing):
         if shared.sd_model_type != 'cogvideox':
             return []
@@ -124,10 +137,10 @@ class Script(scripts.Script):
             )
             if getattr(p, 'image', False):
                 raise ValueError('CogVideoX: image not supported')
-                # args['latents'] = [p.image]
+                # args['latents'] = self.prepare(p, [p.image])
             elif getattr(p, 'video', False):
                 raise ValueError('CogVideoX: video not supported')
-                # args['video'] = p.video
+                # args['video'] = self.prepare(p, p.video)
             else:
                 args['num_frames'] = p.frames # only txt2vid has num_frames
             if debug: