diff --git a/CHANGELOG.md b/CHANGELOG.md index 90c1918c1..e799e57ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ Major refactor of [FLUX.1](https://blackforestlabs.ai/announcing-black-forest-la - Since both *Optimum-Quanto* and *BitsAndBytes* libraries are limited in their platform support matrix, try enabling **NNCF** for quantization/compression on-the-fly! +And support for [CogVideoX](https://huggingface.co/THUDM/CogVideoX-5b) text to video! + Oh, as a sidenote, and also new auto **HDR** image create for SD and SDXL ;) ### Details @@ -44,6 +46,10 @@ Oh, as a sidenote, and also new auto **HDR** image create for SD and SDXL ;) enable via *settings -> compute -> fused projections* **Other improvements:** +- [CogVideoX](https://huggingface.co/THUDM/CogVideoX-5b) + - support for both **2B** and **5B** variations + - simply select in scripts -> cogvideox + - as with any video modules, includes additional frame interpolation using RIFE - **HDR** high-dynamic-range image create for SD and SDXL create hdr images from in multiple exposures by latent-space modifications during generation use via *scripts -> hdr* diff --git a/modules/api/control.py b/modules/api/control.py index 9c93b6bc6..cf8916095 100644 --- a/modules/api/control.py +++ b/modules/api/control.py @@ -103,6 +103,7 @@ class APIControl(): args['ip_adapter_scales'].append(ipadapter.scale) args['ip_adapter_starts'].append(ipadapter.start) args['ip_adapter_ends'].append(ipadapter.end) + args['ip_adapter_crops'].append(ipadapter.end) args['ip_adapter_images'].append([helpers.decode_base64_to_image(x) for x in ipadapter.images]) if ipadapter.masks: args['ip_adapter_masks'].append([helpers.decode_base64_to_image(x) for x in ipadapter.masks]) diff --git a/modules/api/models.py b/modules/api/models.py index 4aba4fb01..3c5fd6146 100644 --- a/modules/api/models.py +++ b/modules/api/models.py @@ -149,12 +149,13 @@ class ItemEmbedding(BaseModel): vectors: int = Field(title="Vectors", description="The number of vectors in the embedding") class ItemIPAdapter(BaseModel): - adapter: str = Field(title="Adapter", default="Base", description="") - images: List[str] = Field(title="Image", default=[], description="") - masks: Optional[List[str]] = Field(title="Mask", default=[], description="") - scale: float = Field(title="Scale", default=0.5, ge=0, le=1, description="") - start: float = Field(title="Start", default=0.0, ge=0, le=1, description="") - end: float = Field(title="End", default=1.0, gt=0, le=1, description="") + adapter: str = Field(title="Adapter", default="Base", description="IP adapter name") + images: List[str] = Field(title="Image", default=[], description="IP adapter input images") + masks: Optional[List[str]] = Field(title="Mask", default=[], description="IP adapter mask images") + scale: float = Field(title="Scale", default=0.5, ge=0, le=1, description="IP adapter scale") + start: float = Field(title="Start", default=0.0, ge=0, le=1, description="IP adapter start step") + end: float = Field(title="End", default=1.0, gt=0, le=1, description="IP adapter end step") + crop: bool = Field(title="Crop", default=False, description="IP adapter crop face from input") class ItemFace(BaseModel): mode: str = Field(title="Mode", default="FaceID", description="The mode to use (available values: FaceID, FaceSwap, PhotoMaker, InstantID).") diff --git a/scripts/cogvideo.py b/scripts/cogvideo.py index fb5cbdce3..71fab47b9 100644 --- a/scripts/cogvideo.py +++ b/scripts/cogvideo.py @@ -3,6 +3,7 @@ models: https://huggingface.co/THUDM/CogVideoX-2b https://huggingface.co/THUDM/C source: https://github.com/THUDM/CogVideo quanto: https://gist.github.com/a-r-r-o-w/31be62828b00a9292821b85c1017effa torchao: https://gist.github.com/a-r-r-o-w/4d9732d17412888c885480c6521a9897 +venhancer: https://github.com/THUDM/CogVideo/blob/dcb82ae30b454ab898aeced0633172d75dbd55b8/tools/venhancer/README.md """ import os import time @@ -96,6 +97,18 @@ class Script(scripts.Script): shared.sd_model.vae.enable_slicing() shared.sd_model.vae.enable_tiling() + def prepare(self, p, video): + import imageio # TODO dont use imageio + from torchvision import transforms + reader = imageio.get_reader(video, "ffmpeg") + frames = [transforms.ToTensor()(frame) for frame in reader] + frames = [transforms.Resize((p.height, p.width))(frame) for frame in frames] + frames = frames[:p.frames] # TODO drop interim frames instead of cropping the list + reader.close() + tensor = torch.stack(frames).to(devices.device).permute(1, 0, 2, 3).unsqueeze(0).to(devices.dtype) + encoded = shared.sd_model.vae.encode(tensor)[0].sample() + return encoded + def generate(self, p: processing.StableDiffusionProcessing): if shared.sd_model_type != 'cogvideox': return [] @@ -124,10 +137,10 @@ class Script(scripts.Script): ) if getattr(p, 'image', False): raise ValueError('CogVideoX: image not supported') - # args['latents'] = [p.image] + # args['latents'] = self.prepare(p, [p.image]) elif getattr(p, 'video', False): raise ValueError('CogVideoX: video not supported') - # args['video'] = p.video + # args['video'] = self.prepare(p, p.video) else: args['num_frames'] = p.frames # only txt2vid has num_frames if debug: