diff --git a/expose.h b/expose.h index fcf0e4073..39d4a4a8e 100644 --- a/expose.h +++ b/expose.h @@ -201,6 +201,7 @@ struct sd_load_model_inputs const char * clip1_filename = nullptr; const char * clip2_filename = nullptr; const char * vae_filename = nullptr; + const char * audio_vae_filename = nullptr; const int lora_len = 0; const char ** lora_filenames = nullptr; const float * lora_multipliers = nullptr; diff --git a/koboldcpp.py b/koboldcpp.py index d8f50c8b1..ea21ab1e0 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -385,6 +385,7 @@ class sd_load_model_inputs(ctypes.Structure): ("clip1_filename", ctypes.c_char_p), ("clip2_filename", ctypes.c_char_p), ("vae_filename", ctypes.c_char_p), + ("audio_vae_filename", ctypes.c_char_p), ("lora_len", ctypes.c_int), ("lora_filenames", ctypes.POINTER(ctypes.c_char_p)), ("lora_multipliers", ctypes.POINTER(ctypes.c_float)), @@ -2443,7 +2444,7 @@ def sd_resolve_device(name, default_=-1): name = str(max(name, -2)) return sd_get_device_number(name) -def sd_load_model(model_filename,vae_filename,t5xxl_filename,clip1_filename,clip2_filename,photomaker_filename,upscaler_filename): +def sd_load_model(model_filename,vae_filename,t5xxl_filename,clip1_filename,clip2_filename,photomaker_filename,upscaler_filename,audio_vae_filename): global args inputs = sd_load_model_inputs() inputs.model_filename = model_filename.encode("UTF-8") @@ -2467,6 +2468,7 @@ def sd_load_model(model_filename,vae_filename,t5xxl_filename,clip1_filename,clip inputs.taesd = True if args.sdvaeauto else False inputs.tiled_vae_threshold = args.sdtiledvae inputs.vae_filename = vae_filename.encode("UTF-8") + inputs.audio_vae_filename = audio_vae_filename.encode("UTF-8") inputs.t5xxl_filename = t5xxl_filename.encode("UTF-8") inputs.clip1_filename = clip1_filename.encode("UTF-8") inputs.clip2_filename = clip2_filename.encode("UTF-8") @@ -7676,6 +7678,7 @@ def show_gui(): sd_lora_var = ctk.StringVar() sd_loramult_var = ctk.StringVar(value="1.0") sd_vae_var = ctk.StringVar() + sd_audio_vae_var = ctk.StringVar() sd_t5xxl_var = ctk.StringVar() sd_clip1_var = ctk.StringVar() sd_clip2_var = ctk.StringVar() @@ -8602,6 +8605,8 @@ def show_gui(): sdvaeitem1,sdvaeitem2,sdvaeitem3 = makefileentry(images_tab, "Image VAE:", "Select Optional SD VAE file",sd_vae_var, 40, width=280, singlerow=True, filetypes=[("*.safetensors *.gguf", "*.safetensors *.gguf")],tooltiptxt="Select a .safetensors or .gguf SD VAE file to be loaded.") + makefileentry(images_tab, "Audio VAE:", "Select Audio VAE file for LTX video gen",sd_audio_vae_var, 42, width=280, singlerow=True, filetypes=[("*.safetensors *.gguf","*.safetensors *.gguf")],tooltiptxt="Select an Audio VAE file for LTX video generation") + def toggletaesd(a,b,c): if sd_vaeauto_var.get()==1: sdvaeitem1.grid_remove() @@ -8612,10 +8617,10 @@ def show_gui(): sdvaeitem1.grid() sdvaeitem2.grid() sdvaeitem3.grid() - makecheckbox(images_tab, "Automatic VAE (TAE SD)", sd_vaeauto_var, 42,command=toggletaesd,tooltiptxt="Replace VAE with TAESD. May fix bad VAE.") - makelabelcombobox(images_tab, "Conv2D Direct:", sd_convdirect_var, row=42, labelpadx=(220), padx=(310), width=90, tooltiptxt="Use Conv2D Direct operation. May save memory or improve performance.\nMight crash if not supported by the backend.\n", values=sd_convdirect_choices) - makelabelentry(images_tab, "VAE Tiling Threshold:", sd_tiled_vae_var, 44, 50, padx=(144),singleline=True,tooltip="Enable VAE Tiling for images above this size, to save memory.\nSet to 0 to disable VAE tiling.") - makecheckbox(images_tab, "SD Flash Attention", sd_flash_attention_var, 44,padx=(230), tooltiptxt="Enable Flash Attention for image diffusion. May save memory or improve performance.") + makecheckbox(images_tab, "Automatic VAE (TAE SD)", sd_vaeauto_var, 44,command=toggletaesd,tooltiptxt="Replace VAE with TAESD. May fix bad VAE.") + makelabelcombobox(images_tab, "Conv2D Direct:", sd_convdirect_var, row=44, labelpadx=(220), padx=(310), width=90, tooltiptxt="Use Conv2D Direct operation. May save memory or improve performance.\nMight crash if not supported by the backend.\n", values=sd_convdirect_choices) + makelabelentry(images_tab, "VAE Tiling Threshold:", sd_tiled_vae_var, 46, 50, padx=(144),singleline=True,tooltip="Enable VAE Tiling for images above this size, to save memory.\nSet to 0 to disable VAE tiling.") + makecheckbox(images_tab, "SD Flash Attention", sd_flash_attention_var, 46,padx=(230), tooltiptxt="Enable Flash Attention for image diffusion. May save memory or improve performance.") makecheckbox(images_tab, "Model Offload", sd_offload_cpu_var, 50,padx=8, tooltiptxt="Offload image weights in RAM to save VRAM, swap into VRAM when needed.") makelabelcombobox(images_tab, "VAE dev:", sd_vae_device_var, 50,labelpadx=(140),padx=(200), width=70, tooltiptxt="Change VAE device for image generation.", values=sd_device_choices) makelabelcombobox(images_tab, "CLIP dev:", sd_clip_device_var, 50,labelpadx=(280),padx=340, width=70, tooltiptxt="Change CLIP / T5 / LLM device for image generation.", values=sd_device_choices) @@ -8961,6 +8966,7 @@ def show_gui(): args.sdvae = "" if sd_vae_var.get() != "": args.sdvae = sd_vae_var.get() + args.sdaudiovae = sd_audio_vae_var.get() if sd_audio_vae_var.get() != "" else "" args.sdconvdirect = sd_convdirect_option(sd_convdirect_var.get()) args.sdt5xxl = sd_t5xxl_var.get() if sd_t5xxl_var.get() != "" else "" args.sdclip1 = sd_clip1_var.get() if sd_clip1_var.get() != "" else "" @@ -9254,6 +9260,7 @@ def show_gui(): sd_clip_device_var.set(sd_get_device_name(sd_resolve_device(mydict.get("sdclipdevice"), default_sdclipdevice))) sd_convdirect_var.set(sd_convdirect_option(mydict.get("sdconvdirect"))) sd_vae_var.set(mydict["sdvae"] if ("sdvae" in mydict and mydict["sdvae"]) else "") + sd_audio_vae_var.set(mydict["sdaudiovae"] if ("sdaudiovae" in mydict and mydict["sdaudiovae"]) else "") sd_t5xxl_var.set(mydict["sdt5xxl"] if ("sdt5xxl" in mydict and mydict["sdt5xxl"]) else "") sd_clip1_var.set(mydict["sdclip1"] if ("sdclip1" in mydict and mydict["sdclip1"]) else "") sd_clip2_var.set(mydict["sdclip2"] if ("sdclip2" in mydict and mydict["sdclip2"]) else "") @@ -10635,7 +10642,7 @@ def disableSwappedFieldsInConfig(args, swapReqType): for e in ["musicllm", "musicembeddings", "musicdiffusion", "musicvae"]: setattr(args, e, "") if swapReqType != "image": - for e in ["sdmodel", "sdt5xxl", "sdclip1", "sdclip2", "sdphotomaker", "sdupscaler", "sdvae", "sdlora"]: + for e in ["sdmodel", "sdt5xxl", "sdclip1", "sdclip2", "sdphotomaker", "sdupscaler", "sdvae", "sdaudiovae", "sdlora"]: setattr(args, e, "") @@ -10826,6 +10833,10 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False): dlfile = download_model_from_url(args.sdvae,[".gguf",".safetensors"],min_file_size=500000) if dlfile: args.sdvae = dlfile + if args.sdaudiovae and args.sdaudiovae!="": + dlfile = download_model_from_url(args.sdaudiovae,[".gguf",".safetensors"],min_file_size=500000) + if dlfile: + args.sdaudiovae = dlfile if args.sdlora and len(args.sdlora)>0: for i in range(0,len(args.sdlora)): dlfile = download_model_from_url(args.sdlora[i],[".gguf",".safetensors"],min_file_size=500000) @@ -11171,6 +11182,7 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False): imgclip2 = "" imgphotomaker = "" imgupscaler = "" + imgaudiovae = "" global imglora_preload, imglora_bypath, imglora_name2path imglora_preload, imglora_bypath, imglora_name2path = mk_lora_info(args.sdlora, args.sdloramult) if args.sdvae: @@ -11178,6 +11190,11 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False): imgvae = os.path.abspath(args.sdvae) else: print("Missing SD VAE model file...") + if args.sdaudiovae: + if os.path.exists(args.sdaudiovae): + imgaudiovae = os.path.abspath(args.sdaudiovae) + else: + print("Missing SD Audio VAE model file...") if args.sdt5xxl: if os.path.exists(args.sdt5xxl): imgt5xxl = os.path.abspath(args.sdt5xxl) @@ -11209,7 +11226,7 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False): friendlysdmodelname = os.path.basename(imgmodel) friendlysdmodelname = os.path.splitext(friendlysdmodelname)[0] friendlysdmodelname = sanitize_string(friendlysdmodelname) - loadok = sd_load_model(imgmodel,imgvae,imgt5xxl,imgclip1,imgclip2,imgphotomaker,imgupscaler) + loadok = sd_load_model(imgmodel,imgvae,imgt5xxl,imgclip1,imgclip2,imgphotomaker,imgupscaler,imgaudiovae) cached_sd_info = sd_get_info() print("Load Image Model OK: " + str(loadok)) if not loadok: @@ -11790,6 +11807,7 @@ if __name__ == '__main__': sdparsergroupvae = sdparsergroup.add_mutually_exclusive_group() sdparsergroupvae.add_argument("--sdvae", metavar=('[filename]'), help="Specify an image generation safetensors VAE which replaces the one in the model.", default="") sdparsergroupvae.add_argument("--sdvaeauto", help="Uses a built-in tiny VAE via TAE SD, which is very fast, and fixed bad VAEs.", action='store_true') + sdparsergroup.add_argument("--sdaudiovae", metavar=('[filename]'), help="Specify an image generation audio VAE for LTX2.3 video generation.", default="") sdparsergrouplora = sdparsergroup.add_mutually_exclusive_group() sdparsergrouplora.add_argument("--sdquant", metavar=('[quantization level 0/1/2]'), help="If specified, loads the model quantized to save memory. 0=off, 1=q8, 2=q4", type=int, choices=[0,1,2], nargs="?", const=2, default=0) sdparsergrouplora.add_argument("--sdlora", metavar=('[filename]'), help="Specify image generation LoRAs safetensors models to be applied. Multiple LoRAs are accepted.", nargs='+') diff --git a/otherarch/sdcpp/sdtype_adapter.cpp b/otherarch/sdcpp/sdtype_adapter.cpp index 6a5bd5bb0..007992123 100644 --- a/otherarch/sdcpp/sdtype_adapter.cpp +++ b/otherarch/sdcpp/sdtype_adapter.cpp @@ -100,6 +100,7 @@ struct SDParams { std::string t5xxl_path; std::string diffusion_model_path; std::string vae_path; + std::string audio_vae_path; std::string taesd_path; std::string stacked_id_embeddings_path; sd_type_t wtype = SD_TYPE_COUNT; @@ -340,6 +341,7 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) { lora_map.add_lora(inputs.lora_filenames[i], inputs.lora_multipliers[i]); } std::string vaefilename = inputs.vae_filename; + std::string audiovaefilename = inputs.audio_vae_filename; std::string t5xxl_filename = inputs.t5xxl_filename; std::string clip1_filename = inputs.clip1_filename; std::string clip2_filename = inputs.clip2_filename; @@ -394,6 +396,10 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) { { printf("With Custom VAE: %s\n",vaefilename.c_str()); } + if(audiovaefilename!="") + { + printf("With Audio VAE: %s\n",audiovaefilename.c_str()); + } if(t5xxl_filename!="") { printf("With Custom T5-XXL Model: %s\n",t5xxl_filename.c_str()); @@ -457,6 +463,7 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) { sd_params->diffusion_conv_direct = inputs.diffusion_conv_direct; sd_params->vae_conv_direct = inputs.vae_conv_direct; sd_params->vae_path = vaefilename; + sd_params->audio_vae_path = audiovaefilename; sd_params->taesd_path = taesdpath; sd_params->t5xxl_path = t5xxl_filename; sd_params->clip_l_path = clip1_filename; @@ -490,6 +497,7 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) { params.t5xxl_path = sd_params->t5xxl_path.c_str(); params.diffusion_model_path = sd_params->diffusion_model_path.c_str(); params.vae_path = sd_params->vae_path.c_str(); + params.audio_vae_path = sd_params->audio_vae_path.c_str(); params.taesd_path = sd_params->taesd_path.c_str(); params.photo_maker_path = sd_params->stacked_id_embeddings_path.c_str(); diff --git a/otherarch/sdcpp/stable-diffusion.cpp b/otherarch/sdcpp/stable-diffusion.cpp index 9252c8294..21d6d68e4 100644 --- a/otherarch/sdcpp/stable-diffusion.cpp +++ b/otherarch/sdcpp/stable-diffusion.cpp @@ -277,13 +277,14 @@ public: } max_vram = sd::ggml_graph_cut::resolve_max_vram_gib(max_vram, backend_for(SDBackendModule::DIFFUSION)); - std::string clip_vision_fixed = SAFE_STR(sd_ctx_params->clip_vision_path); - std::string clipg_path_fixed = SAFE_STR(sd_ctx_params->clip_g_path); - std::string clipl_path_fixed = SAFE_STR(sd_ctx_params->clip_l_path); - std::string llm_path_fixed = SAFE_STR(sd_ctx_params->llm_path); + std::string clip_vision_fixed = SAFE_STR(sd_ctx_params->clip_vision_path); + std::string clipg_path_fixed = SAFE_STR(sd_ctx_params->clip_g_path); + std::string clipl_path_fixed = SAFE_STR(sd_ctx_params->clip_l_path); + std::string llm_path_fixed = SAFE_STR(sd_ctx_params->llm_path); std::string llm_vision_path_fixed = SAFE_STR(sd_ctx_params->llm_vision_path); - std::string t5_path_fixed = SAFE_STR(sd_ctx_params->t5xxl_path); - std::string taesd_path_fixed = SAFE_STR(sd_ctx_params->taesd_path); + std::string t5_path_fixed = SAFE_STR(sd_ctx_params->t5xxl_path); + std::string taesd_path_fixed = SAFE_STR(sd_ctx_params->taesd_path); + std::string embed_connector_fixed = SAFE_STR(sd_ctx_params->embeddings_connectors_path); ModelLoader model_loader; @@ -353,7 +354,8 @@ public: bool is_ernie = sd_version_is_ernie_image(tempver); bool is_longcat = sd_version_is_longcat(tempver); bool is_lens = sd_version_is_lens(tempver); - bool conditioner_is_llm = (is_qwenimg || iszimg || isflux2 || is_ovis || is_anima || is_ernie || is_longcat || is_lens); + bool is_ltx = sd_version_is_ltxav(tempver); + bool conditioner_is_llm = (is_qwenimg || iszimg || isflux2 || is_ovis || is_anima || is_ernie || is_longcat || is_lens || is_ltx); //kcpp qol fallback: if a llm was loaded as t5 by mistake if(conditioner_is_llm && t5_path_fixed!="") @@ -366,7 +368,7 @@ public: else if(clipl_path_fixed!="" && clipg_path_fixed=="") { //very tricky case. see if we can tell if clipl is an mmproj, if so move to right place - if(toLowerCase(clipl_path_fixed).find("mmproj") != std::string::npos) + if(toLowerCase(clipl_path_fixed).find("mmproj") != std::string::npos || is_ltx) { clipg_path_fixed = clipl_path_fixed; clipl_path_fixed = t5_path_fixed; @@ -409,6 +411,11 @@ public: llm_vision_path_fixed = clipg_path_fixed; clipg_path_fixed = ""; } + else if(is_ltx) + { + embed_connector_fixed = clipg_path_fixed; + clipg_path_fixed = ""; + } } //settle possible inversions for mmproj @@ -476,13 +483,14 @@ public: sd_ctx_params->photo_maker_path = ""; } - sd_ctx_params->clip_g_path = clipg_path_fixed.c_str(); - sd_ctx_params->clip_l_path = clipl_path_fixed.c_str(); - sd_ctx_params->clip_vision_path = clip_vision_fixed.c_str(); - sd_ctx_params->llm_path = llm_path_fixed.c_str(); - sd_ctx_params->llm_vision_path = llm_vision_path_fixed.c_str(); - sd_ctx_params->t5xxl_path = t5_path_fixed.c_str(); - sd_ctx_params->taesd_path = taesd_path_fixed.c_str(); + sd_ctx_params->clip_g_path = clipg_path_fixed.c_str(); + sd_ctx_params->clip_l_path = clipl_path_fixed.c_str(); + sd_ctx_params->clip_vision_path = clip_vision_fixed.c_str(); + sd_ctx_params->llm_path = llm_path_fixed.c_str(); + sd_ctx_params->llm_vision_path = llm_vision_path_fixed.c_str(); + sd_ctx_params->t5xxl_path = t5_path_fixed.c_str(); + sd_ctx_params->taesd_path = taesd_path_fixed.c_str(); + sd_ctx_params->embeddings_connectors_path = embed_connector_fixed.c_str(); //debug print // printf("\n\nclip_g: %s\nclip_l: %s\nclip_vision: %s\nllm: %s\nllm_vision: %s\nt5xxl: %s\ntaesd: %s\n", // sd_ctx_params->clip_g_path, sd_ctx_params->clip_l_path, sd_ctx_params->clip_vision_path,