diff --git a/embd_res/kcpp_sdui.embd b/embd_res/kcpp_sdui.embd
index 2cae75fea..85bd8361e 100644
--- a/embd_res/kcpp_sdui.embd
+++ b/embd_res/kcpp_sdui.embd
@@ -5,18 +5,18 @@
Stable UI for KoboldCpp
-
-
diff --git a/expose.h b/expose.h
index f0fef3cc1..311bf5111 100644
--- a/expose.h
+++ b/expose.h
@@ -225,10 +225,12 @@ struct sd_generation_inputs
const char * negative_prompt = nullptr;
const char * init_images = "";
const char * mask = "";
- const char * audio_data = "";
+ const char * video_start_frame = "";
+ const char * video_end_frame = "";
const int extra_images_len = 0;
const char ** extra_images = nullptr;
- const bool reverse_refimg = false;
+ const int ref_audios_len = 0;
+ const char ** ref_audios = nullptr;
const bool flip_mask = false;
const float denoising_strength = 0.0f;
const float cfg_scale = 0.0f;
diff --git a/koboldcpp.py b/koboldcpp.py
index 2a29699db..d2d854ca7 100644
--- a/koboldcpp.py
+++ b/koboldcpp.py
@@ -462,10 +462,12 @@ class sd_generation_inputs(ctypes.Structure):
("negative_prompt", ctypes.c_char_p),
("init_images", ctypes.c_char_p),
("mask", ctypes.c_char_p),
- ("audio_data", ctypes.c_char_p),
+ ("video_start_frame", ctypes.c_char_p),
+ ("video_end_frame", ctypes.c_char_p),
("extra_images_len", ctypes.c_int),
("extra_images", ctypes.POINTER(ctypes.c_char_p)),
- ("reverse_refimg", ctypes.c_bool),
+ ("ref_audios_len", ctypes.c_int),
+ ("ref_audios", ctypes.POINTER(ctypes.c_char_p)),
("flip_mask", ctypes.c_bool),
("denoising_strength", ctypes.c_float),
("cfg_scale", ctypes.c_float),
@@ -2931,6 +2933,8 @@ def sd_generate(genparams):
init_images = ("" if (not init_images_arr or len(init_images_arr)==0 or not init_images_arr[0]) else init_images_arr[0])
init_images = strip_base64_prefix(init_images)
mask = strip_base64_prefix(genparams.get("mask", ""))
+ video_start_frame = strip_base64_prefix(genparams.get("video_start_frame", ""))
+ video_end_frame = strip_base64_prefix(genparams.get("video_end_frame", ""))
flip_mask = genparams.get("inpainting_mask_invert", 0)
denoising_strength = tryparsefloat(genparams.get("denoising_strength", 0.6),0.6)
cfg_scale = tryparsefloat(genparams.get("cfg_scale", 5),5)
@@ -2959,9 +2963,9 @@ def sd_generate(genparams):
extra_images_arr = ([] if not extra_images_arr else extra_images_arr)
extra_images_arr = [img for img in extra_images_arr if img not in (None, "")]
- audio_data = next((img for img in extra_images_arr if img.startswith("data:audio")), None)
- extra_images_arr = [img for img in extra_images_arr if not img.startswith("data:audio")]
- audio_data = strip_base64_prefix(audio_data)
+ audio_refs_arr = [img for img in extra_images_arr if isinstance(img, str) and img.startswith("data:audio")]
+ extra_images_arr = [img for img in extra_images_arr if not (isinstance(img, str) and img.startswith("data:audio"))]
+ audio_refs_arr = [strip_base64_prefix(aud) for aud in audio_refs_arr]
extra_images_arr = extra_images_arr[:extra_images_max]
lora_filenames, lora_multipliers = prepare_lora_multipliers(genparams.get("lora", []))
@@ -2979,8 +2983,9 @@ def sd_generate(genparams):
vid_fps = (16 if vid_fps < 16 else (32 if vid_fps > 32 else vid_fps))
swap_refimg = (True if tryparseint(genparams.get("send_as_refimg", 0),0) else False)
- reverse_refimg = (True if tryparseint(genparams.get("reverse_refimg", 0),0) else False)
if swap_refimg and init_images and init_images != "" and not mask:
+ if not video_start_frame:
+ video_start_frame = init_images
extra_images_arr = [init_images] + extra_images_arr
init_images = ""
@@ -2989,13 +2994,17 @@ def sd_generate(genparams):
inputs.negative_prompt = negative_prompt.encode("UTF-8")
inputs.init_images = init_images.encode("UTF-8")
inputs.mask = "".encode("UTF-8") if not mask else mask.encode("UTF-8")
- inputs.audio_data = "".encode("UTF-8") if not audio_data else audio_data.encode("UTF-8")
+ inputs.video_start_frame = "".encode("UTF-8") if not video_start_frame else video_start_frame.encode("UTF-8")
+ inputs.video_end_frame = "".encode("UTF-8") if not video_end_frame else video_end_frame.encode("UTF-8")
inputs.extra_images_len = len(extra_images_arr)
inputs.extra_images = (ctypes.c_char_p * inputs.extra_images_len)()
for n, estr in enumerate(extra_images_arr):
extra_image = strip_base64_prefix(estr)
inputs.extra_images[n] = extra_image.encode("UTF-8")
- inputs.reverse_refimg = reverse_refimg
+ inputs.ref_audios_len = len(audio_refs_arr)
+ inputs.ref_audios = (ctypes.c_char_p * inputs.ref_audios_len)()
+ for n, aud in enumerate(audio_refs_arr):
+ inputs.ref_audios[n] = aud.encode("UTF-8")
inputs.flip_mask = flip_mask
inputs.cfg_scale = cfg_scale
if distilled_guidance is not None:
diff --git a/otherarch/sdcpp/sdtype_adapter.cpp b/otherarch/sdcpp/sdtype_adapter.cpp
index c443845b8..e4975c5b9 100644
--- a/otherarch/sdcpp/sdtype_adapter.cpp
+++ b/otherarch/sdcpp/sdtype_adapter.cpp
@@ -1117,12 +1117,18 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
std::string img2img_data = std::string(inputs.init_images);
std::string img2img_mask = std::string(inputs.mask);
- std::string input_audio_data = std::string(inputs.audio_data ? inputs.audio_data : "");
+ std::string video_start_frame_data = std::string(inputs.video_start_frame ? inputs.video_start_frame : "");
+ std::string video_end_frame_data = std::string(inputs.video_end_frame ? inputs.video_end_frame : "");
std::vector extra_image_data;
for(int i=0;i ref_audio_data;
+ for(int i=0;iprompt = inputs.prompt;
sd_params->negative_prompt = inputs.negative_prompt;
sd_params->cfg_scale = inputs.cfg_scale;
@@ -1209,9 +1215,9 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
}
}
- if (is_video_model(info) && extra_image_data.size() == 0 && is_img2img)
+ if (is_video_model(info) && video_start_frame_data.empty() && is_img2img && img2img_mask=="")
{
- extra_image_data.push_back(img2img_data);
+ video_start_frame_data = img2img_data;
}
// limit by image side
@@ -1256,9 +1262,23 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
//for img2img
sd_image_t input_image = {0,0,0,nullptr};
+ sd_image_t video_start_image = {0,0,0,nullptr};
+ sd_image_t video_end_image = {0,0,0,nullptr};
std::vector reference_imgs;
- std::vector wan_imgs;
+ std::vector legacy_video_imgs;
std::vector photomaker_imgs;
+ std::vector reference_audios;
+ struct ReferenceAudioCleanup {
+ std::vector& audios;
+ ~ReferenceAudioCleanup() {
+ for (auto& audio : audios) {
+ if (audio.data) {
+ free(audio.data);
+ audio.data = nullptr;
+ }
+ }
+ }
+ } reference_audio_cleanup { reference_audios };
int nx, ny, nc;
int img2imgW = sd_params->width; //for img2img input
@@ -1275,16 +1295,50 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
fflush(stdout);
+ if(input_extraimage_buffers.size()>0) //just in time free old buffer
+ {
+ for(int i=0;i sd_image_t {
+ sd_image_t image = {0,0,0,nullptr};
+ if (b64.empty()) {
+ return image;
+ }
+ int nx2, ny2, nc2;
+ uint8_t * loaded = resize_to_generation
+ ? load_image_from_b64(b64,nx2,ny2,img2imgW,img2imgH,3)
+ : load_image_from_b64(b64,nx2,ny2);
+ if(loaded)
+ {
+ input_extraimage_buffers.push_back(loaded);
+ image.width = nx2;
+ image.height = ny2;
+ image.channel = 3;
+ image.data = loaded;
+ }
+ return image;
+ };
+
+ auto push_image_if_loaded = [](std::vector& images, sd_image_t image) {
+ if(image.data)
+ {
+ images.push_back(image);
+ }
+ };
+
+ if (is_video_model(info))
+ {
+ video_start_image = load_extra_image(video_start_frame_data, true);
+ video_end_image = load_extra_image(video_end_frame_data, true);
+ }
+
if(extra_image_data.size()>0)
{
- if(input_extraimage_buffers.size()>0) //just in time free old buffer
- {
- for(int i=0;i0)
+ {
+ for(int i=0;i 0);
+ const bool has_minimax_references = info.is_minimaxh3 && (reference_imgs.size() > 0 || reference_audios.size() > 0);
+ if(has_minimax_keyframes && has_minimax_references)
+ {
+ if(!sd_is_quiet && sddebugmode==1)
+ {
+ printf("\nMiniMax-H3: keyframes and Ref2VA references cannot be mixed; using reference media and ignoring keyframes.\n");
+ }
+ video_start_image = {0,0,0,nullptr};
+ video_end_image = {0,0,0,nullptr};
+ legacy_video_imgs.clear();
+ }
+
sd_img_gen_params_t params = {};
sd_img_gen_params_init (¶ms);
params.batch_count = 1;
@@ -1481,24 +1554,38 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
vid_gen_params.video_frames = vid_req_frames;
vid_gen_params.fps = vid_fps;
vid_gen_params.vae_tiling_params = params.vae_tiling_params;
- if (!input_audio_data.empty()) {
- input_audio = load_audio_from_b64(input_audio_data);
+ if (!info.is_minimaxh3 && ref_audio_data.size()>0) {
+ input_audio = load_audio_from_b64(ref_audio_data[0]);
if (input_audio.data == nullptr) {
return sd_generation.error("KCPP SD: load audio from base64 failed!");
}
vid_gen_params.input_audio = &input_audio;
}
- if (wan_imgs.size() > 0) {
- if (wan_imgs.size() >= 2) {
- vid_gen_params.init_image = wan_imgs[0];
- vid_gen_params.end_image = wan_imgs[1];
- } else if (wan_imgs.size() == 1) {
- if (inputs.reverse_refimg) {
- vid_gen_params.end_image = wan_imgs[0];
- } else {
- vid_gen_params.init_image = wan_imgs[0];
- }
+ if (info.is_minimaxh3 && reference_imgs.size() > 0) {
+ vid_gen_params.ref_images = reference_imgs.data();
+ vid_gen_params.ref_images_count = (int)reference_imgs.size();
+ }
+ if (info.is_minimaxh3 && reference_audios.size() > 0) {
+ vid_gen_params.ref_audios = reference_audios.data();
+ vid_gen_params.ref_audios_count = (int)reference_audios.size();
+ }
+ if (legacy_video_imgs.size() >= 2) {
+ if(!video_start_image.data) {
+ video_start_image = legacy_video_imgs[0];
}
+ if(!video_end_image.data) {
+ video_end_image = legacy_video_imgs[1];
+ }
+ } else if (legacy_video_imgs.size() == 1) {
+ if(!video_start_image.data && !video_end_image.data) {
+ video_start_image = legacy_video_imgs[0];
+ }
+ }
+ if (video_start_image.data) {
+ vid_gen_params.init_image = video_start_image;
+ }
+ if (video_end_image.data) {
+ vid_gen_params.end_image = video_end_image;
}
if(!sd_is_quiet && sddebugmode==1)
{
@@ -1512,7 +1599,9 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
<< "\nSTRENGTH:" << vid_gen_params.strength
<< "\nFRAMES:" << vid_gen_params.video_frames
<< "\nCTRL_FRM:" << vid_gen_params.control_frames_size
- << "\nINIT_IMGS:" << wan_imgs.size()
+ << "\nINIT_IMGS:" << ((vid_gen_params.init_image.data ? 1 : 0) + (vid_gen_params.end_image.data ? 1 : 0))
+ << "\nREF_IMGS:" << vid_gen_params.ref_images_count
+ << "\nREF_AUDIOS:" << vid_gen_params.ref_audios_count
<< "\nINPUT_AUDIO:" << (vid_gen_params.input_audio ? "true" : "false")
<< "\n\n";
printf("%s", ss.str().c_str());