diff --git a/embd_res/kcpp_musicui.embd b/embd_res/kcpp_musicui.embd
index fadb267de..9d7bb02c0 100644
--- a/embd_res/kcpp_musicui.embd
+++ b/embd_res/kcpp_musicui.embd
@@ -213,6 +213,14 @@ input[type="checkbox"] {
+
+
+
+
+
+
+
+
@@ -314,7 +322,7 @@ function toggleAdvanced(){
function getFormData(){
const ids=["caption","lyrics","bpm","duration","keyscale","timesignature",
"vocal_language","seed","lm_temperature","lm_cfg_scale","lm_top_p","lm_top_k","lm_rep_pen","inference_steps",
- "codes_top_p","codes_top_k","codes_temperature",
+ "codes_top_p","codes_top_k","codes_temperature","audio_cover_strength",
"guidance_scale","shift","audio_codes"];
const data={};
ids.forEach(id=>{
@@ -379,12 +387,30 @@ async function planSong(){
}
}
+function fileToBase64(file){
+ return new Promise((resolve,reject)=>{
+ const reader=new FileReader();
+ reader.onload=()=>{
+ const base64=reader.result.split(",")[1];
+ resolve(base64);
+ };
+ reader.onerror=reject;
+ reader.readAsDataURL(file);
+ });
+}
+
async function generateSong(){
try{
currentController=new AbortController();
setLoading(true);
const payload=getFormData();
+
+ const refFile=document.getElementById("music_reference_audio");
+ if(refFile && refFile.files && refFile.files.length > 0){
+ payload.music_reference_audio_data=await fileToBase64(refFile.files[0]);
+ }
+
const res=await fetch(buildUrl("/api/extra/music/generate"),{
method:"POST",
headers:{"Content-Type":"application/json"},
@@ -493,6 +519,10 @@ function deleteTrack(id){
tx.objectStore(STORE).delete(id);
tx.oncomplete=loadLibrary;
}
+function clearReferenceAudio(){
+ const input = document.getElementById("music_reference_audio");
+ input.value = "";
+}
function clearFields()
{
const fields = document.querySelectorAll('.form-grid input, #advanced input');
@@ -502,6 +532,7 @@ function clearFields()
document.getElementById("caption").value = "";
document.getElementById("lyrics").value = "";
document.getElementById("audio_codes").value = "";
+ clearReferenceAudio();
}
function exportPlan(){
const data=getFormData();
diff --git a/otherarch/acestep/dit-vae.cpp b/otherarch/acestep/dit-vae.cpp
index c3b85bb36..ac18bd1e1 100644
--- a/otherarch/acestep/dit-vae.cpp
+++ b/otherarch/acestep/dit-vae.cpp
@@ -785,12 +785,27 @@ std::string acestep_generate_audio(const music_generation_inputs inputs)
int vae_chunk = 256;
int vae_overlap = 64;
+ // Parse request JSON
+ AceRequest req;
+ std::string injson = inputs.input_json;
+ request_init(&req);
+ if (!request_parse_from_str(&req, injson)) {
+ fprintf(stderr, "ERROR: failed to parse music gen request\n");
+ return "";
+ }
+ if (req.caption.empty()) {
+ req.caption = "An interesting song";
+ }
+ req.thinking = false;
+ req.inference_steps = (req.inference_steps>100?100:req.inference_steps); //clamp to 100
+ req.duration = (req.duration>420?420:req.duration); //clamp to 7 min
+
// Cover mode: load VAE encoder and encode source audio
bool have_cover = false;
std::vector cover_latents; // [T_cover, 64] time-major
int T_cover = 0;
std::string custom_reference_audio_str = inputs.music_reference_audio_data;
- if (custom_reference_audio_str!="")
+ if (custom_reference_audio_str!="" && req.audio_cover_strength>0)
{
if(!acestep_vae_enc_loaded)
{
@@ -799,21 +814,19 @@ std::string acestep_generate_audio(const music_generation_inputs inputs)
}
music_dit_timer.reset();
- int T_audio = 0, wav_sr = 0;
+ int T_audio = 0;
+ int wav_sr = 48000;
std::vector media_data_buffer = kcpp_base64_decode(custom_reference_audio_str);
- std::vector custom_reference_audio_pcmf32;
- bool ok = kcpp_decode_audio_from_buf(media_data_buffer.data(), media_data_buffer.size(), 48000, custom_reference_audio_pcmf32);
+ std::vector pcm;
+ bool ok = kcpp_decode_audio_to_f32_stereo_48k(media_data_buffer.data(), media_data_buffer.size(), pcm, T_audio);
if (!ok) {
- printf("\nError: Cannot read input audio file.\n");
+ printf("\nError: Cannot decode audio\n");
return "";
}
+ float *wav_data = pcm.data();
- wav_sr = 48000;
- T_audio = custom_reference_audio_pcmf32.size();
- float * wav_data = custom_reference_audio_pcmf32.data();
-
- fprintf(stderr, "[Cover] Source audio: %.2fs\n", (float)T_audio / (float)(wav_sr > 0 ? wav_sr : 48000));
+ fprintf(stderr, "[Cover] Source audio: %.2fs, SR:%d, WavDataSize:%zu\n", (float)T_audio / (float)(wav_sr > 0 ? wav_sr : 48000),wav_sr,T_audio);
int max_T_lat = (T_audio / 1920) + 64;
cover_latents.resize(max_T_lat * 64);
T_cover = vae_enc_encode_tiled(&vae_enc, wav_data, T_audio,
@@ -846,21 +859,6 @@ std::string acestep_generate_audio(const music_generation_inputs inputs)
}
}
- // Parse request JSON
- AceRequest req;
- std::string injson = inputs.input_json;
- request_init(&req);
- if (!request_parse_from_str(&req, injson)) {
- fprintf(stderr, "ERROR: failed to parse music gen request\n");
- return "";
- }
- if (req.caption.empty()) {
- req.caption = "An interesting song";
- }
- req.thinking = false;
- req.inference_steps = (req.inference_steps>100?100:req.inference_steps); //clamp to 100
- req.duration = (req.duration>420?420:req.duration); //clamp to 7 min
-
// Extract params
const char * caption = req.caption.c_str();
const char * lyrics = req.lyrics.empty() ? "[Instrumental]" : req.lyrics.c_str();
diff --git a/otherarch/utils.cpp b/otherarch/utils.cpp
index b90f673eb..228e90dd2 100644
--- a/otherarch/utils.cpp
+++ b/otherarch/utils.cpp
@@ -977,6 +977,51 @@ bool kcpp_decode_audio_from_buf(const unsigned char * buf_in, size_t len, int ta
return true;
}
+//this version is specifically required for ace-step
+bool kcpp_decode_audio_to_f32_stereo_48k(const uint8_t * data, size_t data_size, std::vector & pcm, int & T_audio) {
+ ma_result result;
+
+ // Force the exact format expected by the VAE
+ ma_decoder_config config =
+ ma_decoder_config_init(ma_format_f32, 2, 48000);
+
+ ma_decoder decoder;
+
+ result = ma_decoder_init_memory(data, data_size, &config, &decoder);
+ if (result != MA_SUCCESS)
+ return false;
+
+ ma_uint64 frame_count = 0;
+
+ result = ma_decoder_get_length_in_pcm_frames(&decoder, &frame_count);
+ if (result != MA_SUCCESS) {
+ ma_decoder_uninit(&decoder);
+ return false;
+ }
+
+ // allocate stereo
+ pcm.resize(frame_count * 2);
+
+ ma_uint64 frames_read = 0;
+
+ result = ma_decoder_read_pcm_frames(
+ &decoder,
+ pcm.data(),
+ frame_count,
+ &frames_read
+ );
+
+ ma_decoder_uninit(&decoder);
+
+ if (result != MA_SUCCESS)
+ return false;
+
+ pcm.resize(frames_read * 2);
+ T_audio = (int)frames_read;
+
+ return true;
+}
+
static std::vector kcpp_string_split(const std::string & input, char separator)
{
std::vector parts;
diff --git a/otherarch/utils.h b/otherarch/utils.h
index 5863fa7ea..9b0803810 100644
--- a/otherarch/utils.h
+++ b/otherarch/utils.h
@@ -71,6 +71,7 @@ int32_t kcpp_quick_sample(float * logits, const int n_logits, const std::vector<
std::vector split_string(const std::string& input, const std::string& separator);
bool kcpp_decode_audio_from_buf(const unsigned char * buf_in, size_t len, int target_sampler_rate, std::vector & pcmf32_mono);
+bool kcpp_decode_audio_to_f32_stereo_48k(const uint8_t * data, size_t data_size, std::vector & pcm, int & T_audio);
std::vector kcpp_parse_device_list(const std::string & value);