cover mode is now working

embd_res/kcpp_musicui.embd

@@ -213,6 +213,14 @@ input[type="checkbox"] {
     <div>
       <div><label>AudioCodes</label><textarea id="audio_codes"></textarea></div>
     </div>
+    <div style="margin-top:8px">
+      <label>Music Reference Audio (.wav / .mp3)</label>
+      <div style="display:flex; gap:6px; align-items:center;">
+        <input id="music_reference_audio" type="file" accept=".wav,.mp3,audio/wav,audio/mpeg">
+        <button type="button" class="secondary" onclick="clearReferenceAudio()">Clear</button>
+      </div>
+      <div><label>Reference Audio Strength (0.0 to 1.0)</label><input id="audio_cover_strength" type="number" step="0.1" value="0.5"></div>
+    </div>
   </div>
 
   <div style="margin-top:14px">
@@ -314,7 +322,7 @@ function toggleAdvanced(){
 function getFormData(){
   const ids=["caption","lyrics","bpm","duration","keyscale","timesignature",
   "vocal_language","seed","lm_temperature","lm_cfg_scale","lm_top_p","lm_top_k","lm_rep_pen","inference_steps",
-  "codes_top_p","codes_top_k","codes_temperature",
+  "codes_top_p","codes_top_k","codes_temperature","audio_cover_strength",
   "guidance_scale","shift","audio_codes"];
   const data={};
   ids.forEach(id=>{
@@ -379,12 +387,30 @@ async function planSong(){
   }
 }
 
+function fileToBase64(file){
+  return new Promise((resolve,reject)=>{
+    const reader=new FileReader();
+    reader.onload=()=>{
+      const base64=reader.result.split(",")[1];
+      resolve(base64);
+    };
+    reader.onerror=reject;
+    reader.readAsDataURL(file);
+  });
+}
+
 async function generateSong(){
   try{
     currentController=new AbortController();
     setLoading(true);
 
     const payload=getFormData();
+
+    const refFile=document.getElementById("music_reference_audio");
+    if(refFile && refFile.files && refFile.files.length > 0){
+      payload.music_reference_audio_data=await fileToBase64(refFile.files[0]);
+    }
+
     const res=await fetch(buildUrl("/api/extra/music/generate"),{
       method:"POST",
       headers:{"Content-Type":"application/json"},
@@ -493,6 +519,10 @@ function deleteTrack(id){
   tx.objectStore(STORE).delete(id);
   tx.oncomplete=loadLibrary;
 }
+function clearReferenceAudio(){
+  const input = document.getElementById("music_reference_audio");
+  input.value = "";
+}
 function clearFields()
 {
     const fields = document.querySelectorAll('.form-grid input, #advanced input');
@@ -502,6 +532,7 @@ function clearFields()
     document.getElementById("caption").value = "";
     document.getElementById("lyrics").value = "";
     document.getElementById("audio_codes").value = "";
+    clearReferenceAudio();
 }
 function exportPlan(){
   const data=getFormData();

otherarch/acestep/dit-vae.cpp

@@ -785,12 +785,27 @@ std::string acestep_generate_audio(const music_generation_inputs inputs)
     int vae_chunk              = 256;
     int vae_overlap            = 64;
 
+    // Parse request JSON
+    AceRequest req;
+    std::string injson =  inputs.input_json;
+    request_init(&req);
+    if (!request_parse_from_str(&req, injson)) {
+        fprintf(stderr, "ERROR: failed to parse music gen request\n");
+        return "";
+    }
+    if (req.caption.empty()) {
+        req.caption = "An interesting song";
+    }
+    req.thinking = false;
+    req.inference_steps = (req.inference_steps>100?100:req.inference_steps); //clamp to 100
+    req.duration = (req.duration>420?420:req.duration); //clamp to 7 min
+
     // Cover mode: load VAE encoder and encode source audio
     bool have_cover = false;
     std::vector<float> cover_latents;  // [T_cover, 64] time-major
     int T_cover = 0;
     std::string custom_reference_audio_str = inputs.music_reference_audio_data;
-    if (custom_reference_audio_str!="")
+    if (custom_reference_audio_str!="" && req.audio_cover_strength>0)
     {
         if(!acestep_vae_enc_loaded)
         {
@@ -799,21 +814,19 @@ std::string acestep_generate_audio(const music_generation_inputs inputs)
         }
 
         music_dit_timer.reset();
-        int T_audio = 0, wav_sr = 0;
+        int T_audio = 0;
+        int wav_sr = 48000;
 
         std::vector<uint8_t> media_data_buffer = kcpp_base64_decode(custom_reference_audio_str);
-        std::vector<float> custom_reference_audio_pcmf32;
-        bool ok = kcpp_decode_audio_from_buf(media_data_buffer.data(), media_data_buffer.size(), 48000, custom_reference_audio_pcmf32);
+        std::vector<float> pcm;
+        bool ok = kcpp_decode_audio_to_f32_stereo_48k(media_data_buffer.data(), media_data_buffer.size(), pcm, T_audio);
         if (!ok) {
-            printf("\nError: Cannot read input audio file.\n");
+            printf("\nError: Cannot decode audio\n");
             return "";
         }
+        float *wav_data = pcm.data();
 
-        wav_sr = 48000;
-        T_audio = custom_reference_audio_pcmf32.size();
-        float * wav_data = custom_reference_audio_pcmf32.data();
-
-        fprintf(stderr, "[Cover] Source audio: %.2fs\n", (float)T_audio / (float)(wav_sr > 0 ? wav_sr : 48000));
+        fprintf(stderr, "[Cover] Source audio: %.2fs, SR:%d, WavDataSize:%zu\n", (float)T_audio / (float)(wav_sr > 0 ? wav_sr : 48000),wav_sr,T_audio);
         int max_T_lat = (T_audio / 1920) + 64;
         cover_latents.resize(max_T_lat * 64);
         T_cover = vae_enc_encode_tiled(&vae_enc, wav_data, T_audio,
@@ -846,21 +859,6 @@ std::string acestep_generate_audio(const music_generation_inputs inputs)
         }
     }
 
-    // Parse request JSON
-    AceRequest req;
-    std::string injson =  inputs.input_json;
-    request_init(&req);
-    if (!request_parse_from_str(&req, injson)) {
-        fprintf(stderr, "ERROR: failed to parse music gen request\n");
-        return "";
-    }
-    if (req.caption.empty()) {
-        req.caption = "An interesting song";
-    }
-    req.thinking = false;
-    req.inference_steps = (req.inference_steps>100?100:req.inference_steps); //clamp to 100
-    req.duration = (req.duration>420?420:req.duration); //clamp to 7 min
-
     // Extract params
     const char * caption  = req.caption.c_str();
     const char * lyrics   = req.lyrics.empty() ? "[Instrumental]" : req.lyrics.c_str();

otherarch/utils.cpp

@@ -977,6 +977,51 @@ bool kcpp_decode_audio_from_buf(const unsigned char * buf_in, size_t len, int ta
     return true;
 }
 
+//this version is specifically required for ace-step
+bool kcpp_decode_audio_to_f32_stereo_48k(const uint8_t * data, size_t data_size, std::vector<float> & pcm, int & T_audio) {
+    ma_result result;
+
+    // Force the exact format expected by the VAE
+    ma_decoder_config config =
+        ma_decoder_config_init(ma_format_f32, 2, 48000);
+
+    ma_decoder decoder;
+
+    result = ma_decoder_init_memory(data, data_size, &config, &decoder);
+    if (result != MA_SUCCESS)
+        return false;
+
+    ma_uint64 frame_count = 0;
+
+    result = ma_decoder_get_length_in_pcm_frames(&decoder, &frame_count);
+    if (result != MA_SUCCESS) {
+        ma_decoder_uninit(&decoder);
+        return false;
+    }
+
+    // allocate stereo
+    pcm.resize(frame_count * 2);
+
+    ma_uint64 frames_read = 0;
+
+    result = ma_decoder_read_pcm_frames(
+        &decoder,
+        pcm.data(),
+        frame_count,
+        &frames_read
+    );
+
+    ma_decoder_uninit(&decoder);
+
+    if (result != MA_SUCCESS)
+        return false;
+
+    pcm.resize(frames_read * 2);
+    T_audio = (int)frames_read;
+
+    return true;
+}
+
 static std::vector<std::string> kcpp_string_split(const std::string & input, char separator)
 {
     std::vector<std::string> parts;

otherarch/utils.h

@@ -71,6 +71,7 @@ int32_t kcpp_quick_sample(float * logits, const int n_logits, const std::vector<
 
 std::vector<std::string> split_string(const std::string& input, const std::string& separator);
 bool kcpp_decode_audio_from_buf(const unsigned char * buf_in, size_t len, int target_sampler_rate, std::vector<float> & pcmf32_mono);
+bool kcpp_decode_audio_to_f32_stereo_48k(const uint8_t * data, size_t data_size, std::vector<float> & pcm, int & T_audio);
 
 std::vector<ggml_backend_dev_t> kcpp_parse_device_list(const std::string & value);