diff --git a/embd_res/kcpp_musicui.embd b/embd_res/kcpp_musicui.embd new file mode 100644 index 000000000..00b99b79e --- /dev/null +++ b/embd_res/kcpp_musicui.embd @@ -0,0 +1,496 @@ + + + + + +KoboldCpp Music Generation + + + + + + +
🎵 KoboldCpp Music Generation
+ +
+ +
+

Song Setup

+ + + + +
+ + +
+ +
+
+
+
+
+
+
+ +
âš™ Advanced Settings
+ + + +
+ +
+ +
+
+ +
+
+ + + + + +
+ + + + + +
+ +
+ +
+

Your Library

+
+ +
+ +
+ +
+ + + + + \ No newline at end of file diff --git a/koboldcpp.py b/koboldcpp.py index 4949890dc..6e3e683aa 100755 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -138,6 +138,8 @@ embedded_kcpp_docs_gz = None embedded_kcpp_sdui = None embedded_kcpp_sdui_gz = None embedded_lcpp_ui_gz = None +embedded_musicui = None +embedded_musicui_gz = None voicebank = {} voicelist = ["kobo","cheery","sleepy","shouty","chatty"] sslvalid = False @@ -3926,7 +3928,7 @@ Change Mode
self.wfile.write(finalhtml) def do_GET(self): - global embedded_kailite, embedded_kcpp_docs, embedded_kcpp_sdui, embedded_kailite_gz, embedded_kcpp_docs_gz, embedded_kcpp_sdui_gz, embedded_lcpp_ui_gz + global embedded_kailite, embedded_kcpp_docs, embedded_kcpp_sdui, embedded_kailite_gz, embedded_kcpp_docs_gz, embedded_kcpp_sdui_gz, embedded_lcpp_ui_gz, embedded_musicui, embedded_musicui_gz global last_req_time, start_time, cached_chat_template, has_vision_support, has_audio_support, has_whisper, friendlymodelname global savedata_obj, has_multiplayer, multiplayer_turn_major, multiplayer_turn_minor, multiplayer_story_data_compressed, multiplayer_dataformat, multiplayer_lastactive, maxctx, maxhordelen, friendlymodelname, lastuploadedcomfyimg, lastgeneratedcomfyimg, KcppVersion, totalgens, preloaded_story, exitcounter, currentusergenkey, friendlysdmodelname, fullsdmodelpath, password, friendlyembeddingsmodelname, voicelist @@ -4230,6 +4232,16 @@ Change Mode
else: response_body = ("KoboldCpp API is running, but KCPP SDUI is not loaded").encode() + elif clean_path.startswith(("/musicui")): + content_type = 'text/html' + if supports_gzip and embedded_musicui_gz is not None: + response_body = embedded_musicui_gz + content_encoding = 'gzip' + elif embedded_musicui is not None: + response_body = embedded_musicui + else: + response_body = ("KoboldCpp API is running, but KCPP MusicUI is not loaded").encode() + elif clean_path=="/v1": content_type = 'text/html' response_body = ("KoboldCpp OpenAI compatible endpoint is running!
For usage reference, see https://platform.openai.com/docs/api-reference
For other endpoints, see KoboldCpp API Documentation").encode() @@ -8269,7 +8281,7 @@ def main(launch_args, default_args): input() def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False): - global embedded_kailite, embedded_kcpp_docs, embedded_kcpp_sdui, embedded_kailite_gz, embedded_kcpp_docs_gz, embedded_kcpp_sdui_gz, embedded_lcpp_ui_gz, start_time, exitcounter, global_memory, using_gui_launcher + global embedded_kailite, embedded_kcpp_docs, embedded_kcpp_sdui, embedded_kailite_gz, embedded_kcpp_docs_gz, embedded_kcpp_sdui_gz, embedded_lcpp_ui_gz, embedded_musicui, embedded_musicui_gz, start_time, exitcounter, global_memory, using_gui_launcher global libname, args, friendlymodelname, friendlysdmodelname, fullsdmodelpath, password, fullwhispermodelpath, ttsmodelpath, embeddingsmodelpath, musicdiffusionmodelpath, friendlyembeddingsmodelname, has_audio_support, has_vision_support, cached_chat_template start_server = True @@ -8882,6 +8894,15 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False): except Exception: print("Could not find Embedded llama.cpp UI.") + try: + with open(os.path.join(embddir, "kcpp_musicui.embd"), mode='rb') as f: + embedded_musicui = f.read() + embedded_musicui_gz = gzip.compress(embedded_musicui) + if args.musicllm or args.musicdiffusion: + print("Embedded MusicUI loaded.") + except Exception: + print("Could not find Embedded MusicUI.") + # load all TTS audio files if args.ttsdir and args.ttsmodel and os.path.isdir(args.ttsdir): try: @@ -8946,6 +8967,7 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False): enabledmlist.append("VectorEmbeddings") if "embeddings" in caps and caps["embeddings"] else disabledmlist.append("VectorEmbeddings") enabledmlist.append("AdminControl") if "admin" in caps and caps["admin"]!=0 else disabledmlist.append("AdminControl") enabledmlist.append("MCPBridge") if "mcp" in caps and caps["mcp"] else disabledmlist.append("MCPBridge") + enabledmlist.append("MusicGen") if "music" in caps and caps["music"] else disabledmlist.append("MusicGen") print(f"======\nActive Modules: {' '.join(enabledmlist)}") print(f"Inactive Modules: {' '.join(disabledmlist)}") @@ -8974,6 +8996,8 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False): print(f"Starting llama.cpp secondary WebUI at {endpoint_url}/lcpp/") if args.sdmodel: print(f"StableUI is available at {endpoint_url}/sdui/") + if args.musicdiffusion or args.musicllm: + print(f"MusicUI is available at {endpoint_url}/musicui/") elif global_memory: val = global_memory["tunnel_url"] if val: @@ -8984,6 +9008,8 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False): print(f"Starting llama.cpp secondary WebUI at {endpoint_url}/lcpp/") if args.sdmodel: print(f"StableUI is available at {endpoint_url}/sdui/") + if args.musicdiffusion or args.musicllm: + print(f"MusicUI is available at {endpoint_url}/musicui/") global_memory["load_complete"] = True if args.launch: def launch_browser_thread(): diff --git a/otherarch/acestep/ace-qwen3.cpp b/otherarch/acestep/ace-qwen3.cpp index 0fab30bba..7d12b0ebe 100644 --- a/otherarch/acestep/ace-qwen3.cpp +++ b/otherarch/acestep/ace-qwen3.cpp @@ -1520,23 +1520,30 @@ std::string acestep_prepare_request(const music_generation_inputs inputs) bool user_has_codes = !req.audio_codes.empty(); bool need_lm_codes = req.thinking && !user_has_codes; - bool is_simple = ace.lyrics.empty() && - ace.bpm <= 0 && ace.duration <= 0 && - ace.keyscale.empty() && ace.timesignature.empty(); + bool is_simple = ace.lyrics.empty(); std::vector prompt; std::vector aces; // populated by Phase 1 (simple or partial) // Preprocessor: simple mode generates lyrics + metas from caption if (is_simple) { - fprintf(stderr, "[Simple] Inspiration\n"); + fprintf(stderr, "[Simple] Inspiration, Language: %s\n",ace.vocal_language.c_str()); + const char * sys = "# Instruction\n" - "Expand the user's input into a more detailed" - " and specific musical description:\n"; - std::string user_msg = ace.caption + "\n\ninstrumental: " - + std::string(req.instrumental ? "true" : "false"); + "Expand the user's input into a more detailed and specific musical description:\n"; + bool forcelang = (ace.vocal_language != "unknown" && !ace.vocal_language.empty()); + std::string langstr = forcelang?("language: "+ace.vocal_language+"\n"):""; + std::string cap = ""; + std::string instru = "instrumental: false\n"; + if(ace.caption!="") + { + cap = "Music Caption: " + ace.caption; + } + std::string user_msg = cap + "\n\n"+instru+langstr; + + printf("\n[Prompt: %s]\n",user_msg.c_str()); prompt = build_custom_prompt(acestep_bpe, sys, user_msg.c_str()); // FSM: reset then optionally force language (shared for both paths) @@ -1549,7 +1556,7 @@ std::string acestep_prepare_request(const music_generation_inputs inputs) prompt.size(), batch_size, seed, seed + batch_size - 1); auto phase1_texts = generate_phase1_batch( - &acestep_llm, &acestep_bpe, prompt, 2048, temperature, 1.0f, 0, + &acestep_llm, &acestep_bpe, prompt, 2048, temperature, 0.95f, 0, seed, batch_size, use_fsm ? &fsm : nullptr, true); parse_phase1_into_aces(phase1_texts, ace, aces, seed, "Simple", true); diff --git a/otherarch/acestep/cond.h b/otherarch/acestep/cond.h index 9b76131e8..9581806dd 100644 --- a/otherarch/acestep/cond.h +++ b/otherarch/acestep/cond.h @@ -179,9 +179,21 @@ static void cond_ggml_forward(CondGGML * m, // Linear embed: [1024, S_lyric] -> [2048, S_lyric] struct ggml_tensor * lyric_h = qwen3_linear_bias(ctx, m->lyric_embed_w, m->lyric_embed_b, t_lyric_in); - // 8 layers + final norm (bidirectional: mask=NULL) - lyric_h = qwen3_build_layers(ctx, m->lyric_cfg, m->lyric_layers, m->lyric_norm, - lyric_h, lyric_pos, NULL, S_lyric); + + // Bidirectional sliding window mask for even layers (|i-j| <= 128) + // Python: layer_types = [sliding, full, sliding, full, ...] + // sliding_window = 128 + struct ggml_tensor * lyric_slide_mask = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, S_lyric, S_lyric); + ggml_set_name(lyric_slide_mask, "lyric_slide_mask"); + ggml_set_input(lyric_slide_mask); + + // 8 layers with alternating masks + final norm + for (int i = 0; i < m->lyric_cfg.n_layers; i++) { + struct ggml_tensor * layer_mask = (i % 2 == 0) ? lyric_slide_mask : NULL; + lyric_h = qwen3_build_layer(ctx, m->lyric_cfg, &m->lyric_layers[i], + lyric_h, lyric_pos, layer_mask, S_lyric); + } + lyric_h = qwen3_rms_norm(ctx, lyric_h, m->lyric_norm, m->lyric_cfg.rms_norm_eps); ggml_set_name(lyric_h, "lyric_out"); ggml_set_output(lyric_h); @@ -200,6 +212,7 @@ static void cond_ggml_forward(CondGGML * m, struct ggml_tensor * timbre_out = NULL; struct ggml_tensor * t_timbre_in = NULL; struct ggml_tensor * timbre_pos = NULL; + struct ggml_tensor * timbre_slide_mask = NULL; if (has_timbre) { timbre_pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, S_ref); @@ -213,9 +226,19 @@ static void cond_ggml_forward(CondGGML * m, // Linear embed: [64, S_ref] -> [2048, S_ref] struct ggml_tensor * timbre_h = qwen3_linear_bias(ctx, m->timbre_embed_w, m->timbre_embed_b, t_timbre_in); - // 4 layers + final norm - timbre_h = qwen3_build_layers(ctx, m->timbre_cfg, m->timbre_layers, m->timbre_norm, - timbre_h, timbre_pos, NULL, S_ref); + + // Bidirectional sliding window mask for even layers + timbre_slide_mask = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, S_ref, S_ref); + ggml_set_name(timbre_slide_mask, "timbre_slide_mask"); + ggml_set_input(timbre_slide_mask); + + // 4 layers with alternating masks + final norm + for (int i = 0; i < m->timbre_cfg.n_layers; i++) { + struct ggml_tensor * layer_mask = (i % 2 == 0) ? timbre_slide_mask : NULL; + timbre_h = qwen3_build_layer(ctx, m->timbre_cfg, &m->timbre_layers[i], + timbre_h, timbre_pos, layer_mask, S_ref); + } + timbre_h = qwen3_rms_norm(ctx, timbre_h, m->timbre_norm, m->timbre_cfg.rms_norm_eps); // Take first frame: [2048, S_ref] -> view [2048, 1] timbre_out = ggml_view_2d(ctx, timbre_h, H, 1, @@ -242,11 +265,39 @@ static void cond_ggml_forward(CondGGML * m, ggml_backend_tensor_set(lyric_pos, pos.data(), 0, S_lyric * sizeof(int)); } + // Lyric sliding window mask: bidirectional, |i-j| <= 128 + { + const int W = 128; + std::vector mask_data(S_lyric * S_lyric); + for (int i = 0; i < S_lyric; i++) { + for (int j = 0; j < S_lyric; j++) { + int d = i - j; if (d < 0) d = -d; + mask_data[i * S_lyric + j] = ggml_fp32_to_fp16(d <= W ? 0.0f : -INFINITY); + } + } + ggml_backend_tensor_set(lyric_slide_mask, mask_data.data(), 0, + S_lyric * S_lyric * sizeof(uint16_t)); + fprintf(stderr, "[CondEnc] Lyric sliding mask: %dx%d, window=%d\n", S_lyric, S_lyric, W); + } + if (has_timbre) { ggml_backend_tensor_set(t_timbre_in, timbre_feats, 0, 64 * S_ref * sizeof(float)); std::vector pos(S_ref); for (int i = 0; i < S_ref; i++) pos[i] = i; ggml_backend_tensor_set(timbre_pos, pos.data(), 0, S_ref * sizeof(int)); + + // Timbre sliding window mask: bidirectional, |i-j| <= 128 + const int W = 128; + std::vector mask_data(S_ref * S_ref); + for (int i = 0; i < S_ref; i++) { + for (int j = 0; j < S_ref; j++) { + int d = i - j; if (d < 0) d = -d; + mask_data[i * S_ref + j] = ggml_fp32_to_fp16(d <= W ? 0.0f : -INFINITY); + } + } + ggml_backend_tensor_set(timbre_slide_mask, mask_data.data(), 0, + S_ref * S_ref * sizeof(uint16_t)); + fprintf(stderr, "[CondEnc] Timbre sliding mask: %dx%d, window=%d\n", S_ref, S_ref, W); } // Compute @@ -292,4 +343,4 @@ static void cond_ggml_free(CondGGML * m) { if (m->cpu_backend) ggml_backend_free(m->cpu_backend); wctx_free(&m->wctx); *m = {}; -} +} \ No newline at end of file diff --git a/otherarch/acestep/dit-vae.cpp b/otherarch/acestep/dit-vae.cpp index c723844b3..af4ccfd1a 100644 --- a/otherarch/acestep/dit-vae.cpp +++ b/otherarch/acestep/dit-vae.cpp @@ -762,7 +762,7 @@ std::string acestep_generate_audio(const music_generation_inputs inputs) const char * keyscale = req.keyscale.empty() ? "N/A" : req.keyscale.c_str(); const char * timesig = req.timesignature.empty() ? "N/A" : req.timesignature.c_str(); const char * language = req.vocal_language.empty() ? "en" : req.vocal_language.c_str(); - float duration = req.duration > 0 ? req.duration : 120.0f; + float duration = req.duration > 0 ? req.duration : 60.0f; long long seed = req.seed; int num_steps = req.inference_steps > 0 ? req.inference_steps : 8; float guidance_scale = req.guidance_scale > 0 ? req.guidance_scale : 7.0f; @@ -869,8 +869,8 @@ std::string acestep_generate_audio(const music_generation_inputs inputs) // Context building // Silence latent for this T - std::vector silence(Oc * T); - memcpy(silence.data(), silence_full.data(), (size_t)(Oc * T) * sizeof(float)); + // std::vector silence(Oc * T); + // memcpy(silence.data(), silence_full.data(), (size_t)(Oc * T) * sizeof(float)); // Decode audio codes if provided int decoded_T = 0; @@ -896,7 +896,7 @@ std::string acestep_generate_audio(const music_generation_inputs inputs) for (int t = 0; t < T; t++) { const float * src = (t < decoded_T) ? decoded_latents.data() + t * Oc - : silence.data() + t * Oc; + : silence_full.data() + (t - decoded_T) * Oc; for (int c = 0; c < Oc; c++) context_single[t * ctx_ch + c] = src[c]; for (int c = 0; c < Oc; c++) diff --git a/otherarch/acestep/request.cpp b/otherarch/acestep/request.cpp index 0d39c30cb..eae6ea952 100644 --- a/otherarch/acestep/request.cpp +++ b/otherarch/acestep/request.cpp @@ -19,7 +19,7 @@ void request_init(AceRequest * r) { r->duration = -1.0f; r->keyscale = ""; r->timesignature = ""; - r->vocal_language = "unknown"; + r->vocal_language = "en"; r->task_type = "text2music"; r->seed = -1; r->thinking = false;