diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp index 9315a65cd2..fd9d6ca429 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -401,7 +401,7 @@ private: LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n"); return false; } - const std::string marker = mtmd_default_marker(); + const std::string marker = mtmd_get_marker(mctx); mtmd_input_text text{ marker.c_str(), marker.size(), false, true }; mtmd_input_chunks * chunks = mtmd_input_chunks_init(); const mtmd_bitmap * bptr = bitmap; diff --git a/tools/server/README.md b/tools/server/README.md index 57839407b0..51b2ccb407 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -739,7 +739,7 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For more info `lang`: Language code for the utterance (model-dependent, e.g. `en`, `zh`). Optional. -`speaker_ref_b64`: Base64-encoded reference WAV to clone the speaker's voice. Optional. +`speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) — if both are provided, the uploaded file takes precedence. `top_k`, `top_p`: Sampling params for the acoustic code predictor. Optional, model-dependent defaults apply. @@ -756,12 +756,21 @@ Note: it's highly recommended to always provide a speaker reference voice; other *Examples:* ```shell -curl -X POST http://127.0.0.1:8012/tts \ +curl -X POST http://127.0.0.1:9931/tts \ -H "Content-Type: application/json" \ -d '{"input": "Hello, this is a test."}' \ -o output.wav ``` +With a speaker reference uploaded as a file (`multipart/form-data`), instead of base64-encoding it into the JSON body: + +```shell +curl -X POST http://127.0.0.1:9931/tts \ + -F "input=Hello, this is a test." \ + -F "speaker_ref=@/path/to/speaker-reference.wav;type=audio/wav" \ + -o output.wav +``` + ### POST `/infill`: For code infilling. Takes a prefix and a suffix and returns the predicted completion as stream. diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index c4f3bebe34..6684ea8ce1 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -5237,8 +5237,6 @@ void server_routes::init_routes() { task.params.stream = stream; task.params.n_predict = json_value(body, "n_predict", -1); task.params.sampling = params.sampling; // baseline defaults, then apply overrides below - // codec token streams need repetition penalty over the whole generation, or the - // backbone loops and re-generates the same utterance (see LLAMA_EXAMPLE_TTS in arg.cpp) task.params.sampling.penalty_repeat = json_value(body, "repeat_penalty", 1.05f); task.params.sampling.penalty_last_n = -1; if (task.tts_inp.data.top_k > 0) { @@ -5248,15 +5246,33 @@ void server_routes::init_routes() { task.params.sampling.top_p = task.tts_inp.data.top_p; } - std::string speaker_ref_b64 = json_value(body, "speaker_ref_b64", std::string()); - if (!speaker_ref_b64.empty()) { - std::string bytes = base64::decode(speaker_ref_b64); - auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, (const unsigned char *) bytes.data(), bytes.size(), false); + // speaker reference: either an uploaded form file ("speaker_ref") or a base64 JSON field ("speaker_ref_b64") + const unsigned char * speaker_ref_data = nullptr; + size_t speaker_ref_len = 0; + std::string speaker_ref_b64_decoded; + + auto speaker_ref_file = req.files.find("speaker_ref"); + if (speaker_ref_file != req.files.end()) { + speaker_ref_data = speaker_ref_file->second.data.data(); + speaker_ref_len = speaker_ref_file->second.data.size(); + } else { + std::string speaker_ref_b64 = json_value(body, "speaker_ref_b64", std::string()); + if (!speaker_ref_b64.empty()) { + speaker_ref_b64_decoded = base64::decode(speaker_ref_b64); + speaker_ref_data = (const unsigned char *) speaker_ref_b64_decoded.data(); + speaker_ref_len = speaker_ref_b64_decoded.size(); + } + } + + if (speaker_ref_len > 0) { + auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false); if (!wrapper.bitmap) { - res->error(format_error_response("failed to decode \"speaker_ref_b64\"", ERROR_TYPE_INVALID_REQUEST)); + res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST)); return res; } task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap)); + } else { + SRV_WRN("no speaker reference provided, the model may behave randomly\n"); } auto & rd = res->rd; diff --git a/tools/tts/README.md b/tools/tts/README.md index dd84336c39..612f555e9d 100644 --- a/tools/tts/README.md +++ b/tools/tts/README.md @@ -12,6 +12,8 @@ Simple usage: llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF -p "Hello world" --output out.wav ``` +Note: it's highly recommended to always provide a speaker reference voice (via `--tts-speaker-file`); otherwise, the model's performance may be degraded. + Common params: - Sampling params such as `--top-k`, `--top-p`, `--temp`, etc. - `-n ` limits the output length, e.g. `-n 500`. Note that how many milliseconds each frame represents varies by model