upload speaker_ref via form-data

tools/mtmd/mtmd-helper-gen.cpp

@@ -401,7 +401,7 @@ private:
             LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n");
             return false;
         }
-        const std::string  marker = mtmd_default_marker();
+        const std::string  marker = mtmd_get_marker(mctx);
         mtmd_input_text     text{ marker.c_str(), marker.size(), false, true };
         mtmd_input_chunks * chunks = mtmd_input_chunks_init();
         const mtmd_bitmap * bptr = bitmap;

tools/server/README.md

@@ -739,7 +739,7 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For more info
 
 `lang`: Language code for the utterance (model-dependent, e.g. `en`, `zh`). Optional.
 
-`speaker_ref_b64`: Base64-encoded reference WAV to clone the speaker's voice. Optional.
+`speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) — if both are provided, the uploaded file takes precedence.
 
 `top_k`, `top_p`: Sampling params for the acoustic code predictor. Optional, model-dependent defaults apply.
 
@@ -756,12 +756,21 @@ Note: it's highly recommended to always provide a speaker reference voice; other
 *Examples:*
 
 ```shell
-curl -X POST http://127.0.0.1:8012/tts \
+curl -X POST http://127.0.0.1:9931/tts \
     -H "Content-Type: application/json" \
     -d '{"input": "Hello, this is a test."}' \
     -o output.wav
 ```
 
+With a speaker reference uploaded as a file (`multipart/form-data`), instead of base64-encoding it into the JSON body:
+
+```shell
+curl -X POST http://127.0.0.1:9931/tts \
+    -F "input=Hello, this is a test." \
+    -F "speaker_ref=@/path/to/speaker-reference.wav;type=audio/wav" \
+    -o output.wav
+```
+
 ### POST `/infill`: For code infilling.
 
 Takes a prefix and a suffix and returns the predicted completion as stream.

tools/server/server-context.cpp

@@ -5237,8 +5237,6 @@ void server_routes::init_routes() {
         task.params.stream    = stream;
         task.params.n_predict = json_value(body, "n_predict", -1);
         task.params.sampling  = params.sampling; // baseline defaults, then apply overrides below
-        // codec token streams need repetition penalty over the whole generation, or the
-        // backbone loops and re-generates the same utterance (see LLAMA_EXAMPLE_TTS in arg.cpp)
         task.params.sampling.penalty_repeat  = json_value(body, "repeat_penalty", 1.05f);
         task.params.sampling.penalty_last_n  = -1;
         if (task.tts_inp.data.top_k > 0) {
@@ -5248,15 +5246,33 @@ void server_routes::init_routes() {
             task.params.sampling.top_p = task.tts_inp.data.top_p;
         }
 
-        std::string speaker_ref_b64 = json_value(body, "speaker_ref_b64", std::string());
-        if (!speaker_ref_b64.empty()) {
-            std::string bytes = base64::decode(speaker_ref_b64);
-            auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, (const unsigned char *) bytes.data(), bytes.size(), false);
+        // speaker reference: either an uploaded form file ("speaker_ref") or a base64 JSON field ("speaker_ref_b64")
+        const unsigned char * speaker_ref_data = nullptr;
+        size_t speaker_ref_len  = 0;
+        std::string speaker_ref_b64_decoded;
+
+        auto speaker_ref_file = req.files.find("speaker_ref");
+        if (speaker_ref_file != req.files.end()) {
+            speaker_ref_data = speaker_ref_file->second.data.data();
+            speaker_ref_len  = speaker_ref_file->second.data.size();
+        } else {
+            std::string speaker_ref_b64 = json_value(body, "speaker_ref_b64", std::string());
+            if (!speaker_ref_b64.empty()) {
+                speaker_ref_b64_decoded = base64::decode(speaker_ref_b64);
+                speaker_ref_data = (const unsigned char *) speaker_ref_b64_decoded.data();
+                speaker_ref_len  = speaker_ref_b64_decoded.size();
+            }
+        }
+
+        if (speaker_ref_len > 0) {
+            auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false);
             if (!wrapper.bitmap) {
-                res->error(format_error_response("failed to decode \"speaker_ref_b64\"", ERROR_TYPE_INVALID_REQUEST));
+                res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST));
                 return res;
             }
             task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap));
+        } else {
+            SRV_WRN("no speaker reference provided, the model may behave randomly\n");
         }
 
         auto & rd = res->rd;

tools/tts/README.md

@@ -12,6 +12,8 @@ Simple usage:
 llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF -p "Hello world" --output out.wav
 ```
 
+Note: it's highly recommended to always provide a speaker reference voice (via `--tts-speaker-file`); otherwise, the model's performance may be degraded.
+
 Common params:
 - Sampling params such as `--top-k`, `--top-p`, `--temp`, etc.
 - `-n <number_of_frames>` limits the output length, e.g. `-n 500`. Note that how many milliseconds each frame represents varies by model