diff --git a/expose.h b/expose.h index 67d12707f..3d1da4fd0 100644 --- a/expose.h +++ b/expose.h @@ -323,6 +323,7 @@ struct tts_generation_inputs const char * custom_speaker_data = ""; const char * reference_audio = ""; const char * speaker_instruction = ""; + const char * language = ""; const bool use_mp3 = false; }; struct tts_generation_outputs diff --git a/koboldcpp.py b/koboldcpp.py index c8c70d63a..05fcd6001 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -505,6 +505,7 @@ class tts_generation_inputs(ctypes.Structure): ("custom_speaker_data", ctypes.c_char_p), ("reference_audio", ctypes.c_char_p), ("speaker_instruction", ctypes.c_char_p), + ("language", ctypes.c_char_p), ("use_mp3", ctypes.c_bool)] class tts_generation_outputs(ctypes.Structure): @@ -3068,10 +3069,12 @@ def tts_generate(genparams): inputs = tts_generation_inputs() inputs.custom_speaker_voice = normalized_voice.encode("UTF-8") ttsinstruction = genparams.get("instruction", "") + ttslang = genparams.get("language", "en") # if no instruction provided, extract from text if not genparams.get("instruction", ""): prompt, ttsinstruction = tts_extract_instruction(prompt) inputs.speaker_instruction = ttsinstruction.encode("UTF-8") + inputs.language = ttslang.encode("UTF-8") response_format_mp3 = True if (genparams.get("response_format")=="mp3") else False inputs.use_mp3 = genparams.get("use_mp3", response_format_mp3) inputs.prompt = prompt.encode("UTF-8") diff --git a/otherarch/qwen3tts/qwen3_tts.cpp b/otherarch/qwen3tts/qwen3_tts.cpp index c3c92a155..86174b9a6 100644 --- a/otherarch/qwen3tts/qwen3_tts.cpp +++ b/otherarch/qwen3tts/qwen3_tts.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -73,6 +74,46 @@ void Qwen3TTS::set_seed(int seed) this->transformer_.set_seed(seed); } +static int q3ttslang = 2050; +void Qwen3TTS::set_language(std::string lang) +{ + q3ttslang = 2050; //fallback to english + + for (char & c : lang) { + c = (char)std::tolower((unsigned char)c); + } + + const size_t region_pos = lang.find_first_of("-_"); + if (region_pos != std::string::npos) { + lang.resize(region_pos); + } + + struct lang_code { + const char * name; + int code; + }; + + static const lang_code codes[] = { + {"en", 2050}, {"eng", 2050}, {"english", 2050}, + {"zh", 2055}, {"zho", 2055}, {"chi", 2055}, {"chinese", 2055}, {"mandarin", 2055}, + {"de", 2053}, {"deu", 2053}, {"ger", 2053}, {"german", 2053}, + {"es", 2054}, {"spa", 2054}, {"spanish", 2054}, + {"ja", 2058}, {"jpn", 2058}, {"japanese", 2058}, + {"fr", 2061}, {"fra", 2061}, {"fre", 2061}, {"french", 2061}, + {"ko", 2064}, {"kor", 2064}, {"korean", 2064}, + {"ru", 2069}, {"rus", 2069}, {"russian", 2069}, + {"it", 2070}, {"ita", 2070}, {"italian", 2070}, + {"pt", 2071}, {"por", 2071}, {"portuguese", 2071}, + }; + + for (const lang_code & entry : codes) { + if (lang == entry.name) { + q3ttslang = entry.code; + return; + } + } +} + bool Qwen3TTS::load_models(const std::string & model_dir) { // Construct model paths std::string tts_model_path = model_dir + "/qwen3-tts-0.6b-f16.gguf"; @@ -343,7 +384,7 @@ tts_result Qwen3TTS::synthesize_internal(const std::string & text, const std::st std::vector speech_codes; if (!transformer_.generate(text_tokens.data(), (int32_t)text_tokens.size(), speaker_embedding, params.max_audio_tokens, speech_codes, - 2050, params.repetition_penalty, + q3ttslang, params.repetition_penalty, params.temperature, params.top_k, speakerid, instruct_tok_data, instruct_tok_count)) { result.error_msg = "Failed to generate speech codes: " + transformer_.get_error(); return result; diff --git a/otherarch/qwen3tts/qwen3_tts.h b/otherarch/qwen3tts/qwen3_tts.h index f8b535ed0..0f68843ec 100644 --- a/otherarch/qwen3tts/qwen3_tts.h +++ b/otherarch/qwen3tts/qwen3_tts.h @@ -82,6 +82,7 @@ public: ~Qwen3TTS(); void set_seed(int seed); + void set_language(std::string lang); // Load all models from directory // model_dir should contain: transformer.gguf, tokenizer.gguf, vocoder.gguf diff --git a/otherarch/tts_adapter.cpp b/otherarch/tts_adapter.cpp index cf61b2241..00d701995 100644 --- a/otherarch/tts_adapter.cpp +++ b/otherarch/tts_adapter.cpp @@ -1178,6 +1178,7 @@ static tts_generation_outputs ttstype_generate_qwen3tts(const tts_generation_inp std::string custom_reference_audio_str = inputs.reference_audio; std::vector custom_reference_audio_pcmf32; std::string speaker_instruction = inputs.speaker_instruction; + std::string ttslanguage = inputs.language; int speakerID = inputs.speaker_seed; //{"aiden":2861, "dylan":2878, "eric":2875, "ono_anna":2873,"ryan":3061, "serena":3066, "sohee":2864, "uncle_fu":3010, "vivian":3065} @@ -1200,6 +1201,7 @@ static tts_generation_outputs ttstype_generate_qwen3tts(const tts_generation_inp printf("\nUsing Audio Seed: %d, SpeakerID: %d", audio_seed, speakerID); } qwen3tts_runner.set_seed(audio_seed); + qwen3tts_runner.set_language(ttslanguage); if(custom_reference_audio_str!="") {