Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-20 01:31:31 +02:00
Code Issues Packages Projects Releases Wiki Activity

load gen_code model ok

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-07-29 21:38:24 +02:00
parent ed0b0503d6
commit bd38cb5cc2
8 changed files with 98 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conversion/qwen3tts.py
+3
View File
@@ -235,7 +235,10 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
# handle code predictor config
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.QWEN3TTS_GEN)
code_predictor_config = self.global_config["talker_config"]["code_predictor_config"]
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
self.gguf_writer.add_gen_audio_embedding_length(code_predictor_config["hidden_size"])
self.gguf_writer.add_gen_audio_feed_forward_length(code_predictor_config["intermediate_size"])
self.gguf_writer.add_gen_audio_block_count(code_predictor_config["num_hidden_layers"])
gguf-py/gguf/constants.py
+6 -2
View File
@@ -322,6 +322,7 @@ class Keys:
PROJECTOR_TYPE = "clip.projector_type"
HAS_VISION_ENCODER = "clip.has_vision_encoder"
HAS_AUDIO_ENCODER = "clip.has_audio_encoder"
HAS_GEN_AUDIO_ENCODER = "clip.has_gen_audio_encoder"
HAS_LLAVA_PROJECTOR = "clip.has_llava_projector"
class ClipVision:
@@ -396,9 +397,11 @@ class Keys:
HEAD_COUNT = "clip.audio.projector.head_count"
class ClipGenAudio:
PROJECTOR_TYPE = "clip.gen.audio.projector_type" # for mixed modality models
EMBEDDING_LENGTH = "clip.gen.audio.embedding_length"
FEED_FORWARD_LENGTH = "clip.gen.audio.feed_forward_length"
BLOCK_COUNT = "clip.gen.audio.block_count"
PROJECTION_DIM = "clip.gen.audio.projection_dim"
class Attention:
HEAD_COUNT = "clip.gen.audio.attention.head_count"
@@ -1607,14 +1610,14 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.A_GEN_CODE_EMBD: "a.gen.code.embd",
MODEL_TENSOR.A_GEN_CODE_HEAD: "a.gen.code.head",
MODEL_TENSOR.A_GEN_CODE_OUT_EMBD: "a.gen.code.out_embd",
MODEL_TENSOR.A_GEN_CODE_ATTN_NORM: "a.gen.code.blk.{bid}.attn_norm",
MODEL_TENSOR.A_GEN_CODE_ATTN_NORM: "a.gen.code.blk.{bid}.ln1", # reuses the generic clip.cpp block loader (TN_LN_1)
MODEL_TENSOR.A_GEN_CODE_ATTN_Q: "a.gen.code.blk.{bid}.attn_q",
MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM: "a.gen.code.blk.{bid}.attn_q_norm",
MODEL_TENSOR.A_GEN_CODE_ATTN_K: "a.gen.code.blk.{bid}.attn_k",
MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM: "a.gen.code.blk.{bid}.attn_k_norm",
MODEL_TENSOR.A_GEN_CODE_ATTN_V: "a.gen.code.blk.{bid}.attn_v",
MODEL_TENSOR.A_GEN_CODE_ATTN_OUT: "a.gen.code.blk.{bid}.attn_out",
MODEL_TENSOR.A_GEN_CODE_FFN_NORM: "a.gen.code.blk.{bid}.ffn_norm",
MODEL_TENSOR.A_GEN_CODE_FFN_NORM: "a.gen.code.blk.{bid}.ln2", # reuses the generic clip.cpp block loader (TN_LN_2)
MODEL_TENSOR.A_GEN_CODE_FFN_GATE: "a.gen.code.blk.{bid}.ffn_gate",
MODEL_TENSOR.A_GEN_CODE_FFN_UP: "a.gen.code.blk.{bid}.ffn_up",
MODEL_TENSOR.A_GEN_CODE_FFN_DOWN: "a.gen.code.blk.{bid}.ffn_down",
@@ -4965,6 +4968,7 @@ class VisionProjectorType:
YOUTUVL = "youtuvl"
NEMOTRON_V2_VL = "nemotron_v2_vl"
QWEN3TTS_SPKENC = "qwen3tts_spkenc" # audio: ECAPA-TDNN speaker encoder
QWEN3TTS_GEN = "qwen3tts_gen" # audio generation: code_predictor
HUNYUANVL = "hunyuanvl"
PARAKEET = "parakeet" # audio
MINIMAXM3 = "minimax_m3"
gguf-py/gguf/gguf_writer.py
+12 -3
View File
@@ -1199,6 +1199,9 @@ class GGUFWriter:
def add_clip_has_audio_encoder(self, value: bool) -> None:
self.add_bool(Keys.Clip.HAS_AUDIO_ENCODER, value)
def add_clip_has_gen_audio_encoder(self, value: bool) -> None:
self.add_bool(Keys.Clip.HAS_GEN_AUDIO_ENCODER, value)
def add_clip_projector_type(self, value: str) -> None:
self.add_string(Keys.Clip.PROJECTOR_TYPE, value)
@@ -1398,11 +1401,17 @@ class GGUFWriter:
def add_audio_projector_head_count(self, value: int) -> None:
self.add_uint32(Keys.ClipAudio.Projector.HEAD_COUNT, value)
def add_gen_audio_embedding_length(self, value: int) -> None:
self.add_uint32(Keys.ClipGenAudio.EMBEDDING_LENGTH, value)
# audio generation (mmproj)
def add_clip_gen_audio_projector_type(self, value: str) -> None:
self.add_string(Keys.ClipGenAudio.PROJECTOR_TYPE, value)
def add_gen_audio_projection_dim(self, value: int) -> None:
self.add_uint32(Keys.ClipGenAudio.PROJECTION_DIM, value)
def add_gen_audio_embedding_length(self, value: int) -> None:
self.add_uint32(Keys.ClipGenAudio.EMBEDDING_LENGTH, value)
def add_gen_audio_feed_forward_length(self, value: int) -> None:
self.add_uint32(Keys.ClipGenAudio.FEED_FORWARD_LENGTH, value)
tools/mtmd/clip-impl.h
+13 -1
View File
@@ -32,6 +32,7 @@
#define KEY_PROJ_TYPE "clip.projector_type"
#define KEY_HAS_AUDIO_ENC "clip.has_audio_encoder"
#define KEY_HAS_VISION_ENC "clip.has_vision_encoder"
#define KEY_HAS_GEN_AUDIO_ENC "clip.has_gen_audio_encoder"
#define KEY_USE_GELU "clip.use_gelu"
#define KEY_USE_SILU "clip.use_silu"
@@ -88,6 +89,8 @@
#define KEY_A_ATTN_WINDOW_SIZE "clip.audio.window_size" // mimo-audio-tokenizer: sliding-window radius
#define KEY_A_LOCAL_BLOCK_COUNT "clip.audio.local_block_count" // mimo-v2.5: input_local_transformer layer count
#define KEY_A_LOCAL_GROUP_SIZE "clip.audio.local_group_size" // mimo-v2.5: input_local_transformer grouping size
// audio generation (gen-audio)-specific
#define KEY_GEN_AUDIO_PROJ_TYPE "clip.gen.audio.projector_type" // for models with mixed modalities
#define KEY_AUDIO_SUBSAMPLING_FACTOR "clip.audio.subsampling_factor"
//
@@ -207,6 +210,13 @@
#define TN_A_ASP_ATTN "a.asp_attn.%s"
#define TN_A_ASP_TDNN "a.asp_tdnn.%s"
// qwen3tts code_predictor
#define TN_A_GEN_CODE_PROJ_IN "a.gen.code.proj_in.%s"
#define TN_A_GEN_CODE_EMBD "a.gen.code.embd.%s"
#define TN_A_GEN_CODE_HEAD "a.gen.code.head.%s"
#define TN_A_GEN_CODE_OUT_EMBD "a.gen.code.out_embd.%s"
#define TN_A_GEN_CODE_NORM "a.gen.code.output_norm.%s"
// cogvlm
#define TN_MM_POST_FC_NORM "mm.post_fc_norm.%s"
#define TN_MM_H_TO_4H "mm.up.%s"
@@ -415,6 +425,7 @@ enum projector_type {
PROJECTOR_TYPE_GRANITE4_VISION,
PROJECTOR_TYPE_MIMO_AUDIO,
PROJECTOR_TYPE_QWEN3TTS_SPKENC,
PROJECTOR_TYPE_QWEN3TTS_GEN,
PROJECTOR_TYPE_UNKNOWN,
};
@@ -471,8 +482,9 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_MINIMAX_M3, "minimax_m3"},
{ PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"},
{ PROJECTOR_TYPE_MIMO_AUDIO, "mimo_audio"},
{ PROJECTOR_TYPE_QWEN3TTS_SPKENC, "qwen3tts_spkenc"},
{ PROJECTOR_TYPE_PARAKEET, "parakeet"},
{ PROJECTOR_TYPE_QWEN3TTS_SPKENC, "qwen3tts_spkenc"},
{ PROJECTOR_TYPE_QWEN3TTS_GEN, "qwen3tts_gen"},
};
static projector_type clip_projector_type_from_string(const std::string & str) {
tools/mtmd/clip-model.h
+8
View File
@@ -592,6 +592,14 @@ struct clip_model {
ggml_tensor * spk_asp_tdnn_w = nullptr;
ggml_tensor * spk_asp_tdnn_b = nullptr;
// qwen3tts code_predictor
ggml_tensor * gen_code_proj_in_w = nullptr; // small_to_mtp_projection
ggml_tensor * gen_code_proj_in_b = nullptr;
ggml_tensor * gen_code_embd_w = nullptr; // per-codebook embedding, merged 3D
ggml_tensor * gen_code_head_w = nullptr; // per-codebook output head, merged 3D
ggml_tensor * gen_code_out_embd_w = nullptr; // codebook-0 embedding, fed back into the talker
ggml_tensor * gen_code_norm_w = nullptr; // final norm
// cogvlm
ggml_tensor * mm_post_fc_norm_w = nullptr;
ggml_tensor * mm_post_fc_norm_b = nullptr;
tools/mtmd/clip.cpp
+49 -10
View File
@@ -1069,8 +1069,9 @@ struct clip_model_loader {
size_t model_size = 0; // in bytes
bool has_vision = false;
bool has_audio = false;
bool has_vision = false;
bool has_audio = false;
bool has_gen_audio = false;
mtmd_progress_callback progress_callback = nullptr;
void * progress_callback_user_data = nullptr;
@@ -1116,8 +1117,9 @@ struct clip_model_loader {
// modalities
{
get_bool(KEY_HAS_VISION_ENC, has_vision, false);
get_bool(KEY_HAS_AUDIO_ENC, has_audio, false);
get_bool(KEY_HAS_VISION_ENC, has_vision, false);
get_bool(KEY_HAS_AUDIO_ENC, has_audio, false);
get_bool(KEY_HAS_GEN_AUDIO_ENC, has_gen_audio, false);
if (has_vision) {
LOG_INF("%s: has vision encoder\n", __func__);
@@ -1125,6 +1127,9 @@ struct clip_model_loader {
if (has_audio) {
LOG_INF("%s: has audio encoder\n", __func__);
}
if (has_gen_audio) {
LOG_INF("%s: has audio generation (gen) encoder\n", __func__);
}
}
// tensors
@@ -1151,6 +1156,8 @@ struct clip_model_loader {
GGML_ASSERT(has_vision);
} else if (modality == CLIP_MODALITY_AUDIO) {
GGML_ASSERT(has_audio);
} else if (modality == CLIP_MODALITY_GEN_AUDIO) {
GGML_ASSERT(has_gen_audio);
}
model.modality = modality;
@@ -1167,6 +1174,8 @@ struct clip_model_loader {
get_string(KEY_VISION_PROJ_TYPE, proj_type, false);
} else if (modality == CLIP_MODALITY_AUDIO) {
get_string(KEY_AUDIO_PROJ_TYPE, proj_type, false);
} else if (modality == CLIP_MODALITY_GEN_AUDIO) {
get_string(KEY_GEN_AUDIO_PROJ_TYPE, proj_type, false);
} else {
GGML_ABORT("unknown modality");
}
@@ -1186,12 +1195,13 @@ struct clip_model_loader {
}
}
const bool is_vision = model.modality == CLIP_MODALITY_VISION;
const bool is_audio = model.modality == CLIP_MODALITY_AUDIO;
const bool is_vision = model.modality == CLIP_MODALITY_VISION;
const bool is_audio = model.modality == CLIP_MODALITY_AUDIO;
const bool is_gen_audio = model.modality == CLIP_MODALITY_GEN_AUDIO;
// other hparams
{
const char * prefix = is_vision ? "vision" : "audio";
const char * prefix = is_vision ? "vision" : (is_audio ? "audio" : "gen.audio");
get_u32(string_format(KEY_N_EMBD, prefix), hparams.n_embd);
get_u32(string_format(KEY_N_HEAD, prefix), hparams.n_head);
get_u32(string_format(KEY_N_FF, prefix), hparams.n_ff);
@@ -1229,6 +1239,11 @@ struct clip_model_loader {
hparams.image_size = 0;
hparams.patch_size = 1;
} else if (is_gen_audio) {
// these are unused, but still need to be set to avoid issues
hparams.image_size = 0;
hparams.patch_size = 1;
} else {
GGML_ASSERT(false && "unknown modality");
}
@@ -1658,6 +1673,10 @@ struct clip_model_loader {
hparams.audio_window_len = 1024;
hparams.audio_hop_len = 256;
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
{
// discrete-token autoregressive predictor, no mel-frontend needed
} break;
case PROJECTOR_TYPE_PADDLEOCR:
{
hparams.n_merge = 2;
@@ -1882,7 +1901,9 @@ struct clip_model_loader {
}
// TODO @ngxson : support both audio and video in the future
const char * prefix = model.modality == CLIP_MODALITY_AUDIO ? "a" : "v";
const char * prefix = model.modality == CLIP_MODALITY_AUDIO ? "a"
: model.modality == CLIP_MODALITY_GEN_AUDIO ? "a.gen.code"
: "v";
// get offsets
for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) {
@@ -2647,6 +2668,16 @@ struct clip_model_loader {
model.mm_fc_w = get_tensor(string_format(TN_MM_AUDIO_FC, "weight"));
model.mm_fc_b = get_tensor(string_format(TN_MM_AUDIO_FC, "bias"));
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
{
// code_predictor
model.gen_code_proj_in_w = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "weight"));
model.gen_code_proj_in_b = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "bias"));
model.gen_code_embd_w = get_tensor(string_format(TN_A_GEN_CODE_EMBD, "weight"));
model.gen_code_head_w = get_tensor(string_format(TN_A_GEN_CODE_HEAD, "weight"));
model.gen_code_out_embd_w = get_tensor(string_format(TN_A_GEN_CODE_OUT_EMBD, "weight"));
model.gen_code_norm_w = get_tensor(string_format(TN_A_GEN_CODE_NORM, "weight"));
} break;
case PROJECTOR_TYPE_VOXTRAL:
{
model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
@@ -3475,6 +3506,7 @@ struct clip_model_loader {
struct clip_init_result clip_init(const char * fname, struct clip_context_params ctx_params) {
clip_ctx * ctx_vision = nullptr;
clip_ctx * ctx_audio = nullptr;
clip_ctx * ctx_gen_audio = nullptr;
try {
clip_model_loader loader(fname,
@@ -3507,16 +3539,23 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params
}
}
if (loader.has_gen_audio) {
ctx_gen_audio = new clip_ctx(ctx_params);
loader.load_hparams(ctx_gen_audio->model, CLIP_MODALITY_GEN_AUDIO);
loader.load_tensors(*ctx_gen_audio);
}
} catch (const std::exception & e) {
LOG_ERR("%s: failed to load model '%s': %s\n", __func__, fname, e.what());
delete ctx_vision;
delete ctx_audio;
delete ctx_gen_audio;
return {nullptr, nullptr};
return {nullptr, nullptr, nullptr};
}
return {ctx_vision, ctx_audio};
return {ctx_vision, ctx_audio, ctx_gen_audio};
}
struct clip_cap clip_get_cap(const char * fname) {
tools/mtmd/clip.h
+2
View File
@@ -37,6 +37,7 @@ struct clip_image_f32_batch;
enum clip_modality {
CLIP_MODALITY_VISION,
CLIP_MODALITY_AUDIO,
CLIP_MODALITY_GEN_AUDIO,
};
enum clip_flash_attn_type {
@@ -61,6 +62,7 @@ struct clip_context_params {
struct clip_init_result {
struct clip_ctx * ctx_v; // vision context
struct clip_ctx * ctx_a; // audio context
struct clip_ctx * ctx_gen_a; // audio generation context
};
struct clip_init_result clip_init(const char * fname, struct clip_context_params ctx_params);
tools/mtmd/mtmd.cpp
+5
View File
@@ -262,6 +262,9 @@ struct mtmd_context {
struct clip_ctx * ctx_a; // audio
std::vector<float> out_embd; // image embedding vector
// generation context
struct clip_ctx * ctx_gen_a; // audio
bool print_timings;
int n_threads;
std::string media_marker;
@@ -354,6 +357,7 @@ struct mtmd_context {
auto res = clip_init(mmproj_fname, ctx_clip_params);
ctx_v = res.ctx_v;
ctx_a = res.ctx_a;
ctx_gen_a = res.ctx_gen_a;
if (!ctx_v && !ctx_a) {
throw std::runtime_error(string_format("Failed to load CLIP model from %s\n", mmproj_fname));
}
@@ -784,6 +788,7 @@ struct mtmd_context {
~mtmd_context() {
clip_free(ctx_a);
clip_free(ctx_v);
clip_free(ctx_gen_a);
}
private: