Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-20 01:31:31 +02:00
Code Issues Packages Projects Releases Wiki Activity

clean up

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-08-03 00:16:40 +02:00
parent d827cdedcf
commit 2f240361fd
5 changed files with 23 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip.cpp
+3 -3
View File
@@ -1056,7 +1056,7 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
{
const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_CODE_GEN;
const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE;
const int top_k = params ? params->top_k : 50;
const float top_p = params ? params->top_p : 1.0f;
builder = std::make_unique<clip_graph_qwen3tts_gen>(ctx, img, gen_process, top_k, top_p);
@@ -4172,7 +4172,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
}
set_input_f32("inp_raw", inp_raw);
} else if (!(ctx->proj_type() == PROJECTOR_TYPE_QWEN3TTS_GEN && params->gen_process == CLIP_GEN_PROCESS_CODE2WAV)) {
} else if (!(ctx->proj_type() == PROJECTOR_TYPE_QWEN3TTS_GEN && params->gen_process == CLIP_GEN_PROCESS_GEN_WAV)) {
// audio input (code2wav has no hidden-state/raw input at all, its only input is the "inp_codes" tensor handled in the switch below)
GGML_ASSERT(imgs.entries.size() == 1);
@@ -4735,7 +4735,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
{
if (params->gen_process == CLIP_GEN_PROCESS_CODE2WAV) {
if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) {
GGML_ASSERT(params->codes != nullptr);
// reorder frame-major input to the group-major layout the graph wants,
tools/mtmd/clip.h
+8 -7
View File
@@ -87,19 +87,20 @@ bool clip_image_encode (struct clip_ctx * ctx, int n_threads, const clip_im
bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, std::vector<float> & out_batch_embd);
enum clip_gen_process_type {
CLIP_GEN_PROCESS_CODE_GEN, // h_state to codes
CLIP_GEN_PROCESS_CODE2WAV, // codes to raw PCM audio
CLIP_GEN_PROCESS_GEN_UNKNOWN,
CLIP_GEN_PROCESS_GEN_CODE, // h_state to codes
CLIP_GEN_PROCESS_GEN_WAV, // codes to raw PCM audio
};
struct clip_encode_params {
int n_threads = 1;
const clip_image_f32_batch * imgs = nullptr;
std::vector<float> * out_embd = nullptr;
// for audio gen, imgs has exactly one entry (unused content for CODE2WAV,
// for CODE_GEN it holds the hidden state from backbone, size (n_text_embd, 1))
clip_gen_process_type gen_process = CLIP_GEN_PROCESS_CODE_GEN;
// for audio gen, imgs has exactly one entry (unused content for GEN_WAV,
// for GEN_CODE it holds the hidden state from backbone, size (n_text_embd, 1))
clip_gen_process_type gen_process = CLIP_GEN_PROCESS_GEN_UNKNOWN;
// CODE_GEN: code0 is the sampled semantic code from backbone, out_codes
// GEN_CODE: code0 is the sampled semantic code from backbone, out_codes
// receives this frame's 16 sampled codes, out_embd receives the embd to
// be fed back to the backbone for the next frame
int32_t code0 = 0;
@@ -107,7 +108,7 @@ struct clip_encode_params {
float top_p = 1.0f;
std::vector<int32_t> * out_codes = nullptr;
// CODE2WAV: codes holds this frame's 16 RVQ codes, out_audio receives the
// GEN_WAV: codes holds this frame's 16 RVQ codes, out_audio receives the
// decoded PCM samples (F32). state_in is the state from the previous
// call (null or wrong size means cold start, state is zero-filled).
// state_out receives the state to pass into the next call.
tools/mtmd/models/models.h
+1 -1
View File
@@ -239,7 +239,7 @@ struct clip_graph_qwen3tts_gen : clip_graph {
// which sub-graph build() constructs, fixed at graph-build time
clip_gen_process_type gen_process;
// sampling params, fixed at graph-build time (CODE_GEN only)
// sampling params, fixed at graph-build time (GEN_CODE only)
int top_k;
float top_p;
tools/mtmd/models/qwen3tts-gen.cpp
+5 -5
View File
@@ -666,13 +666,13 @@ ggml_cgraph * clip_graph_qwen3tts_gen::build() {
int idx;
switch (gen_process) {
case CLIP_GEN_PROCESS_CODE_GEN: idx = 0; break;
case CLIP_GEN_PROCESS_CODE2WAV: idx = 1; break;
case CLIP_GEN_PROCESS_GEN_CODE: idx = 0; break;
case CLIP_GEN_PROCESS_GEN_WAV: idx = 1; break;
default: GGML_ABORT("unknown gen_process");
}
// ---- CLIP_GEN_PROCESS_CODE_GEN: backbone hidden state -> 16 RVQ codes + next-step embd ----
// fixed-size [n_mmproj_embd] input; not build_inp_raw(), since a CODE2WAV call's `img` has no hidden-state data
// ---- CLIP_GEN_PROCESS_GEN_CODE: backbone hidden state -> 16 RVQ codes + next-step embd ----
// fixed-size [n_mmproj_embd] input; not build_inp_raw(), since a GEN_WAV call's `img` has no hidden-state data
ggml_tensor * h_state = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_mmproj_embd);
ggml_set_name(h_state, "inp_raw"); // must keep this exact name, clip_encode() sets it by name
ggml_set_input(h_state);
@@ -738,7 +738,7 @@ ggml_cgraph * clip_graph_qwen3tts_gen::build() {
out_embd = ggml_reshape_2d(ctx0, out_embd, out_embd->ne[0], 1);
cb(out_embd, "gen_audio_out", -1);
// ---- CLIP_GEN_PROCESS_CODE2WAV: 16 RVQ codes -> raw PCM ----
// ---- CLIP_GEN_PROCESS_GEN_WAV: 16 RVQ codes -> raw PCM ----
const int n_frames = hparams.wav_tfm_swa; // frames per batch, == the attention window
ggml_tensor * inp_codes = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_frames, n_codes);
tools/mtmd/mtmd.cpp
+6 -6
View File
@@ -264,10 +264,10 @@ struct mtmd_context {
// generation context
struct clip_ctx * ctx_gen_a; // audio
std::vector<int32_t> gen_out_codes; // this frame's 16 sampled codes (CODE_GEN)
std::vector<float> gen_out_embd; // next-step hidden state fed back to backbone (CODE_GEN)
std::vector<float> gen_out_audio; // decoded PCM samples for the current frame (CODE2WAV)
std::vector<uint8_t> gen_out_state; // state to feed into the next CODE2WAV call
std::vector<int32_t> gen_out_codes; // this frame's 16 sampled codes (GEN_CODE)
std::vector<float> gen_out_embd; // next-step hidden state fed back to backbone (GEN_CODE)
std::vector<float> gen_out_audio; // decoded PCM samples for the current frame (GEN_WAV)
std::vector<uint8_t> gen_out_state; // state to feed into the next GEN_WAV call
bool print_timings;
int n_threads;
@@ -1621,7 +1621,7 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, const mtmd_gen_in
clip_encode_params params;
params.imgs = &batch;
params.n_threads = ctx->n_threads;
params.gen_process = CLIP_GEN_PROCESS_CODE_GEN;
params.gen_process = CLIP_GEN_PROCESS_GEN_CODE;
params.out_embd = &out_embd;
params.out_codes = &out_codes;
params.code0 = inp->code0;
@@ -1666,7 +1666,7 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, const mtmd_gen_in
clip_encode_params params;
params.imgs = &batch;
params.n_threads = ctx->n_threads;
params.gen_process = CLIP_GEN_PROCESS_CODE2WAV;
params.gen_process = CLIP_GEN_PROCESS_GEN_WAV;
params.codes = &in_codes;
params.out_audio = &ctx->gen_out_audio;
params.state_in = inp->state_data ? &in_state : nullptr;