Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-20 01:31:31 +02:00
Code Issues Packages Projects Releases Wiki Activity

convert encoder ok

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-07-29 01:02:45 +02:00
parent 8892b6c60b
commit de0ac58c1f
4 changed files with 87 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conversion/__init__.py
+1
View File
@@ -303,6 +303,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"Qwen2_5_VLForConditionalGeneration": "qwenvl",
"Qwen3ASRForConditionalGeneration": "qwen3vl",
"Qwen3OmniMoeForConditionalGeneration": "qwen3vl",
"Qwen3TTSForConditionalGeneration": "qwen3tts",
"Qwen3VLForConditionalGeneration": "qwen3vl",
"Qwen3VLMoeForConditionalGeneration": "qwen3vl",
"Qwen3_5ForConditionalGeneration": "qwen3vl",
conversion/qwen3tts.py
+48 -13
View File
@@ -1,14 +1,14 @@
from __future__ import annotations
from pathlib import Path
from typing import Callable, Iterable, TYPE_CHECKING
from typing import Any, Callable, Iterable, TYPE_CHECKING
import torch.nn.functional as F
if TYPE_CHECKING:
from torch import Tensor
from .base import ModelBase, TextModel, gguf, logger
from .base import ModelBase, MmprojModel, TextModel, gguf, logger
# torch activation functions used by Qwen3TTSTalkerResizeMLP (config's hidden_act)
@@ -21,10 +21,6 @@ _ACT2FN = {
@ModelBase.register("Qwen3TTSForConditionalGeneration")
class Qwen3TTSTalkerModel(TextModel):
"""Converts only the talker's backbone transformer (text-conditioned codec
token predictor). The speaker encoder and the small code_predictor
sub-model are not handled yet."""
model_arch = gguf.MODEL_ARCH.QWEN3TTS
_TEXT_PROJ_KEYS = (
@@ -41,11 +37,7 @@ class Qwen3TTSTalkerModel(TextModel):
hparams = kwargs.pop("hparams", None)
if hparams is None:
hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
# reuse TextModel's generic "text_config" flattening for the talker's own config
talker_config = dict(hparams["talker_config"])
# talker_config's own "vocab_size" is the codec vocab (talker.codec_head /
# talker.model.codec_embedding), not the BPE text vocab that "vocab_size" is
# normally expected to describe; use text_vocab_size (matches embed_tokens) instead
talker_config["vocab_size"] = talker_config["text_vocab_size"]
hparams["text_config"] = talker_config
super().__init__(dir_model, *args, hparams=hparams, **kwargs)
@@ -84,9 +76,7 @@ class Qwen3TTSTalkerModel(TextModel):
if len(self._text_proj_buffer) < len(self._TEXT_PROJ_KEYS):
return
# the talker only ever consumes text_embedding through text_projection
# (a 2-layer MLP: fc2(act(fc1(x)))), so fold it into the embedding table
# at conversion time instead of carrying the MLP weights around
# fold MLP into the embedding table at conversion time, MLP won't be used at inference time anyway
act_fn = _ACT2FN[self.hparams["hidden_act"]]
embed = self._text_proj_buffer["model.text_embedding.weight"]
hidden = act_fn(F.linear(embed,
@@ -99,3 +89,48 @@ class Qwen3TTSTalkerModel(TextModel):
return
yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("Qwen3TTSForConditionalGeneration")
class Qwen3TTSSpeakerEncoderModel(MmprojModel):
has_vision_encoder = False
has_audio_encoder = True
def __init__(self, dir_model: Path, *args, **kwargs):
hparams = kwargs.pop("hparams", None)
if hparams is None:
hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
hparams["text_config"] = {"hidden_size": hparams["talker_config"]["hidden_size"]}
# ECAPA-TDNN has a fixed 4-stage backbone, not a configurable transformer depth;
# MmprojModel.__init__ still needs one of the n_block_keys to build its tensor map
hparams["speaker_encoder_config"]["n_layers"] = 4
super().__init__(dir_model, *args, hparams=hparams, **kwargs)
def get_audio_config(self) -> dict[str, Any] | None:
return self.global_config.get("speaker_encoder_config")
def set_gguf_parameters(self):
self.gguf_writer.add_file_type(self.ftype)
self.gguf_writer.add_clip_has_audio_encoder(True)
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.QWEN3TTS_SPKENC)
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name, gen = item
if not name.startswith("speaker_encoder."):
return None
return super().filter_tensors((name, gen))
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
if "res2net_block.blocks." in name:
assert bid is not None # the outer stage index, picked up from the tensor name automatically
xid = int(name.split("res2net_block.blocks.")[1].split(".")[0])
suffix = "." + name.rsplit(".", 1)[1]
new_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_ENC_CONV_RES2].format(bid=bid, xid=xid) + suffix
yield (new_name, data_torch)
return
yield from super().modify_tensors(data_torch, name, bid)
gguf-py/gguf/constants.py
+16
View File
@@ -956,6 +956,11 @@ class MODEL_TENSOR(IntEnum):
A_ENC_DOWNSAMPLE_CONV = auto() # mimo-audio-tokenizer: post-transformer downsample conv
A_ENC_DOWNSAMPLE_NORM = auto() # mimo-audio-tokenizer: post-transformer downsample norm
A_ENC_RVQ_CODEBOOK = auto() # mimo-audio-tokenizer: residual vector quantizer codebook, per quantizer index
A_ENC_CONV_RES2 = auto() # qwen3tts
A_ENC_SE_CONV1 = auto() # qwen3tts
A_ENC_SE_CONV2 = auto() # qwen3tts
A_ENC_ASP_ATTN = auto() # qwen3tts
A_ENC_ASP_TDNN = auto() # qwen3tts
A_MMPROJ = auto()
A_MMPROJ_FC = auto()
A_MM_NORM_PRE = auto()
@@ -1558,6 +1563,11 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.A_ENC_DOWNSAMPLE_CONV: "a.downsample.conv",
MODEL_TENSOR.A_ENC_DOWNSAMPLE_NORM: "a.downsample.norm",
MODEL_TENSOR.A_ENC_RVQ_CODEBOOK: "a.rvq.codebook",
MODEL_TENSOR.A_ENC_CONV_RES2: "a.blk.{bid}.res2.{xid}",
MODEL_TENSOR.A_ENC_SE_CONV1: "a.blk.{bid}.se_conv1",
MODEL_TENSOR.A_ENC_SE_CONV2: "a.blk.{bid}.se_conv2",
MODEL_TENSOR.A_ENC_ASP_ATTN: "a.asp_attn",
MODEL_TENSOR.A_ENC_ASP_TDNN: "a.asp_tdnn",
MODEL_TENSOR.A_MMPROJ: "mm.a.mlp.{bid}",
MODEL_TENSOR.A_MMPROJ_FC: "mm.a.fc",
MODEL_TENSOR.A_MM_NORM_PRE: "mm.a.norm_pre",
@@ -1805,6 +1815,11 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.A_ENC_CONV_NORM,
MODEL_TENSOR.A_ENC_CONV_PW1,
MODEL_TENSOR.A_ENC_CONV_PW2,
MODEL_TENSOR.A_ENC_CONV_RES2,
MODEL_TENSOR.A_ENC_SE_CONV1,
MODEL_TENSOR.A_ENC_SE_CONV2,
MODEL_TENSOR.A_ENC_ASP_ATTN,
MODEL_TENSOR.A_ENC_ASP_TDNN,
MODEL_TENSOR.A_MM_INP_PROJ,
MODEL_TENSOR.A_MM_SOFT_EMB_NORM,
MODEL_TENSOR.A_MM_EMBEDDING,
@@ -4867,6 +4882,7 @@ class VisionProjectorType:
GLM4V = "glm4v"
YOUTUVL = "youtuvl"
NEMOTRON_V2_VL = "nemotron_v2_vl"
QWEN3TTS_SPKENC = "qwen3tts_spkenc" # audio: ECAPA-TDNN speaker encoder
HUNYUANVL = "hunyuanvl"
MINIMAXM3 = "minimax_m3"
MINICPMV4_6 = "minicpmv4_6"
gguf-py/gguf/tensor_mapping.py
+22 -1
View File
@@ -2096,6 +2096,7 @@ class TensorNameMap:
"model.audio_tower.subsample_conv_projection.conv_{bid}.conv", # gemma3n
"conformer.subsample_conv_projection.layer{bid}.conv", # gemma4
"encoder.conv{bid}", # mimo-audio-tokenizer
"speaker_encoder.blocks.{bid}.conv", # qwen3tts speaker encoder (only bid=0, the stem TDNN)
),
MODEL_TENSOR.A_ENC_CONV1D_NORM: (
@@ -2113,6 +2114,7 @@ class TensorNameMap:
MODEL_TENSOR.A_ENC_CONV_OUT: (
"audio_tower.conv_out", # qwen3omni
"speaker_encoder.mfa.conv", # qwen3tts speaker encoder: multi-layer feature aggregation
),
MODEL_TENSOR.A_PRE_NORM: (),
@@ -2306,7 +2308,8 @@ class TensorNameMap:
MODEL_TENSOR.A_MMPROJ_FC: (
"audio.multi_modal_projector.linear", # qwen2audio
"audio_tower.proj", # qwen2omni
"model.audio_tower.output_proj" # gemma4
"model.audio_tower.output_proj", # gemma4
"speaker_encoder.fc", # qwen3tts speaker encoder: final speaker embedding projection
),
MODEL_TENSOR.A_MM_NORM_PRE: (
@@ -2369,12 +2372,30 @@ class TensorNameMap:
"conformer.layers.{bid}.conv.pointwise_conv1", # lfm2
"conformer.layers.{bid}.lconv1d.linear_start", # gemma3n
"encoder.layers.{bid}.conv.up_conv", # granite_speech
"speaker_encoder.blocks.{bid}.tdnn1.conv", # qwen3tts speaker encoder
),
MODEL_TENSOR.A_ENC_CONV_PW2: (
"conformer.layers.{bid}.conv.pointwise_conv2", # lfm2
"conformer.layers.{bid}.lconv1d.linear_end", # gemma3n
"encoder.layers.{bid}.conv.down_conv", # granite_speech
"speaker_encoder.blocks.{bid}.tdnn2.conv", # qwen3tts speaker encoder
),
MODEL_TENSOR.A_ENC_SE_CONV1: (
"speaker_encoder.blocks.{bid}.se_block.conv1", # qwen3tts
),
MODEL_TENSOR.A_ENC_SE_CONV2: (
"speaker_encoder.blocks.{bid}.se_block.conv2", # qwen3tts
),
MODEL_TENSOR.A_ENC_ASP_ATTN: (
"speaker_encoder.asp.conv", # qwen3tts
),
MODEL_TENSOR.A_ENC_ASP_TDNN: (
"speaker_encoder.asp.tdnn.conv", # qwen3tts
),
MODEL_TENSOR.A_ENC_NORM_CONV: (