Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 01:05:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source 
# Conflicts:
#	.github/labeler.yml
#	.github/workflows/build-self-hosted.yml
#	benches/nemotron/nemotron-dgx-spark.md
#	docs/ops.md
#	docs/ops/SYCL.csv
#	ggml/src/ggml-cpu/kleidiai/kleidiai.cpp
#	ggml/src/ggml-sycl/backend.hpp
#	ggml/src/ggml-sycl/element_wise.cpp
#	ggml/src/ggml-sycl/element_wise.hpp
#	ggml/src/ggml-sycl/ggml-sycl.cpp
#	scripts/sync-ggml.last
#	tests/test-jinja.cpp
#	tests/test-llama-archs.cpp
This commit is contained in:
Concedo
2026-03-17 14:05:23 +08:00
parent 6d3f01d139 b6c83aad55
commit f31b040941
24 changed files with 1016 additions and 318 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/chat.cpp
+4 -1
View File
@@ -1534,7 +1534,6 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
// map developer to system for all models except for GPT-OSS
workaround::map_developer_role_to_system(params.messages);
}
workaround::func_args_not_string(params.messages);
if (!tmpl.original_caps().supports_system_role) {
workaround::system_message_not_supported(params.messages);
@@ -1547,6 +1546,10 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
workaround::requires_non_null_content(params.messages);
}
if (tmpl.original_caps().supports_object_arguments) {
workaround::func_args_not_string(params.messages);
}
params.extra_context = common_chat_extra_context();
for (auto el : inputs.chat_template_kwargs) {
params.extra_context[el.first] = json::parse(el.second);
common/jinja/caps.cpp
+108 -12
View File
@@ -75,6 +75,7 @@ std::map<std::string, bool> caps::to_map() const {
{"supports_parallel_tool_calls", supports_parallel_tool_calls},
{"supports_system_role", supports_system_role},
{"supports_preserve_reasoning", supports_preserve_reasoning},
{"supports_object_arguments", supports_object_arguments},
};
}
@@ -158,9 +159,9 @@ caps caps_get(jinja::program & prog) {
}
);
JJ_DEBUG("%s\n", ">>> Running capability check: single tool support");
JJ_DEBUG("%s\n", ">>> Running capability check: single tool with object arguments support");
// case: tools support: single call
// case: tools support: single call with object arguments
caps_try_execute(
prog,
[&]() {
@@ -226,9 +227,7 @@ caps caps_get(jinja::program & prog) {
},
[&](bool success, value & messages, value & tools) {
if (!success) {
result.supports_tool_calls = false;
result.supports_tools = false;
return;
return; // Nothing can be inferred
}
auto & tool_name = tools->at(0)->at("function")->at("name");
@@ -242,16 +241,117 @@ caps caps_get(jinja::program & prog) {
caps_print_stats(tool_calls, "messages[1].tool_calls");
if (!tool_calls->stats.used) {
result.supports_tool_calls = false;
return;
}
auto & tool_arg = tool_calls->at(0)->at("function")->at("arguments")->at("arg");
caps_print_stats(tool_arg, "messages[1].tool_calls[0].function.arguments.arg");
if (tool_arg->stats.used) {
result.supports_object_arguments = true;
}
}
);
if (!result.supports_object_arguments) {
JJ_DEBUG("%s\n", ">>> Running capability check: single tool with string arguments support");
// case: tools support: single call with string arguments
caps_try_execute(
prog,
[&]() {
// messages
return json::array({
{
{"role", "user"},
{"content", "User message"},
},
{
{"role", "assistant"},
{"content", ""}, // Some templates expect content to be empty with tool calls
{"tool_calls", json::array({
{
{"id", "call00001"},
{"type", "function"},
{"function", {
{"name", "tool1"},
{"arguments", R"({"arg": "value"})"}
}}
}
})}
},
{
{"role", "tool"},
{"content", "Tool response"},
{"tool_call_id", "call00001"}
},
{
{"role", "assistant"},
{"content", "The tool response was 'tool response'"}
},
{
{"role", "user"},
{"content", "User message"},
},
});
},
[&]() {
// tools
return json::array({
{
{"name", "tool"},
{"type", "function"},
{"function", {
{"name", "tool1"},
{"description", "Tool description"},
{"parameters", {
{"type", "object"},
{"properties", {
{"arg", {
{"type", "string"},
{"description", "Arg description"},
}},
}},
{"required", json::array({ "arg" })},
}},
}},
},
});
},
[&](bool success, value & messages, value & tools) {
if (!success) {
result.supports_tool_calls = false;
result.supports_tools = false;
return;
}
auto & tool_name = tools->at(0)->at("function")->at("name");
caps_print_stats(tool_name, "tools[0].function.name");
caps_print_stats(tools, "tools");
if (!tool_name->stats.used) {
result.supports_tools = false;
}
auto & tool_calls = messages->at(1)->at("tool_calls");
caps_print_stats(tool_calls, "messages[1].tool_calls");
if (!tool_calls->stats.used) {
result.supports_tool_calls = false;
return;
}
}
);
}
JJ_DEBUG("%s\n", ">>> Running capability check: parallel tool support");
// case: tools support: parallel calls
caps_try_execute(
prog,
[&]() {
json args = json(R"({"arg": "value"})");
if (result.supports_object_arguments) {
args = json{{"arg", "value"}};
}
// messages
return json::array({
{
@@ -267,9 +367,7 @@ caps caps_get(jinja::program & prog) {
{"type", "function"},
{"function", {
{"name", "tool1"},
{"arguments", {
{"arg", "value"}
}}
{"arguments", args}
}}
},
{
@@ -277,9 +375,7 @@ caps caps_get(jinja::program & prog) {
{"type", "function"},
{"function", {
{"name", "tool1"},
{"arguments", {
{"arg", "value"}
}}
{"arguments", args}
}}
}
})}
@@ -328,7 +424,7 @@ caps caps_get(jinja::program & prog) {
return;
}
auto & tool_calls = messages->at(1)->at("tool_calls");;
auto & tool_calls = messages->at(1)->at("tool_calls");
caps_print_stats(tool_calls, "messages[1].tool_calls");
// check for second tool call usage
common/jinja/caps.h
+2
View File
@@ -18,6 +18,8 @@ struct caps {
bool supports_string_content = true;
bool supports_typed_content = false;
bool supports_object_arguments = false;
// for reporting on server
std::map<std::string, bool> to_map() const;
convert_hf_to_gguf.py
+83 -39
View File
@@ -298,11 +298,16 @@ class ModelBase:
scale = scale.float()
if block_size is not None:
dim_offset = scale.ndim - len(block_size)
for i, size in enumerate(block_size):
scale = scale.repeat_interleave(size, i)
scale = scale.repeat_interleave(size, dim_offset + i)
# unpad the scale (e.g. when the tensor size isn't a multiple of the block size)
scale = scale[tuple(slice(0, size) for size in weight.shape)]
# align scale dims to weight for correct broadcasting (e.g. [128] -> [128, 1, 1])
while scale.ndim < weight.ndim:
scale = scale.unsqueeze(-1)
return weight.float() * scale
# ref: https://github.com/ModelCloud/GPTQModel/blob/037c5c0f6c9e33c500d975b038d02e7ca437546d/gptqmodel/nn_modules/qlinear/__init__.py#L437-L476
@@ -393,7 +398,7 @@ class ModelBase:
elif quant_method == "fp8":
block_size = quant_config.get("weight_block_size")
for name in self.model_tensors.keys():
if name.endswith(".weight_scale_inv"):
if name.endswith("_scale_inv"):
weight_name = name.removesuffix("_scale_inv")
w = self.model_tensors[weight_name]
s = self.model_tensors[name]
@@ -401,6 +406,8 @@ class ModelBase:
tensors_to_remove.append(name)
if name.endswith(".activation_scale"): # unused
tensors_to_remove.append(name)
if name.endswith("_activation_scale"): # Mistral-Small-4-119B-2602, unused
tensors_to_remove.append(name)
# mistral format
if name.endswith(".qscale_weight"):
weight_name = name.removesuffix("qscale_weight") + "weight"
@@ -3031,10 +3038,16 @@ class LlavaVisionModel(MmprojModel):
def get_token_id(self, token: str) -> int:
tokenizer_config_file = self.dir_model / 'tokenizer_config.json'
with open(tokenizer_config_file, "r", encoding="utf-8") as f:
added_tokens_decoder = json.load(f)['added_tokens_decoder']
added_tokens_decoder = json.load(f).get('added_tokens_decoder') or {}
for id_, token_data in added_tokens_decoder.items():
if token_data["content"] == token:
if token_data.get("content") == token:
return int(id_)
# fallthrough to tokenizer.json
with open(self.dir_model / "tokenizer.json", "r", encoding="utf-8") as f:
tokenizer_json = json.load(f)
for token_data in tokenizer_json["added_tokens"]:
if token_data["content"] == token:
return int(token_data["id"])
raise ValueError(f"Token '{token}' not found in tokenizer config.")
def set_gguf_parameters(self):
@@ -3198,40 +3211,6 @@ class Llama4VisionModel(MmprojModel):
yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register(
"Mistral3ForConditionalGeneration",
"Ministral3ForCausalLM",
)
class Mistral3Model(LlamaModel):
model_arch = gguf.MODEL_ARCH.MISTRAL3
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# for compatibility, we use LLAMA arch for older models
# TODO: remove this once everyone has migrated to newer version of llama.cpp
if self.hparams.get("model_type") != "ministral3":
self.model_arch = gguf.MODEL_ARCH.LLAMA
self.gguf_writer.arch = gguf.MODEL_ARCH_NAMES[self.model_arch]
self.gguf_writer.add_architecture()
self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
def set_gguf_parameters(self):
super().set_gguf_parameters()
rope_params = self.rope_parameters
if self.hparams.get("model_type") == "ministral3":
assert rope_params, "ministral3 must have 'rope_parameters' config"
assert rope_params["rope_type"] == "yarn", "ministral3 rope_type must be 'yarn'"
self.gguf_writer.add_rope_scaling_yarn_log_mul(rope_params["mscale_all_dim"])
self.gguf_writer.add_attn_temperature_scale(rope_params["llama_4_scaling_beta"])
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):
name = name.replace("language_model.", "")
if "multi_modal_projector" in name or "vision_tower" in name:
return
yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("DeciLMForCausalLM")
class DeciModel(TextModel):
model_arch = gguf.MODEL_ARCH.DECI
@@ -8271,6 +8250,8 @@ class DeepseekV2Model(TextModel):
# TODO @ngxson : remove this when we support MTP for deepseek models
skip_mtp = True
merge_expert = True
def set_vocab(self):
try:
self._set_vocab_gpt2()
@@ -8409,7 +8390,7 @@ class DeepseekV2Model(TextModel):
return
# process the experts separately
if name.find("mlp.experts") != -1:
if self.merge_expert and name.find("mlp.experts") != -1:
n_experts = self.hparams["n_routed_experts"]
assert bid is not None
@@ -8468,6 +8449,69 @@ class DeepseekV2Model(TextModel):
raise ValueError(f"Unprocessed experts: {experts}")
@ModelBase.register(
"Mistral3ForConditionalGeneration",
"Ministral3ForCausalLM",
)
class Mistral3Model(TextModel):
class Ministral3Model(LlamaModel):
model_arch = gguf.MODEL_ARCH.MISTRAL3
def set_gguf_parameters(self):
super().set_gguf_parameters()
rope_params = self.rope_parameters
if self.hparams.get("model_type") == "ministral3":
assert rope_params, "ministral3 must have 'rope_parameters' config"
assert rope_params["rope_type"] == "yarn", "ministral3 rope_type must be 'yarn'"
self.gguf_writer.add_rope_scaling_yarn_log_mul(rope_params["mscale_all_dim"])
self.gguf_writer.add_attn_temperature_scale(rope_params["llama_4_scaling_beta"])
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):
name = name.replace("language_model.", "")
if "multi_modal_projector" in name or "vision_tower" in name:
return
yield from super().modify_tensors(data_torch, name, bid)
class Mistral4Model(DeepseekV2Model):
model_arch = gguf.MODEL_ARCH.MISTRAL4
skip_mtp = False # model contains no MTP layers, so no need to skip
merge_expert = False # experts are already stacked as 3D
def modify_tensors(self, data_torch, name, bid):
if name.endswith(".down_proj") or name.endswith(".gate_up_proj"):
name = name + ".weight"
yield from super().modify_tensors(data_torch, name, bid)
model_arch = gguf.MODEL_ARCH.MISTRAL3 # unused
impl: TextModel
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if self.hparams.get("model_type") == "mistral4":
self.impl = Mistral3Model.Mistral4Model(*args, **kwargs)
else:
self.impl = Mistral3Model.Ministral3Model(*args, **kwargs)
def set_vocab(self):
self.impl.set_vocab()
def set_gguf_parameters(self):
self.impl.set_gguf_parameters()
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):
yield from self.impl.modify_tensors(data_torch, name, bid)
def prepare_tensors(self):
self.impl.prepare_tensors()
def write_vocab(self):
self.impl.write_vocab()
def write(self):
self.impl.write()
@ModelBase.register("MiniMaxM2ForCausalLM")
class MiniMaxM2Model(TextModel):
model_arch = gguf.MODEL_ARCH.MINIMAXM2
ggml/src/ggml-cpu/arch/arm/quants.c
+1 -1
View File
@@ -666,7 +666,7 @@ void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo
float sumf = 0;
#if defined __ARM_NEON
#if defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA)
const int8x16_t values = vld1q_s8(kvalues_mxfp4);
const uint8x16_t m4b = vdupq_n_u8(0x0f);
float32x4_t acc = vdupq_n_f32(0.0f);
ggml/src/ggml-cpu/ops.cpp
+13 -4
View File
@@ -6205,7 +6205,7 @@ static void ggml_compute_forward_im2col_f16(
const ggml_tensor * src1 = dst->src[1];
GGML_ASSERT(src0->type == GGML_TYPE_F16);
GGML_ASSERT(src1->type == GGML_TYPE_F32);
GGML_ASSERT(src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32);
GGML_ASSERT( dst->type == GGML_TYPE_F16);
GGML_TENSOR_BINARY_OP_LOCALS;
@@ -6236,7 +6236,7 @@ static void ggml_compute_forward_im2col_f16(
int ofs1 = is_2D ? nb12 : nb11;
GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
GGML_ASSERT(nb10 == sizeof(float));
GGML_ASSERT(nb10 == ggml_type_size(src1->type));
// im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]
{
@@ -6249,7 +6249,12 @@ static void ggml_compute_forward_im2col_f16(
// micro kernel
ggml_fp16_t * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]
const float * const src_data = (float *)((char *) src1->data + in*ofs0 + iic*ofs1); // [IH, IW]
const float * const src_data_f32 = src1->type == GGML_TYPE_F32
? (const float *)((const char *) src1->data + in*ofs0 + iic*ofs1)
: nullptr; // [IH, IW]
const ggml_fp16_t * const src_data_f16 = src1->type == GGML_TYPE_F16
? (const ggml_fp16_t *)((const char *) src1->data + in*ofs0 + iic*ofs1)
: nullptr; // [IH, IW]
for (int64_t ikh = 0; ikh < KH; ikh++) { // 1
for (int64_t ikw = 0; ikw < KW; ikw++) {
@@ -6259,7 +6264,11 @@ static void ggml_compute_forward_im2col_f16(
if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {
dst_data[iic*(KH*KW) + ikh*KW + ikw] = 0;
} else {
dst_data[iic*(KH*KW) + ikh*KW + ikw] = GGML_CPU_FP32_TO_FP16(src_data[iih*IW + iiw]);
if (src_data_f32 != nullptr) {
dst_data[iic*(KH*KW) + ikh*KW + ikw] = GGML_CPU_FP32_TO_FP16(src_data_f32[iih*IW + iiw]);
} else {
dst_data[iic*(KH*KW) + ikh*KW + ikw] = src_data_f16[iih*IW + iiw];
}
}
}
}
ggml/src/ggml-sycl/upscale.cpp
+410
View File
@@ -0,0 +1,410 @@
#include "upscale.hpp"
static void upscale_f32(const float * x, float * dst,
const int nb00, const int nb01, const int nb02, const int nb03,
const int ne10, const int ne11, const int ne12, const int ne13,
const float sf0, const float sf1, const float sf2, const float sf3) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
int index = item_ct1.get_local_id(2) + item_ct1.get_group(2) * item_ct1.get_local_range(2);
if (index >= ne10 * ne11 * ne12 * ne13) {
return;
}
int i10 = index % ne10;
int i11 = (index / ne10) % ne11;
int i12 = (index / (ne10 * ne11)) % ne12;
int i13 = (index / (ne10 * ne11 * ne12)) % ne13;
int i00 = i10 / sf0;
int i01 = i11 / sf1;
int i02 = i12 / sf2;
int i03 = i13 / sf3;
dst[index] = *((const float*)((const char*)x + i03 * nb03 + i02 * nb02 +
i01 * nb01 + i00 * nb00));
}
static void upscale_f32_bilinear(const float * x, float * dst,
const int nb00, const int nb01, const int nb02, const int nb03,
const int ne00_src, const int ne01_src,
const int ne10_dst, const int ne11_dst, const int ne12_dst, const int ne13_dst,
const float sf0, const float sf1, const float sf2, const float sf3,
const float pixel_offset) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
const int64_t index = item_ct1.get_local_id(2) +
item_ct1.get_group(2) * item_ct1.get_local_range(2);
const int64_t dst_total_elements = ne10_dst * ne11_dst * ne12_dst * ne13_dst;
if (index >= dst_total_elements) {
return;
}
const int i10_dst = index % ne10_dst;
const int i11_dst = (index / ne10_dst) % ne11_dst;
const int i12_dst = (index / (ne10_dst * ne11_dst)) % ne12_dst;
const int i13_dst = index / (ne10_dst * ne11_dst * ne12_dst);
const int i02_src = (int)(i12_dst / sf2);
const int i03_src = (int)(i13_dst / sf3);
const float y_src_f = ((float)i11_dst + pixel_offset) / sf1 - pixel_offset;
int y0_src = (int) sycl::floor((float) y_src_f);
int y1_src = y0_src + 1;
y0_src = sycl::max(0, sycl::min(y0_src, ne01_src - 1));
y1_src = sycl::max(0, sycl::min(y1_src, ne01_src - 1));
float dy = y_src_f - (float)y0_src;
dy = sycl::max(0.0f, sycl::min(dy, 1.0f));
float x_src_f = ((float)i10_dst + pixel_offset) / sf0 - pixel_offset;
int x0_src = (int) sycl::floor(x_src_f);
int x1_src = x0_src + 1;
x0_src = sycl::max(0, sycl::min(x0_src, ne00_src - 1));
x1_src = sycl::max(0, sycl::min(x1_src, ne00_src - 1));
float dx = x_src_f - (float)x0_src;
dx = sycl::max(0.0f, sycl::min(dx, 1.0f));
const float* p_a =
(const float*)((const char*)x + (int64_t)x0_src * nb00 +
(int64_t)y0_src * nb01 + (int64_t)i02_src * nb02 +
(int64_t)i03_src * nb03);
const float* p_b =
(const float*)((const char*)x + (int64_t)x1_src * nb00 +
(int64_t)y0_src * nb01 + (int64_t)i02_src * nb02 +
(int64_t)i03_src * nb03);
const float* p_c =
(const float*)((const char*)x + (int64_t)x0_src * nb00 +
(int64_t)y1_src * nb01 + (int64_t)i02_src * nb02 +
(int64_t)i03_src * nb03);
const float* p_d =
(const float*)((const char*)x + (int64_t)x1_src * nb00 +
(int64_t)y1_src * nb01 + (int64_t)i02_src * nb02 +
(int64_t)i03_src * nb03);
const float val_a = *p_a;
const float val_b = *p_b;
const float val_c = *p_c;
const float val_d = *p_d;
float result = val_a * (1.0f - dx) * (1.0f - dy) +
val_b * dx * (1.0f - dy) +
val_c * (1.0f - dx) * dy +
val_d * dx * dy;
dst[index] = result;
}
// Similar to F.interpolate(..., mode="bilinear", align_corners=False, antialias=True)
// https://github.com/pytorch/pytorch/blob/8871ff29b743948d1225389d5b7068f37b22750b/aten/src/ATen/native/cpu/UpSampleKernel.cpp
static void upscale_f32_bilinear_antialias(const float * src0,
float * dst,
const int nb00,
const int nb01,
const int nb02,
const int nb03,
const int ne00_src,
const int ne01_src,
const int ne10_dst,
const int ne11_dst,
const int ne12_dst,
const int ne13_dst,
const float sf0,
const float sf1,
const float sf2,
const float sf3,
const float pixel_offset) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
const int64_t index = item_ct1.get_local_id(2) +
item_ct1.get_group(2) * item_ct1.get_local_range(2);
const int64_t dst_total_elements = ne10_dst * ne11_dst * ne12_dst * ne13_dst;
if (index >= dst_total_elements) {
return;
}
const int i10_dst = index % ne10_dst;
const int i11_dst = (index / ne10_dst) % ne11_dst;
const int i12_dst = (index / (ne10_dst * ne11_dst)) % ne12_dst;
const int i13_dst = index / (ne10_dst * ne11_dst * ne12_dst);
const int i02_src = (int)(i12_dst / sf2);
const int i03_src = (int)(i13_dst / sf3);
const float y = ((float)i11_dst + pixel_offset) / sf1;
const float x = ((float)i10_dst + pixel_offset) / sf0;
// support and invscale, minimum 1 pixel for bilinear
const float support1 = sycl::max(1.0f / sf1, 1.0f);
const float invscale1 = 1.0f / support1;
const float support0 = sycl::max(1.0f / sf0, 1.0f);
const float invscale0 = 1.0f / support0;
// the range of source pixels that contribute
const int64_t x_min = sycl::max(int64_t(0), int64_t(x - support0 + pixel_offset));
const int64_t x_max = sycl::min(int64_t(ne00_src), int64_t(x + support0 + pixel_offset));
const int64_t y_min = sycl::max(int64_t(0), int64_t(y - support1 + pixel_offset));
const int64_t y_max = sycl::min(int64_t(ne01_src), int64_t(y + support1 + pixel_offset));
// bilinear filter with antialiasing
float val = 0.0f;
float total_weight = 0.0f;
auto triangle_filter = [](float x) -> float {
return sycl::max(1.0f - sycl::fabs(x), 0.0f);
};
for (int64_t sy = y_min; sy < y_max; sy++) {
const float weight_y = triangle_filter((sy - y + pixel_offset) * invscale1);
for (int64_t sx = x_min; sx < x_max; sx++) {
const float weight_x = triangle_filter((sx - x + pixel_offset) * invscale0);
const float weight = weight_x * weight_y;
if (weight <= 0.0f) {
continue;
}
const float pixel =
*(const float*)((const char*)src0 + sx * nb00 + sy * nb01 +
i02_src * nb02 + i03_src * nb03);
val += pixel * weight;
total_weight += weight;
}
}
if (total_weight > 0.0f) {
val /= total_weight;
}
dst[index] = val;
}
namespace bicubic_interpolation {
static float weight1(float x, const float &a) { return ((a + 2) * x - (a + 3)) * x * x + 1; };
static float weight2(float x, const float &a) { return ((a * x - 5 * a) * x + 8 * a) * x - 4 * a; };
static float bicubic(float p0, float p1, float p2, float p3, float x, float a) {
const float w0 = weight2(x + 1, a);
const float w1 = weight1(x + 0, a);
const float w2 = weight1(1 - x, a);
const float w3 = weight2(2 - x, a);
return p0 * w0 + p1 * w1 + p2 * w2 + p3 * w3;
};
}
static void upscale_f32_bicubic(const float * x, float * dst,
const int nb00, const int nb01, const int nb02, const int nb03,
const int ne00_src, const int ne01_src,
const int ne10_dst, const int ne11_dst, const int ne12_dst, const int ne13_dst,
const float sf0, const float sf1, const float sf2, const float sf3,
const float pixel_offset) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
const float a = -0.75f;
using bicubic_interpolation::bicubic;
const int64_t index = item_ct1.get_local_id(2) +
item_ct1.get_group(2) * item_ct1.get_local_range(2);
const int64_t dst_total_elements =
ne10_dst * ne11_dst * ne12_dst * ne13_dst;
if (index >= dst_total_elements) {
return;
}
const int i10_dst = index % ne10_dst;
const int i11_dst = (index / ne10_dst) % ne11_dst;
const int i12_dst = (index / (ne10_dst * ne11_dst)) % ne12_dst;
const int i13_dst = index / (ne10_dst * ne11_dst * ne12_dst);
const int i02_src = (int)(i12_dst / sf2);
const int i03_src = (int)(i13_dst / sf3);
const float y_src_f = ((float)i11_dst + pixel_offset) / sf1 - pixel_offset;
const int y0_src = (int) sycl::floor((float) y_src_f);
const float dy = y_src_f - (float)y0_src;
const float x_src_f = ((float)i10_dst + pixel_offset) / sf0 - pixel_offset;
const int x0_src = (int) sycl::floor((float) x_src_f);
const float dx = x_src_f - (float)x0_src;
const char * x_base = (const char *)x + (int64_t)i02_src * nb02 + (int64_t)i03_src * nb03;
auto load = [=](int x_off, int y_off) -> float {
int i00_src = sycl::max(0, sycl::min(x0_src + x_off, ne00_src - 1));
int i01_src = sycl::max(0, sycl::min(y0_src + y_off, ne01_src - 1));
return *(const float *)(x_base + (int64_t)i00_src * nb00 + (int64_t)i01_src * nb01);
};
const float result = bicubic(
bicubic(load(-1, -1), load(0, -1), load(1, -1), load(2, -1), dx, a),
bicubic(load(-1, 0), load(0, 0), load(1, 0), load(2, 0), dx, a),
bicubic(load(-1, 1), load(0, 1), load(1, 1), load(2, 1), dx, a),
bicubic(load(-1, 2), load(0, 2), load(1, 2), load(2, 2), dx, a),
dy,
a);
dst[index] = result;
}
static void upscale_f32_sycl(const float * x,
float * dst,
const int nb00,
const int nb01,
const int nb02,
const int nb03,
const int ne10,
const int ne11,
const int ne12,
const int ne13,
const float sf0,
const float sf1,
const float sf2,
const float sf3,
dpct::queue_ptr stream) {
const int64_t dst_size = ne10 * ne11 * ne12 * ne13;
const int64_t num_blocks = (dst_size + SYCL_UPSCALE_BLOCK_SIZE - 1) / SYCL_UPSCALE_BLOCK_SIZE;
stream->parallel_for(
sycl::nd_range<3>(
sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_UPSCALE_BLOCK_SIZE),
sycl::range<3>(1, 1, SYCL_UPSCALE_BLOCK_SIZE)),
[=](sycl::nd_item<3> item_ct1) {
upscale_f32(x, dst, nb00, nb01, nb02, nb03, ne10, ne11, ne12, ne13, sf0, sf1, sf2, sf3);
});
}
static void upscale_f32_bilinear_sycl(const float * x,
float * dst,
const int nb00,
const int nb01,
const int nb02,
const int nb03,
const int ne00_src,
const int ne01_src,
const int ne10_dst,
const int ne11_dst,
const int ne12_dst,
const int ne13_dst,
const float sf0,
const float sf1,
const float sf2,
const float sf3,
const float pixel_offset,
bool antialias,
dpct::queue_ptr stream) {
const int64_t dst_size = ne10_dst * ne11_dst * ne12_dst * ne13_dst;
const int64_t num_blocks = (dst_size + SYCL_UPSCALE_BLOCK_SIZE - 1) / SYCL_UPSCALE_BLOCK_SIZE;
if (antialias) {
stream->parallel_for(
sycl::nd_range<3>(
sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_UPSCALE_BLOCK_SIZE),
sycl::range<3>(1, 1, SYCL_UPSCALE_BLOCK_SIZE)),
[=](sycl::nd_item<3> item_ct1) {
upscale_f32_bilinear_antialias(
x, dst, nb00, nb01, nb02, nb03, ne00_src, ne01_src, ne10_dst, ne11_dst,
ne12_dst, ne13_dst, sf0, sf1, sf2, sf3, pixel_offset);
});
} else {
stream->parallel_for(
sycl::nd_range<3>(
sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_UPSCALE_BLOCK_SIZE),
sycl::range<3>(1, 1, SYCL_UPSCALE_BLOCK_SIZE)),
[=](sycl::nd_item<3> item_ct1) {
upscale_f32_bilinear(
x, dst, nb00, nb01, nb02, nb03, ne00_src, ne01_src, ne10_dst, ne11_dst, ne12_dst,
ne13_dst, sf0, sf1, sf2, sf3, pixel_offset);
});
}
}
static void upscale_f32_bicubic_sycl(const float * x,
float * dst,
const int nb00,
const int nb01,
const int nb02,
const int nb03,
const int ne00_src,
const int ne01_src,
const int ne10_dst,
const int ne11_dst,
const int ne12_dst,
const int ne13_dst,
const float sf0,
const float sf1,
const float sf2,
const float sf3,
const float pixel_offset,
dpct::queue_ptr stream) {
const int64_t dst_size = ne10_dst * ne11_dst * ne12_dst * ne13_dst;
const int64_t num_blocks = (dst_size + SYCL_UPSCALE_BLOCK_SIZE - 1) / SYCL_UPSCALE_BLOCK_SIZE;
{
stream->submit([&](sycl::handler & cgh) {
cgh.parallel_for(
sycl::nd_range<3>(
sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_UPSCALE_BLOCK_SIZE),
sycl::range<3>(1, 1, SYCL_UPSCALE_BLOCK_SIZE)),
[=](sycl::nd_item<3> item_ct1) {
upscale_f32_bicubic(
x, dst, nb00, nb01, nb02, nb03, ne00_src, ne01_src, ne10_dst, ne11_dst,
ne12_dst, ne13_dst, sf0, sf1, sf2, sf3, pixel_offset);
});
});
}
}
void ggml_sycl_op_upscale(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
const ggml_tensor * src0 = dst->src[0];
const float * src0_d = (const float *)src0->data;
float * dst_d = (float *)dst->data;
dpct::queue_ptr stream = ctx.stream();
GGML_ASSERT(src0->type == GGML_TYPE_F32);
GGML_ASSERT( dst->type == GGML_TYPE_F32);
const int mode_flags = dst->op_params[0];
const ggml_scale_mode mode = (ggml_scale_mode)(mode_flags & 0xFF);
float sf0 = (float)dst->ne[0]/src0->ne[0];
float sf1 = (float)dst->ne[1]/src0->ne[1];
float sf2 = (float)dst->ne[2]/src0->ne[2];
const float sf3 = (float)dst->ne[3]/src0->ne[3];
float pixel_offset = 0.5f;
if (mode_flags & GGML_SCALE_FLAG_ALIGN_CORNERS) {
sf0 = dst->ne[0] > 1 && src0->ne[0] > 1
? (float)(dst->ne[0] - 1) / (src0->ne[0] - 1)
: sf0;
sf1 = dst->ne[1] > 1 && src0->ne[1] > 1
? (float)(dst->ne[1] - 1) / (src0->ne[1] - 1)
: sf1;
pixel_offset = 0.0f;
}
if (mode == GGML_SCALE_MODE_NEAREST) {
upscale_f32_sycl(
src0_d, dst_d, src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], sf0, sf1, sf2, sf3, stream);
} else if (mode == GGML_SCALE_MODE_BILINEAR) {
const bool antialias = (mode_flags & GGML_SCALE_FLAG_ANTIALIAS);
upscale_f32_bilinear_sycl(
src0_d, dst_d, src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
src0->ne[0], src0->ne[1], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
sf0, sf1, sf2, sf3, pixel_offset, antialias, stream);
} else if (mode == GGML_SCALE_MODE_BICUBIC) {
upscale_f32_bicubic_sycl(
src0_d, dst_d, src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
src0->ne[0], src0->ne[1], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
sf0, sf1, sf2, sf3, pixel_offset, stream);
}
}
void ggml_sycl_upscale(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);
ggml_sycl_op_upscale(ctx, dst);
}
ggml/src/ggml-sycl/upscale.hpp
+9
View File
@@ -0,0 +1,9 @@
#pragma once
#include <sycl/sycl.hpp>
#include "dpct/helper.hpp"
#include "common.hpp"
#define SYCL_UPSCALE_BLOCK_SIZE 256
void ggml_sycl_upscale(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
gguf-py/gguf/constants.py
+33
View File
@@ -478,6 +478,7 @@ class MODEL_ARCH(IntEnum):
RND1 = auto()
PANGU_EMBED = auto()
MISTRAL3 = auto()
MISTRAL4 = auto()
PADDLEOCR = auto()
MIMO2 = auto()
STEP35 = auto()
@@ -924,6 +925,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.RND1: "rnd1",
MODEL_ARCH.PANGU_EMBED: "pangu-embedded",
MODEL_ARCH.MISTRAL3: "mistral3",
MODEL_ARCH.MISTRAL4: "mistral4",
MODEL_ARCH.PADDLEOCR: "paddleocr",
MODEL_ARCH.MIMO2: "mimo2",
MODEL_ARCH.STEP35: "step35",
@@ -3538,6 +3540,37 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.FFN_DOWN_EXP,
MODEL_TENSOR.FFN_UP_EXP,
],
MODEL_ARCH.MISTRAL4: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
MODEL_TENSOR.OUTPUT,
MODEL_TENSOR.ROPE_FREQS,
MODEL_TENSOR.ATTN_NORM,
MODEL_TENSOR.ATTN_Q,
MODEL_TENSOR.ATTN_Q_A,
MODEL_TENSOR.ATTN_Q_B,
MODEL_TENSOR.ATTN_KV_A_MQA,
MODEL_TENSOR.ATTN_KV_B,
MODEL_TENSOR.ATTN_K_B,
MODEL_TENSOR.ATTN_V_B,
MODEL_TENSOR.ATTN_Q_A_NORM,
MODEL_TENSOR.ATTN_KV_A_NORM,
MODEL_TENSOR.ATTN_OUT,
MODEL_TENSOR.ATTN_ROT_EMBD,
MODEL_TENSOR.FFN_GATE_INP,
MODEL_TENSOR.FFN_NORM,
MODEL_TENSOR.FFN_GATE,
MODEL_TENSOR.FFN_DOWN,
MODEL_TENSOR.FFN_UP,
MODEL_TENSOR.FFN_GATE_EXP,
MODEL_TENSOR.FFN_DOWN_EXP,
MODEL_TENSOR.FFN_UP_EXP,
MODEL_TENSOR.FFN_GATE_UP_EXP,
MODEL_TENSOR.FFN_GATE_SHEXP,
MODEL_TENSOR.FFN_DOWN_SHEXP,
MODEL_TENSOR.FFN_UP_SHEXP,
MODEL_TENSOR.FFN_EXP_PROBS_B,
],
MODEL_ARCH.MIMO2: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
src/llama-arch.cpp
+2
View File
@@ -123,6 +123,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_RND1, "rnd1" },
{ LLM_ARCH_PANGU_EMBED, "pangu-embedded" },
{ LLM_ARCH_MISTRAL3, "mistral3" },
{ LLM_ARCH_MISTRAL4, "mistral4" },
{ LLM_ARCH_PADDLEOCR, "paddleocr" },
{ LLM_ARCH_MIMO2, "mimo2" },
{ LLM_ARCH_STEP35, "step35" },
@@ -1589,6 +1590,7 @@ static std::set<llm_tensor> llm_get_tensor_names(llm_arch arch) {
LLM_TENSOR_FFN_UP_SHEXP,
};
case LLM_ARCH_DEEPSEEK2:
case LLM_ARCH_MISTRAL4:
return {
LLM_TENSOR_TOKEN_EMBD,
LLM_TENSOR_OUTPUT_NORM,
src/llama-arch.h
+1
View File
@@ -127,6 +127,7 @@ enum llm_arch {
LLM_ARCH_RND1,
LLM_ARCH_PANGU_EMBED,
LLM_ARCH_MISTRAL3,
LLM_ARCH_MISTRAL4,
LLM_ARCH_PADDLEOCR,
LLM_ARCH_MIMO2,
LLM_ARCH_STEP35,
src/llama-model.cpp
+5 -1
View File
@@ -1701,6 +1701,7 @@ void llama_model::load_hparams(llama_model_loader & ml) {
}
} break;
case LLM_ARCH_DEEPSEEK2:
case LLM_ARCH_MISTRAL4:
{
// lite variants include DeepSeek-V2-Lite, GigaChat3-10B-A1.8B, Kanana-2-30B-A3B
const bool is_lite = (hparams.n_layer == 27 || hparams.n_layer == 26 || (hparams.n_layer == 48 && n_vocab == 128256));
@@ -5041,6 +5042,7 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
}
} break;
case LLM_ARCH_DEEPSEEK2:
case LLM_ARCH_MISTRAL4:
{
const bool is_mla = hparams.is_mla();
@@ -8008,7 +8010,7 @@ void llama_model::print_info() const {
LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale);
}
if (arch == LLM_ARCH_DEEPSEEK2 || arch == LLM_ARCH_GLM_DSA) {
if (arch == LLM_ARCH_DEEPSEEK2 || arch == LLM_ARCH_GLM_DSA || arch == LLM_ARCH_MISTRAL4) {
LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead);
LLAMA_LOG_INFO("%s: n_lora_q = %d\n", __func__, hparams.n_lora_q);
LLAMA_LOG_INFO("%s: n_lora_kv = %d\n", __func__, hparams.n_lora_kv);
@@ -8586,6 +8588,7 @@ ggml_cgraph * llama_model::build_graph(const llm_graph_params & params) const {
} break;
case LLM_ARCH_DEEPSEEK2:
case LLM_ARCH_GLM_DSA:
case LLM_ARCH_MISTRAL4:
{
llm = std::make_unique<llm_build_deepseek2>(*this, params);
} break;
@@ -8997,6 +9000,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_ERNIE4_5:
case LLM_ARCH_ERNIE4_5_MOE:
case LLM_ARCH_MISTRAL3:
case LLM_ARCH_MISTRAL4:
case LLM_ARCH_LLAMA_EMBED:
case LLM_ARCH_MAINCODER:
case LLM_ARCH_GLM_DSA:
src/models/qwen35.cpp
+1 -1
View File
@@ -224,7 +224,7 @@ ggml_tensor * llm_build_qwen35::build_layer_attn_linear(
beta = ggml_sigmoid(ctx0, beta);
ggml_tensor * alpha = build_lora_mm(model.layers[il].ssm_alpha, cur, model.layers[il].ssm_alpha_s);
alpha = ggml_cont_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs);
alpha = ggml_reshape_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs);
cb(alpha, "alpha", il);
ggml_tensor * alpha_biased = ggml_add(ctx0, alpha, model.layers[il].ssm_dt);
src/models/qwen35moe.cpp
+1 -1
View File
@@ -224,7 +224,7 @@ ggml_tensor * llm_build_qwen35moe ::build_layer_attn_linear(
beta = ggml_sigmoid(ctx0, beta);
ggml_tensor * alpha = build_lora_mm(model.layers[il].ssm_alpha, cur, model.layers[il].ssm_alpha_s);
alpha = ggml_cont_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs);
alpha = ggml_reshape_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs);
cb(alpha, "alpha", il);
ggml_tensor * alpha_biased = ggml_add(ctx0, alpha, model.layers[il].ssm_dt);
tools/server/public/index.html.gz
BIN
View File
Binary file not shown.
tools/server/server-common.cpp
+20 -10
View File
@@ -1273,17 +1273,27 @@ json convert_responses_to_chatcmpl(const json & response_body) {
for (const auto & output_text : item.at("content")) {
const std::string type = json_value(output_text, "type", std::string());
if (type != "output_text") {
throw std::invalid_argument("'type' must be 'output_text'");
if (type == "output_text") {
if (!exists_and_is_string(output_text, "text")) {
throw std::invalid_argument("'Output text' requires 'text'");
// Ignore annotations and logprobs for now
chatcmpl_content.push_back({
{"text", output_text.at("text")},
{"type", "text"},
});
}
} else if (type == "refusal") {
if (!exists_and_is_string(output_text, "refusal")) {
throw std::invalid_argument("'Refusal' requires 'refusal'");
// Ignore annotations and logprobs for now
chatcmpl_content.push_back({
{"refusal", output_text.at("refusal")},
{"type", "refusal"},
});
}
} else {
throw std::invalid_argument("'type' must be one of 'output_text' or 'refusal'");
}
if (!exists_and_is_string(output_text, "text")) {
throw std::invalid_argument("'Output text' requires 'text'");
}
// Ignore annotations and logprobs for now
chatcmpl_content.push_back({
{"text", output_text.at("text")},
{"type", "text"},
});
}
if (merge_prev) {
tools/server/webui/src/lib/components/app/actions/ActionIcon.svelte
+1 -1
View File
@@ -11,7 +11,7 @@
iconSize?: string;
class?: string;
disabled?: boolean;
onclick: () => void;
onclick: (e?: MouseEvent) => void;
'aria-label'?: string;
}
tools/server/webui/src/lib/components/app/dialogs/DialogModelInformation.svelte
+46 -8
View File
@@ -5,21 +5,38 @@
import { serverStore } from '$lib/stores/server.svelte';
import { modelsStore, modelOptions, modelsLoading } from '$lib/stores/models.svelte';
import { formatFileSize, formatParameters, formatNumber } from '$lib/utils';
import type { ApiLlamaCppServerProps } from '$lib/types';
interface Props {
open?: boolean;
onOpenChange?: (open: boolean) => void;
// when set, fetch props from the child process (router mode)
modelId?: string | null;
}
let { open = $bindable(), onOpenChange }: Props = $props();
let { open = $bindable(), onOpenChange, modelId = null }: Props = $props();
let serverProps = $derived(serverStore.props);
let modelName = $derived(modelsStore.singleModelName);
let isRouter = $derived(serverStore.isRouterMode);
// per-model props fetched from the child process
let routerModelProps = $state<ApiLlamaCppServerProps | null>(null);
let isLoadingRouterProps = $state(false);
// in router mode use per-model props, otherwise use global props
let serverProps = $derived(isRouter && modelId ? routerModelProps : serverStore.props);
let modelName = $derived(isRouter && modelId ? modelId : modelsStore.singleModelName);
let models = $derived(modelOptions());
let isLoadingModels = $derived(modelsLoading());
// Get the first model for single-model mode display
let firstModel = $derived(models[0] ?? null);
// in router mode, find the model option matching modelId
// in single mode, use the first model as before
let firstModel = $derived.by(() => {
if (isRouter && modelId) {
return models.find((m) => m.model === modelId) ?? null;
}
return models[0] ?? null;
});
// Get modalities from modelStore using the model ID from the first model
let modalities = $derived.by(() => {
@@ -33,10 +50,31 @@
modelsStore.fetch();
}
});
// fetch per-model props from child process when dialog opens in router mode
$effect(() => {
if (open && isRouter && modelId) {
isLoadingRouterProps = true;
modelsStore
.fetchModelProps(modelId)
.then((props) => {
routerModelProps = props;
})
.catch(() => {
routerModelProps = null;
})
.finally(() => {
isLoadingRouterProps = false;
});
}
if (!open) {
routerModelProps = null;
}
});
</script>
<Dialog.Root bind:open {onOpenChange}>
<Dialog.Content class="@container z-9999 !max-w-[60rem] max-w-full">
<Dialog.Content class="@container z-9999 !max-h-[80dvh] !max-w-[60rem] max-w-full">
<style>
@container (max-width: 56rem) {
.resizable-text-container {
@@ -52,7 +90,7 @@
</Dialog.Header>
<div class="space-y-6 py-4">
{#if isLoadingModels}
{#if isLoadingModels || isLoadingRouterProps}
<div class="flex items-center justify-center py-8">
<div class="text-sm text-muted-foreground">Loading model information...</div>
</div>
@@ -212,7 +250,7 @@
<Table.Cell class="align-middle font-medium">Chat Template</Table.Cell>
<Table.Cell class="py-10">
<div class="max-h-120 overflow-y-auto rounded-md bg-muted p-4">
<div class="rounded-md bg-muted p-4">
<pre
class="font-mono text-xs whitespace-pre-wrap">{serverProps.chat_template}</pre>
</div>
tools/server/webui/src/lib/components/app/models/ModelsSelector.svelte
+47 -122
View File
@@ -1,6 +1,5 @@
<script lang="ts">
import { onMount } from 'svelte';
import { SvelteMap } from 'svelte/reactivity';
import { ChevronDown, Loader2, Package } from '@lucide/svelte';
import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
import * as Tooltip from '$lib/components/ui/tooltip';
@@ -19,9 +18,11 @@
DialogModelInformation,
DropdownMenuSearchable,
ModelId,
ModelsSelectorList,
ModelsSelectorOption
} from '$lib/components/app';
import type { ModelOption } from '$lib/types/models';
import { filterModelOptions, groupModelOptions, type ModelItem } from './utils';
interface Props {
class?: string;
@@ -73,89 +74,13 @@
let searchTerm = $state('');
let highlightedIndex = $state<number>(-1);
let filteredOptions: ModelOption[] = $derived.by(() => {
const term = searchTerm.trim().toLowerCase();
if (!term) return options;
let filteredOptions = $derived(filterModelOptions(options, searchTerm));
return options.filter(
(option) =>
option.model.toLowerCase().includes(term) ||
option.name?.toLowerCase().includes(term) ||
option.aliases?.some((alias: string) => alias.toLowerCase().includes(term)) ||
option.tags?.some((tag: string) => tag.toLowerCase().includes(term))
);
});
let groupedFilteredOptions = $derived.by(() => {
const favIds = modelsStore.favouriteModelIds;
const result: {
orgName: string | null;
isFavouritesGroup: boolean;
isLoadedGroup: boolean;
items: { option: ModelOption; flatIndex: number }[];
}[] = [];
// Loaded models group (top)
const loadedItems: { option: ModelOption; flatIndex: number }[] = [];
for (let i = 0; i < filteredOptions.length; i++) {
if (modelsStore.isModelLoaded(filteredOptions[i].model)) {
loadedItems.push({ option: filteredOptions[i], flatIndex: i });
}
}
if (loadedItems.length > 0) {
result.push({
orgName: null,
isFavouritesGroup: false,
isLoadedGroup: true,
items: loadedItems
});
}
// Favourites group
const loadedModelIds = new Set(loadedItems.map((item) => item.option.model));
const favItems: { option: ModelOption; flatIndex: number }[] = [];
for (let i = 0; i < filteredOptions.length; i++) {
if (favIds.has(filteredOptions[i].model) && !loadedModelIds.has(filteredOptions[i].model)) {
favItems.push({ option: filteredOptions[i], flatIndex: i });
}
}
if (favItems.length > 0) {
result.push({
orgName: null,
isFavouritesGroup: true,
isLoadedGroup: false,
items: favItems
});
}
// Org groups (excluding loaded and favourites)
const orgGroups = new SvelteMap<string, { option: ModelOption; flatIndex: number }[]>();
for (let i = 0; i < filteredOptions.length; i++) {
const option = filteredOptions[i];
if (loadedModelIds.has(option.model) || favIds.has(option.model)) continue;
const orgName = option.parsedId?.orgName ?? null;
const key = orgName ?? '';
if (!orgGroups.has(key)) orgGroups.set(key, []);
orgGroups.get(key)!.push({ option, flatIndex: i });
}
for (const [orgName, items] of orgGroups) {
result.push({
orgName: orgName || null,
isFavouritesGroup: false,
isLoadedGroup: false,
items
});
}
return result;
});
let groupedFilteredOptions = $derived(
groupModelOptions(filteredOptions, modelsStore.favouriteModelIds, (m) =>
modelsStore.isModelLoaded(m)
)
);
$effect(() => {
void searchTerm;
@@ -164,6 +89,12 @@
let isOpen = $state(false);
let showModelDialog = $state(false);
let infoModelId = $state<string | null>(null);
function handleInfoClick(modelName: string) {
infoModelId = modelName;
showModelDialog = true;
}
onMount(() => {
modelsStore.fetch().catch((error) => {
@@ -418,45 +349,39 @@
<p class="px-4 py-3 text-sm text-muted-foreground">No models found.</p>
{/if}
{#each groupedFilteredOptions as group (group.isLoadedGroup ? '__loaded__' : group.isFavouritesGroup ? '__favourites__' : group.orgName)}
{#if group.isLoadedGroup}
<p class="px-2 py-2 text-xs font-semibold text-muted-foreground/60 select-none">
Loaded models
</p>
{:else if group.isFavouritesGroup}
<p class="px-2 py-2 text-xs font-semibold text-muted-foreground/60 select-none">
Favourite models
</p>
{:else if group.orgName}
<p
class="px-2 py-2 text-xs font-semibold text-muted-foreground/60 select-none [&:not(:first-child)]:mt-2"
>
{group.orgName}
</p>
{/if}
{#snippet modelOption(item: ModelItem, showOrgName: boolean)}
{@const { option, flatIndex } = item}
{@const isSelected = currentModel === option.model || activeId === option.id}
{@const isHighlighted = flatIndex === highlightedIndex}
{@const isFav = modelsStore.favouriteModelIds.has(option.model)}
{#each group.items as { option, flatIndex } (group.isLoadedGroup ? `loaded-${option.id}` : group.isFavouritesGroup ? `fav-${option.id}` : option.id)}
{@const isSelected = currentModel === option.model || activeId === option.id}
{@const isHighlighted = flatIndex === highlightedIndex}
{@const isFav = modelsStore.favouriteModelIds.has(option.model)}
<ModelsSelectorOption
{option}
{isSelected}
{isHighlighted}
{isFav}
{showOrgName}
onSelect={handleSelect}
onInfoClick={handleInfoClick}
onMouseEnter={() => (highlightedIndex = flatIndex)}
onKeyDown={(e) => {
if (e.key === KeyboardKey.ENTER || e.key === KeyboardKey.SPACE) {
e.preventDefault();
handleSelect(option.id);
}
}}
/>
{/snippet}
<ModelsSelectorOption
{option}
{isSelected}
{isHighlighted}
{isFav}
showOrgName={group.isFavouritesGroup || group.isLoadedGroup}
onSelect={handleSelect}
onMouseEnter={() => (highlightedIndex = flatIndex)}
onKeyDown={(e) => {
if (e.key === KeyboardKey.ENTER || e.key === KeyboardKey.SPACE) {
e.preventDefault();
handleSelect(option.id);
}
}}
/>
{/each}
{/each}
<ModelsSelectorList
groups={groupedFilteredOptions}
{currentModel}
{activeId}
sectionHeaderClass="my-1.5 px-2 py-2 text-[13px] font-semibold text-muted-foreground/70 select-none"
onSelect={handleSelect}
onInfoClick={handleInfoClick}
renderOption={modelOption}
/>
</div>
</DropdownMenuSearchable>
</DropdownMenu.Content>
@@ -500,6 +425,6 @@
{/if}
</div>
{#if showModelDialog && !isRouter}
<DialogModelInformation bind:open={showModelDialog} />
{#if showModelDialog}
<DialogModelInformation bind:open={showModelDialog} modelId={infoModelId} />
{/if}
tools/server/webui/src/lib/components/app/models/ModelsSelectorList.svelte
+72
View File
@@ -0,0 +1,72 @@
<script lang="ts">
import { modelsStore } from '$lib/stores/models.svelte';
import { ModelsSelectorOption } from '$lib/components/app';
import type { GroupedModelOptions, ModelItem } from './utils';
interface Props {
groups: GroupedModelOptions;
currentModel: string | null;
activeId: string | null;
sectionHeaderClass?: string;
orgHeaderClass?: string;
onSelect: (modelId: string) => void;
onInfoClick: (modelName: string) => void;
renderOption?: import('svelte').Snippet<[ModelItem, boolean]>;
}
let {
groups,
currentModel,
activeId,
sectionHeaderClass = 'my-1 px-2 py-2 text-[13px] font-semibold text-muted-foreground/70 select-none',
orgHeaderClass = 'px-2 py-2 text-[11px] font-semibold text-muted-foreground/50 select-none [&:not(:first-child)]:mt-1',
onSelect,
onInfoClick,
renderOption
}: Props = $props();
let render = $derived(renderOption ?? defaultOption);
</script>
{#snippet defaultOption(item: ModelItem, showOrgName: boolean)}
{@const { option } = item}
{@const isSelected = currentModel === option.model || activeId === option.id}
{@const isFav = modelsStore.favouriteModelIds.has(option.model)}
<ModelsSelectorOption
{option}
{isSelected}
isHighlighted={false}
{isFav}
{showOrgName}
{onSelect}
{onInfoClick}
onMouseEnter={() => {}}
onKeyDown={() => {}}
/>
{/snippet}
{#if groups.loaded.length > 0}
<p class={sectionHeaderClass}>Loaded models</p>
{#each groups.loaded as item (`loaded-${item.option.id}`)}
{@render render(item, true)}
{/each}
{/if}
{#if groups.favourites.length > 0}
<p class={sectionHeaderClass}>Favourite models</p>
{#each groups.favourites as item (`fav-${item.option.id}`)}
{@render render(item, true)}
{/each}
{/if}
{#if groups.available.length > 0}
<p class={sectionHeaderClass}>Available models</p>
{#each groups.available as group (group.orgName)}
{#if group.orgName}
<p class={orgHeaderClass}>{group.orgName}</p>
{/if}
{#each group.items as item (item.option.id)}
{@render render(item, false)}
{/each}
{/each}
{/if}
tools/server/webui/src/lib/components/app/models/ModelsSelectorOption.svelte
+27 -4
View File
@@ -1,5 +1,14 @@
<script lang="ts">
import { CircleAlert, Heart, HeartOff, Loader2, Power, PowerOff, RotateCw } from '@lucide/svelte';
import {
CircleAlert,
Heart,
HeartOff,
Info,
Loader2,
Power,
PowerOff,
RotateCw
} from '@lucide/svelte';
import { cn } from '$lib/components/ui/utils';
import { ActionIcon, ModelId } from '$lib/components/app';
import type { ModelOption } from '$lib/types/models';
@@ -15,6 +24,7 @@
onSelect: (modelId: string) => void;
onMouseEnter: () => void;
onKeyDown: (e: KeyboardEvent) => void;
onInfoClick?: (modelName: string) => void;
}
let {
@@ -25,7 +35,8 @@
showOrgName = false,
onSelect,
onMouseEnter,
onKeyDown
onKeyDown,
onInfoClick
}: Props = $props();
let currentRouterModels = $derived(routerModels());
@@ -63,11 +74,11 @@
class="flex-1"
/>
<div class="flex shrink-0 items-center gap-2.5">
<div class="flex shrink-0 items-center gap-1">
<!-- svelte-ignore a11y_no_static_element_interactions -->
<!-- svelte-ignore a11y_click_events_have_key_events -->
<div
class="pointer-events-none flex w-4 items-center justify-center pl-2 opacity-0 group-hover:pointer-events-auto group-hover:opacity-100"
class="pointer-events-none flex items-center justify-center gap-0.75 pl-2 opacity-0 group-hover:pointer-events-auto group-hover:opacity-100"
onclick={(e) => e.stopPropagation()}
>
{#if isFav}
@@ -87,7 +98,19 @@
onclick={() => modelsStore.toggleFavourite(option.model)}
/>
{/if}
<!-- info button: only shown when model is loaded and callback is provided -->
{#if isLoaded && onInfoClick}
<ActionIcon
iconSize="h-2.5 w-2.5"
icon={Info}
tooltip="Model information"
class="h-3 w-3 hover:text-foreground"
onclick={() => onInfoClick(option.model)}
/>
{/if}
</div>
{#if isLoading}
<Loader2 class="h-4 w-4 animate-spin text-muted-foreground" />
{:else if isFailed}
tools/server/webui/src/lib/components/app/models/ModelsSelectorSheet.svelte
+26 -112
View File
@@ -1,6 +1,5 @@
<script lang="ts">
import { onMount } from 'svelte';
import { SvelteMap } from 'svelte/reactivity';
import { ChevronDown, Loader2, Package } from '@lucide/svelte';
import * as Sheet from '$lib/components/ui/sheet';
import { cn } from '$lib/components/ui/utils';
@@ -15,11 +14,12 @@
import { isRouterMode } from '$lib/stores/server.svelte';
import {
DialogModelInformation,
ModelsSelectorList,
SearchInput,
TruncatedText,
ModelsSelectorOption
TruncatedText
} from '$lib/components/app';
import type { ModelOption } from '$lib/types/models';
import { filterModelOptions, groupModelOptions } from './utils';
interface Props {
class?: string;
@@ -73,85 +73,22 @@
let searchTerm = $state('');
let filteredOptions: ModelOption[] = $derived.by(() => {
const term = searchTerm.trim().toLowerCase();
if (!term) return options;
let filteredOptions = $derived(filterModelOptions(options, searchTerm));
return options.filter(
(option) =>
option.model.toLowerCase().includes(term) ||
option.name?.toLowerCase().includes(term) ||
option.aliases?.some((alias: string) => alias.toLowerCase().includes(term)) ||
option.tags?.some((tag: string) => tag.toLowerCase().includes(term))
);
});
let groupedFilteredOptions = $derived.by(() => {
const favIds = modelsStore.favouriteModelIds;
const result: {
orgName: string | null;
isFavouritesGroup: boolean;
isLoadedGroup: boolean;
items: { option: ModelOption; flatIndex: number }[];
}[] = [];
// Loaded models group (top)
const loadedItems: { option: ModelOption; flatIndex: number }[] = [];
for (let i = 0; i < filteredOptions.length; i++) {
if (modelsStore.isModelLoaded(filteredOptions[i].model)) {
loadedItems.push({ option: filteredOptions[i], flatIndex: i });
}
}
if (loadedItems.length > 0) {
result.push({
orgName: null,
isFavouritesGroup: false,
isLoadedGroup: true,
items: loadedItems
});
}
// Favourites group
const loadedModelIds = new Set(loadedItems.map((item) => item.option.model));
const favItems: { option: ModelOption; flatIndex: number }[] = [];
for (let i = 0; i < filteredOptions.length; i++) {
if (favIds.has(filteredOptions[i].model) && !loadedModelIds.has(filteredOptions[i].model)) {
favItems.push({ option: filteredOptions[i], flatIndex: i });
}
}
if (favItems.length > 0) {
result.push({
orgName: null,
isFavouritesGroup: true,
isLoadedGroup: false,
items: favItems
});
}
// Org groups (excluding loaded and favourites)
const orgGroups = new SvelteMap<string, { option: ModelOption; flatIndex: number }[]>();
for (let i = 0; i < filteredOptions.length; i++) {
const option = filteredOptions[i];
if (loadedModelIds.has(option.model) || favIds.has(option.model)) continue;
const orgName = option.parsedId?.orgName ?? null;
const key = orgName ?? '';
if (!orgGroups.has(key)) orgGroups.set(key, []);
orgGroups.get(key)!.push({ option, flatIndex: i });
}
for (const [orgName, items] of orgGroups) {
result.push({
orgName: orgName || null,
isFavouritesGroup: false,
isLoadedGroup: false,
items
});
}
return result;
});
let groupedFilteredOptions = $derived(
groupModelOptions(filteredOptions, modelsStore.favouriteModelIds, (m) =>
modelsStore.isModelLoaded(m)
)
);
let sheetOpen = $state(false);
let showModelDialog = $state(false);
let infoModelId = $state<string | null>(null);
function handleInfoClick(modelName: string) {
infoModelId = modelName;
showModelDialog = true;
}
onMount(() => {
modelsStore.fetch().catch((error) => {
@@ -339,38 +276,15 @@
<p class="px-3 py-3 text-center text-sm text-muted-foreground">No models found.</p>
{/if}
{#each groupedFilteredOptions as group (group.isLoadedGroup ? '__loaded__' : group.isFavouritesGroup ? '__favourites__' : group.orgName)}
{#if group.isLoadedGroup}
<p class="px-2 py-2 text-xs font-semibold text-muted-foreground/60 select-none">
Loaded models
</p>
{:else if group.isFavouritesGroup}
<p class="px-2 py-2 text-xs font-semibold text-muted-foreground/60 select-none">
Favourite models
</p>
{:else if group.orgName}
<p
class="px-2 py-2 text-xs font-semibold text-muted-foreground/60 select-none [&:not(:first-child)]:mt-2"
>
{group.orgName}
</p>
{/if}
{#each group.items as { option } (group.isLoadedGroup ? `loaded-${option.id}` : group.isFavouritesGroup ? `fav-${option.id}` : option.id)}
{@const isSelected = currentModel === option.model || activeId === option.id}
{@const isFav = modelsStore.favouriteModelIds.has(option.model)}
<ModelsSelectorOption
{option}
{isSelected}
isHighlighted={false}
{isFav}
showOrgName={group.isFavouritesGroup || group.isLoadedGroup}
onSelect={handleSelect}
onMouseEnter={() => {}}
onKeyDown={() => {}}
/>
{/each}
{/each}
<ModelsSelectorList
groups={groupedFilteredOptions}
{currentModel}
{activeId}
sectionHeaderClass="px-2 py-2 text-xs font-semibold text-muted-foreground/60 select-none"
orgHeaderClass="px-2 py-2 text-xs font-semibold text-muted-foreground/60 select-none [&:not(:first-child)]:mt-2"
onSelect={handleSelect}
onInfoClick={handleInfoClick}
/>
</div>
</div>
</Sheet.Content>
@@ -403,6 +317,6 @@
{/if}
</div>
{#if showModelDialog && !isRouter}
<DialogModelInformation bind:open={showModelDialog} />
{#if showModelDialog}
<DialogModelInformation bind:open={showModelDialog} modelId={infoModelId} />
{/if}
tools/server/webui/src/lib/components/app/models/index.ts
+29 -1
View File
@@ -44,6 +44,27 @@
*/
export { default as ModelsSelector } from './ModelsSelector.svelte';
/**
* **ModelsSelectorList** - Grouped model options list
*
* Renders grouped model options (loaded, favourites, available) with section
* headers and org subgroups. Shared between ModelsSelector and ModelsSelectorSheet
* to avoid template duplication.
*
* Accepts an optional `renderOption` snippet to customize how each option is
* rendered (e.g., to add keyboard navigation or highlighting).
*/
export { default as ModelsSelectorList } from './ModelsSelectorList.svelte';
/**
* **ModelsSelectorOption** - Single model option row
*
* Renders a single model option with selection state, favourite toggle,
* load/unload actions, status indicators, and an info button.
* Used inside ModelsSelectorList or directly in custom render snippets.
*/
export { default as ModelsSelectorOption } from './ModelsSelectorOption.svelte';
/**
* **ModelsSelectorSheet** - Mobile model selection sheet
*
@@ -80,5 +101,12 @@ export { default as ModelsSelectorSheet } from './ModelsSelectorSheet.svelte';
* ```
*/
export { default as ModelBadge } from './ModelBadge.svelte';
/**
* **ModelId** - Parsed model identifier display
*
* Displays a model ID with optional org name, parameter badges, quantization,
* aliases, and tags. Supports raw mode to show the unprocessed model name.
* Respects the user's `showRawModelNames` setting.
*/
export { default as ModelId } from './ModelId.svelte';
export { default as ModelsSelectorOption } from './ModelsSelectorOption.svelte';
tools/server/webui/src/lib/components/app/models/utils.ts
+75
View File
@@ -0,0 +1,75 @@
import { SvelteMap } from 'svelte/reactivity';
import type { ModelOption } from '$lib/types/models';
export interface ModelItem {
option: ModelOption;
flatIndex: number;
}
export interface OrgGroup {
orgName: string | null;
items: ModelItem[];
}
export interface GroupedModelOptions {
loaded: ModelItem[];
favourites: ModelItem[];
available: OrgGroup[];
}
export function filterModelOptions(options: ModelOption[], searchTerm: string): ModelOption[] {
const term = searchTerm.trim().toLowerCase();
if (!term) return options;
return options.filter(
(option) =>
option.model.toLowerCase().includes(term) ||
option.name?.toLowerCase().includes(term) ||
option.aliases?.some((alias: string) => alias.toLowerCase().includes(term)) ||
option.tags?.some((tag: string) => tag.toLowerCase().includes(term))
);
}
export function groupModelOptions(
filteredOptions: ModelOption[],
favouriteIds: Set<string>,
isModelLoaded: (model: string) => boolean
): GroupedModelOptions {
// Loaded models
const loaded: ModelItem[] = [];
for (let i = 0; i < filteredOptions.length; i++) {
if (isModelLoaded(filteredOptions[i].model)) {
loaded.push({ option: filteredOptions[i], flatIndex: i });
}
}
// Favourites (excluding loaded)
const loadedModelIds = new Set(loaded.map((item) => item.option.model));
const favourites: ModelItem[] = [];
for (let i = 0; i < filteredOptions.length; i++) {
if (
favouriteIds.has(filteredOptions[i].model) &&
!loadedModelIds.has(filteredOptions[i].model)
) {
favourites.push({ option: filteredOptions[i], flatIndex: i });
}
}
// Available models grouped by org (excluding loaded and favourites)
const available: OrgGroup[] = [];
const orgGroups = new SvelteMap<string, ModelItem[]>();
for (let i = 0; i < filteredOptions.length; i++) {
const option = filteredOptions[i];
if (loadedModelIds.has(option.model) || favouriteIds.has(option.model)) continue;
const key = option.parsedId?.orgName ?? '';
if (!orgGroups.has(key)) orgGroups.set(key, []);
orgGroups.get(key)!.push({ option, flatIndex: i });
}
for (const [orgName, items] of orgGroups) {
available.push({ orgName: orgName || null, items });
}
return { loaded, favourites, available };
}