diff --git a/common/common.cpp b/common/common.cpp index 7f75dd38b..6812399e3 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -259,7 +259,7 @@ bool set_process_priority(enum ggml_sched_priority prio) { case GGML_SCHED_PRIO_REALTIME: p = -20; break; } - if (!setpriority(PRIO_PROCESS, 0, p)) { + if (setpriority(PRIO_PROCESS, 0, p) != 0) { LOG_WRN("failed to set process priority %d : %s (%d)\n", prio, strerror(errno), errno); return false; } diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 69abb7367..f893b24c7 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -1696,6 +1696,84 @@ class TextModel(ModelBase): if template is not None: self.gguf_writer.add_chat_template(template) + def _set_vocab_plamo(self): + # PLaMo models use a custom tokenizer with a .jsonl file + tokenizer_jsonl_path = self.dir_model / "tokenizer.jsonl" + tokenizer_config_path = self.dir_model / "tokenizer_config.json" + + if not tokenizer_jsonl_path.is_file(): + raise FileNotFoundError(f"PLaMo tokenizer file not found: {tokenizer_jsonl_path}") + + # Load tokenizer config + with open(tokenizer_config_path, "r", encoding="utf-8") as f: + tokenizer_config = json.load(f) + + # Load tokens from JSONL file (actually a list format) + tokens = [] + scores = [] + toktypes = [] + + with open(tokenizer_jsonl_path, "r", encoding="utf-8") as f: + for line_num, line in enumerate(f): + if line.strip(): + token_data = json.loads(line) + # Format: [token, score, type, ?, ?, ?, ?] + token = token_data[0].encode("utf-8") + score = float(token_data[1]) + token_type_str = token_data[2] if len(token_data) > 2 else "NORMAL" + + tokens.append(token) + scores.append(score) + + if token_type_str == "UNKNOWN": + toktypes.append(gguf.TokenType.UNKNOWN) + elif token_type_str == "CONTROL": + toktypes.append(gguf.TokenType.CONTROL) + elif token_type_str == "BYTE": + toktypes.append(gguf.TokenType.BYTE) + else: + token_str = token_data[0] + if token_str.startswith("<|plamo:") and token_str.endswith("|>"): + toktypes.append(gguf.TokenType.CONTROL) + else: + toktypes.append(gguf.TokenType.NORMAL) + + vocab_size = self.hparams["vocab_size"] + if vocab_size > len(tokens): + pad_count = vocab_size - len(tokens) + logger.debug(f"Padding vocab with {pad_count} token(s) - [PAD1] through [PAD{pad_count}]") + for i in range(1, pad_count + 1): + tokens.append(bytes(f"[PAD{i}]", encoding="utf-8")) + scores.append(-1000.0) + toktypes.append(gguf.TokenType.UNUSED) + + self.gguf_writer.add_tokenizer_model("plamo2") + self.gguf_writer.add_tokenizer_pre("default") + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_scores(scores) + self.gguf_writer.add_token_types(toktypes) + + if "bos_token" in tokenizer_config and tokenizer_config["bos_token"] is not None: + token_id = tokens.index(tokenizer_config["bos_token"].encode("utf-8")) + self.gguf_writer.add_bos_token_id(token_id) + if "eos_token" in tokenizer_config and tokenizer_config["eos_token"] is not None: + token_id = tokens.index(tokenizer_config["eos_token"].encode("utf-8")) + self.gguf_writer.add_eos_token_id(token_id) + if "pad_token" in tokenizer_config and tokenizer_config["pad_token"] is not None: + token_id = tokens.index(tokenizer_config["pad_token"].encode("utf-8")) + self.gguf_writer.add_pad_token_id(token_id) + if "sep_token" in tokenizer_config and tokenizer_config["sep_token"] is not None: + token_id = tokens.index(tokenizer_config["sep_token"].encode("utf-8")) + self.gguf_writer.add_sep_token_id(token_id) + if "unk_token" in tokenizer_config and tokenizer_config["unk_token"] is not None: + token_id = tokens.index(tokenizer_config["unk_token"].encode("utf-8")) + self.gguf_writer.add_unk_token_id(token_id) + + # Add <|plamo:op|> as EOT to ensure appropriate end of generation + self.gguf_writer.add_eot_token_id(4) + + self.gguf_writer.add_add_space_prefix(False) + class MmprojModel(ModelBase): model_type = ModelType.MMPROJ @@ -4798,87 +4876,7 @@ class Plamo2Model(TextModel): model_arch = gguf.MODEL_ARCH.PLAMO2 def set_vocab(self): - # PLaMo 2 uses a custom tokenizer with a .jsonl file - # We need to handle this specially - tokenizer_jsonl_path = self.dir_model / "tokenizer.jsonl" - tokenizer_config_path = self.dir_model / "tokenizer_config.json" - - if not tokenizer_jsonl_path.is_file(): - raise FileNotFoundError(f"PLaMo 2 tokenizer file not found: {tokenizer_jsonl_path}") - - # Load tokenizer config - with open(tokenizer_config_path, 'r', encoding='utf-8') as f: - tokenizer_config = json.load(f) - - # Load tokens from JSONL file (actually a list format) - tokens = [] - scores = [] - toktypes = [] - - with open(tokenizer_jsonl_path, 'r', encoding='utf-8') as f: - for line_num, line in enumerate(f): - if line.strip(): - token_data = json.loads(line) - # Format: [token, score, type, ?, ?, ?, ?] - token = token_data[0].encode("utf-8") - score = float(token_data[1]) - token_type_str = token_data[2] if len(token_data) > 2 else "NORMAL" - - tokens.append(token) - scores.append(score) - - # Map token type strings to GGUF token types - if token_type_str == "UNKNOWN": - toktypes.append(gguf.TokenType.UNKNOWN) - elif token_type_str == "CONTROL": - toktypes.append(gguf.TokenType.CONTROL) - elif token_type_str == "BYTE": - toktypes.append(gguf.TokenType.BYTE) - else: - # Check for PLaMo-2 special tokens - token_str = token_data[0] - if token_str.startswith("<|plamo:") and token_str.endswith("|>"): - toktypes.append(gguf.TokenType.CONTROL) - else: - toktypes.append(gguf.TokenType.NORMAL) - - vocab_size = self.hparams["vocab_size"] - if vocab_size > len(tokens): - pad_count = vocab_size - len(tokens) - logger.debug(f"Padding vocab with {pad_count} token(s) - [PAD1] through [PAD{pad_count}]") - for i in range(1, pad_count + 1): - tokens.append(bytes(f"[PAD{i}]", encoding="utf-8")) - scores.append(-1000.0) - toktypes.append(gguf.TokenType.UNUSED) - - # Use "plamo2" tokenizer type for PLaMo-2's custom Aho-Corasick tokenizer - self.gguf_writer.add_tokenizer_model("plamo2") - self.gguf_writer.add_tokenizer_pre("default") - self.gguf_writer.add_token_list(tokens) - self.gguf_writer.add_token_scores(scores) - self.gguf_writer.add_token_types(toktypes) - - # Add special tokens from config - if "bos_token" in tokenizer_config and tokenizer_config["bos_token"] is not None: - token_id = tokens.index(tokenizer_config["bos_token"].encode("utf-8")) - self.gguf_writer.add_bos_token_id(token_id) - if "eos_token" in tokenizer_config and tokenizer_config["eos_token"] is not None: - token_id = tokens.index(tokenizer_config["eos_token"].encode("utf-8")) - self.gguf_writer.add_eos_token_id(token_id) - if "pad_token" in tokenizer_config and tokenizer_config["pad_token"] is not None: - token_id = tokens.index(tokenizer_config["pad_token"].encode("utf-8")) - self.gguf_writer.add_pad_token_id(token_id) - if "sep_token" in tokenizer_config and tokenizer_config["sep_token"] is not None: - token_id = tokens.index(tokenizer_config["sep_token"].encode("utf-8")) - self.gguf_writer.add_sep_token_id(token_id) - if "unk_token" in tokenizer_config and tokenizer_config["unk_token"] is not None: - token_id = tokens.index(tokenizer_config["unk_token"].encode("utf-8")) - self.gguf_writer.add_unk_token_id(token_id) - - # Add <|plamo:op|> as EOT to ensure appropriate end of generation - self.gguf_writer.add_eot_token_id(4) - - self.gguf_writer.add_add_space_prefix(False) + self._set_vocab_plamo() def set_gguf_parameters(self): hparams = self.hparams @@ -4966,6 +4964,56 @@ class Plamo2Model(TextModel): return [(new_name, data_torch)] +@ModelBase.register("Plamo3ForCausalLM", "PLaMo3ForCausalLM") +class Plamo3Model(TextModel): + model_arch = gguf.MODEL_ARCH.PLAMO3 + + def set_vocab(self): + self._set_vocab_plamo() + + tokenizer_config_path = self.dir_model / "tokenizer_config.json" + tokenizer_config = {} + + if tokenizer_config_path.is_file(): + with open(tokenizer_config_path, encoding="utf-8") as f: + tokenizer_config = json.load(f) + + chat_template = tokenizer_config.get("chat_template") + chat_template_jinja = self.dir_model / "chat_template.jinja" + + if chat_template_jinja.is_file(): + with open(chat_template_jinja, encoding="utf-8") as f: + chat_template = f.read() + + if chat_template: + self.gguf_writer.add_chat_template(chat_template) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + self.gguf_writer.add_vocab_size(self.hparams["vocab_size"]) + if (sliding_window := self.find_hparam(["window_size", "sliding_window"], optional=True)) is not None: + self.gguf_writer.add_sliding_window(sliding_window) + self.gguf_writer.add_sliding_window_pattern(self.hparams["sliding_window_pattern"]) + self.gguf_writer.add_rope_freq_base_swa(self.rope_parameters.get("sliding_attention", {"rope_theta": self.hparams.get("rope_local_theta")})["rope_theta"]) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + + if name.endswith(".pre_mixer_norm.weight"): + data_torch = data_torch + 1.0 + elif name.endswith(".post_mixer_norm.weight"): + data_torch = data_torch + 1.0 / 5 + elif name.endswith(".pre_mlp_norm.weight"): + data_torch = data_torch + 1.0 + elif name.endswith(".post_mlp_norm.weight"): + data_torch = data_torch + 1.0 / (5**1.5) + elif name.endswith((".mixer.q_norm.weight", ".mixer.k_norm.weight")): + data_torch = data_torch + 1.0 + elif name.endswith(".norm.weight"): + data_torch = data_torch + 1.0 + + return [(self.map_tensor_name(name), data_torch)] + + @ModelBase.register("CodeShellForCausalLM") class CodeShellModel(TextModel): model_arch = gguf.MODEL_ARCH.CODESHELL diff --git a/embd_res/klite.embd b/embd_res/klite.embd index b87b259b7..013f47c0f 100644 --- a/embd_res/klite.embd +++ b/embd_res/klite.embd @@ -1166,8 +1166,8 @@ Current version indicated by LITEVER below. } .popupfooter button { width: 100px; - margin-left: 10px; - margin-right: 10px; + margin-left: 6px; + margin-right: 6px; } /* World info table */ @@ -20652,6 +20652,43 @@ Current version indicated by LITEVER below. currmediaidx = 0; } + function download_curr_media() { + let elementId = document.getElementById("zoomedaudio")?"zoomedaudio":"zoomedimg"; + let filename = document.getElementById("zoomedaudio")?"audio.mp3":"image.jpg"; + let triggerDownload = function(url, filename) { + var a = document.createElement("a"); + a.href = url; + a.download = filename; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + // Cleanup blob URLs + if (url.indexOf("blob:") === 0) { + setTimeout(function () { + URL.revokeObjectURL(url); + }, 1000); + } + } + var el = document.getElementById(elementId); + if (!el || !el.src) return; + var src = el.src; + if (src.indexOf("blob:") === 0) { + var xhr = new XMLHttpRequest(); + xhr.open("GET", src, true); + xhr.responseType = "blob"; + + xhr.onload = function () { + if (xhr.status === 200) { + triggerDownload(URL.createObjectURL(xhr.response), filename); + } + }; + xhr.send(); + } + else { + triggerDownload(src, filename); + } + } + function render_all_media_html(text, float=true, center=false) { let siclass = (float?"storyimgfloat":(center?"storyimgcenter":"storyimgsidevertical")); @@ -29511,7 +29548,8 @@ Current version indicated by LITEVER below.
- + +
diff --git a/ggml/src/ggml-cuda/cumsum.cu b/ggml/src/ggml-cuda/cumsum.cu index e82171f9c..3bd1394c5 100644 --- a/ggml/src/ggml-cuda/cumsum.cu +++ b/ggml/src/ggml-cuda/cumsum.cu @@ -61,7 +61,7 @@ static __global__ void cumsum_cub_kernel( // Add offset to each item and store T thread_offset = thread_prefix - thread_sum + block_carry; - #pragma unroll +#pragma unroll for (int i = 0; i < UNROLL_FACTOR; i++) { int64_t idx = start + tid * UNROLL_FACTOR + i; if (idx < ne00) { @@ -69,11 +69,12 @@ static __global__ void cumsum_cub_kernel( } } + __syncthreads(); + // Update carry for next tile if (tid == 0) { block_carry += block_total; } - __syncthreads(); } #else NO_DEVICE_CODE; @@ -175,11 +176,12 @@ static __global__ void cumsum_kernel( } } + __syncthreads(); + // Update carry for next chunk if (tid == 0) { *s_carry += *s_chunk_total; } - __syncthreads(); } } diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index f91598a13..b78dd36b9 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -2212,7 +2212,7 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor const int cc = ggml_cuda_info().devices[id].cc; const int warp_size = ggml_cuda_info().devices[id].warp_size; - use_mul_mat_q = use_mul_mat_q && ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[1]); + use_mul_mat_q = use_mul_mat_q && ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[1], /*n_experts=*/0); use_mul_mat_f = use_mul_mat_f && ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, src1->ne[1], /*mul_mat_id=*/false); use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, src1->ne[1]); any_gpus_with_slow_fp16 = any_gpus_with_slow_fp16 || !fast_fp16_hardware_available(cc); @@ -2220,7 +2220,7 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor } else { const int cc = ggml_cuda_info().devices[ctx.device].cc; const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size; - use_mul_mat_q = use_mul_mat_q && ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[1]); + use_mul_mat_q = use_mul_mat_q && ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[1], /*n_experts=*/0); use_mul_mat_f = use_mul_mat_f && ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, src1->ne[1], /*mul_mat_id=*/false); use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, src1->ne[1]); any_gpus_with_slow_fp16 = any_gpus_with_slow_fp16 || !fast_fp16_hardware_available(cc); @@ -2296,7 +2296,7 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * return; } - if (ggml_cuda_should_use_mmq(src0->type, cc, ne12)) { + if (ggml_cuda_should_use_mmq(src0->type, cc, ne12, /*n_experts=*/ne02)) { ggml_cuda_mul_mat_q(ctx, src0, src1, ids, dst); return; } @@ -4799,6 +4799,16 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t features.push_back({ "FA_ALL_QUANTS", "1" }); #endif + { + const auto & info = ggml_cuda_info(); + for (int id = 0; id < info.device_count; ++id) { + if (blackwell_mma_available(info.devices[id].cc)) { + features.push_back({ "BLACKWELL_NATIVE_FP4", "1"}); + break; + } + } + } + #undef _STRINGIFY #undef STRINGIFY diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu index 7af0af357..53487ac15 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -259,7 +259,7 @@ void ggml_cuda_op_mul_mat_q( GGML_UNUSED_VARS(src1, dst, src1_ddf_i, src1_padded_row_size); } -bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11) { +bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts) { if(!g_mul_mat_q) { @@ -322,7 +322,10 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11) { if (GGML_CUDA_CC_IS_CDNA3(cc)) { return true; } - if (ne11 <= 128 || type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 || type == GGML_TYPE_Q5_0 || type == GGML_TYPE_Q5_1) { + if (n_experts > 64 || ne11 <= 128) { + return true; + } + if (type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 || type == GGML_TYPE_Q5_0 || type == GGML_TYPE_Q5_1) { return true; } if (ne11 <= 256 && (type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K)) { diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index 31fa50834..607d33973 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -4083,6 +4083,6 @@ void ggml_cuda_op_mul_mat_q( const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols, const int64_t src1_padded_row_size, cudaStream_t stream); -bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11); +bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts); extern bool g_mul_mat_q; diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 27578daaf..616b8add3 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -377,6 +377,7 @@ class MODEL_ARCH(IntEnum): PHIMOE = auto() PLAMO = auto() PLAMO2 = auto() + PLAMO3 = auto() CODESHELL = auto() ORION = auto() INTERNLM2 = auto() @@ -773,6 +774,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.PHIMOE: "phimoe", MODEL_ARCH.PLAMO: "plamo", MODEL_ARCH.PLAMO2: "plamo2", + MODEL_ARCH.PLAMO3: "plamo3", MODEL_ARCH.CODESHELL: "codeshell", MODEL_ARCH.ORION: "orion", MODEL_ARCH.INTERNLM2: "internlm2", @@ -1763,6 +1765,21 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.SSM_B_NORM, MODEL_TENSOR.SSM_C_NORM, ], + MODEL_ARCH.PLAMO3: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, + MODEL_TENSOR.ATTN_Q_NORM, + MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_POST_NORM, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_POST_NORM, + ], MODEL_ARCH.GPT2: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.POS_EMBD, diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 1690d991f..115df6c7c 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -595,6 +595,7 @@ class TensorNameMap: "encoder.layer.{bid}.attention.self.layer_norm_q", # jina-bert-v2 "transformer.layers.{bid}.attn.q_norm", # openelm "model.layers.layers.{bid}.mixer.q", # plamo2 + "model.layers.layers.{bid}.mixer.q_norm", # plamo3 "layers.{bid}.self_attn.q_norm", # qwen3-embedding "model.layers.{bid}.attention.query_layernorm", # apertus ), @@ -610,6 +611,7 @@ class TensorNameMap: "encoder.layer.{bid}.attention.self.layer_norm_k", # jina-bert-v2 "transformer.layers.{bid}.attn.k_norm", # openelm "model.layers.layers.{bid}.mixer.k", # plamo2 + "model.layers.layers.{bid}.mixer.k_norm", # plamo3 "layers.{bid}.self_attn.k_norm", # qwen3-embedding "model.layers.{bid}.attention.key_layernorm", # apertus ), diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 75013d8d3..94a6807ea 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -42,6 +42,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_PHIMOE, "phimoe" }, { LLM_ARCH_PLAMO, "plamo" }, { LLM_ARCH_PLAMO2, "plamo2" }, + { LLM_ARCH_PLAMO3, "plamo3" }, { LLM_ARCH_CODESHELL, "codeshell" }, { LLM_ARCH_ORION, "orion" }, { LLM_ARCH_INTERNLM2, "internlm2" }, @@ -1077,6 +1078,22 @@ static std::set llm_get_tensor_names(llm_arch arch) { LLM_TENSOR_ATTN_POST_NORM, LLM_TENSOR_FFN_POST_NORM, }; + case LLM_ARCH_PLAMO3: + return { + LLM_TENSOR_TOKEN_EMBD, + LLM_TENSOR_OUTPUT_NORM, + LLM_TENSOR_OUTPUT, + LLM_TENSOR_ATTN_NORM, + LLM_TENSOR_ATTN_QKV, + LLM_TENSOR_ATTN_Q_NORM, + LLM_TENSOR_ATTN_K_NORM, + LLM_TENSOR_ATTN_OUT, + LLM_TENSOR_ATTN_POST_NORM, + LLM_TENSOR_FFN_NORM, + LLM_TENSOR_FFN_POST_NORM, + LLM_TENSOR_FFN_DOWN, + LLM_TENSOR_FFN_UP, + }; case LLM_ARCH_CODESHELL: return { LLM_TENSOR_TOKEN_EMBD, diff --git a/src/llama-arch.h b/src/llama-arch.h index 27bdedc83..714ead402 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -46,6 +46,7 @@ enum llm_arch { LLM_ARCH_PHIMOE, LLM_ARCH_PLAMO, LLM_ARCH_PLAMO2, + LLM_ARCH_PLAMO3, LLM_ARCH_CODESHELL, LLM_ARCH_ORION, LLM_ARCH_INTERNLM2, diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 359044b2f..eed102b25 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -98,6 +98,7 @@ #include "models/phi3.cpp" #include "models/plamo.cpp" #include "models/plamo2.cpp" +#include "models/plamo3.cpp" #include "models/plm.cpp" #include "models/qwen.cpp" #include "models/qwen2.cpp" @@ -1334,6 +1335,26 @@ void llama_model::load_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH, hparams.n_embd_head_k, false); ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH, hparams.n_embd_head_v, false); } break; + case LLM_ARCH_PLAMO3: + { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); + if (found_swa && hparams.n_swa > 0) { + uint32_t swa_period = 8; + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + hparams.rope_freq_scale_train_swa = 1.0f; + ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa); + ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); + hparams.set_swa_pattern(swa_period); + } else { + hparams.swa_type = LLAMA_SWA_TYPE_NONE; + } + + switch (hparams.n_layer) { + case 24: type = LLM_TYPE_2B; break; + default: type = LLM_TYPE_UNKNOWN; + } + } break; case LLM_ARCH_GPT2: { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); @@ -3988,6 +4009,44 @@ bool llama_model::load_tensors(llama_model_loader & ml) { layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, i), {n_embd}, 0); } } break; + case LLM_ARCH_PLAMO3: + { + const int64_t head_dim_q = hparams.n_embd_head_k; + const int64_t head_dim_v = hparams.n_embd_head_v; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + const int64_t num_attention_heads = hparams.n_head(i); + const int64_t num_key_value_heads = hparams.n_head_kv(i); + const int64_t q_proj_dim = num_attention_heads * head_dim_q; + const int64_t k_proj_dim = num_key_value_heads * head_dim_q; + const int64_t v_proj_dim = num_key_value_heads * head_dim_v; + const int64_t n_ff_cur = hparams.n_ff(i); + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), + {n_embd,q_proj_dim + k_proj_dim + v_proj_dim}, 0); + layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {head_dim_q}, 0); + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {head_dim_q}, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {num_attention_heads * head_dim_v, n_embd}, 0); + layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, i), {n_embd}, 0); + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, i), {n_embd}, 0); + + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff_cur * 2}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff_cur, n_embd}, 0); + } + } break; case LLM_ARCH_GPT2: { tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); @@ -7637,6 +7696,14 @@ ggml_cgraph * llama_model::build_graph(const llm_graph_params & params) const { { llm = std::make_unique(*this, params); } break; + case LLM_ARCH_PLAMO3: + { + if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) { + llm = std::make_unique> (*this, params); + } else { + llm = std::make_unique>(*this, params); + } + } break; case LLM_ARCH_GPT2: { llm = std::make_unique(*this, params); @@ -8141,6 +8208,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_PHIMOE: case LLM_ARCH_PLAMO: case LLM_ARCH_PLAMO2: + case LLM_ARCH_PLAMO3: case LLM_ARCH_GEMMA: case LLM_ARCH_GEMMA2: case LLM_ARCH_GEMMA3: diff --git a/src/models/models.h b/src/models/models.h index dd0e286ed..e2cd4e484 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -406,6 +406,11 @@ struct llm_build_plamo : public llm_graph_context { llm_build_plamo(const llama_model & model, const llm_graph_params & params); }; +template +struct llm_build_plamo3 : public llm_graph_context { + llm_build_plamo3(const llama_model & model, const llm_graph_params & params); +}; + struct llm_build_plm : public llm_graph_context { llm_build_plm(const llama_model & model, const llm_graph_params & params); }; diff --git a/src/models/plamo3.cpp b/src/models/plamo3.cpp new file mode 100644 index 000000000..55c806467 --- /dev/null +++ b/src/models/plamo3.cpp @@ -0,0 +1,128 @@ +#include "models.h" + +template +llm_build_plamo3::llm_build_plamo3(const llama_model & model, const llm_graph_params & params) : + llm_graph_context(params) { + const int64_t head_dim_q = hparams.n_embd_head_k; + const int64_t head_dim_v = hparams.n_embd_head_v; + + ggml_tensor * cur; + ggml_tensor * inpL = build_inp_embd(model.tok_embd); + ggml_tensor * inp_pos = build_inp_pos(); + + using inp_attn_type = std::conditional_t; + inp_attn_type * inp_attn = nullptr; + + if constexpr (iswa) { + inp_attn = build_attn_inp_kv_iswa(); + } else { + inp_attn = build_attn_inp_kv(); + } + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * residual = inpL; + + float freq_base_l = 0.0f; + float freq_scale_l = 0.0f; + if constexpr (iswa) { + freq_base_l = model.get_rope_freq_base (cparams, il); + freq_scale_l = model.get_rope_freq_scale(cparams, il); + } else { + freq_base_l = freq_base; + freq_scale_l = freq_scale; + } + + cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + ggml_tensor * qkv = build_lora_mm(model.layers[il].wqkv, cur); + cb(cur, "wqkv", il); + + const int32_t n_head = hparams.n_head(il); + const int32_t n_head_kv = hparams.n_head_kv(il); + + const int64_t q_offset = 0; + const int64_t k_offset = head_dim_q * n_head; + const int64_t v_offset = k_offset + head_dim_q * n_head_kv; + + ggml_tensor * Qcur = ggml_view_3d(ctx0, qkv, head_dim_q, n_head, n_tokens, + head_dim_q * sizeof(float), qkv->nb[1], q_offset * ggml_element_size(qkv)); + ggml_tensor * Kcur = ggml_view_3d(ctx0, qkv, head_dim_q, n_head_kv, n_tokens, + head_dim_q * sizeof(float), qkv->nb[1], k_offset * ggml_element_size(qkv)); + ggml_tensor * Vcur = ggml_view_3d(ctx0, qkv, head_dim_v, n_head_kv, n_tokens, + head_dim_v * sizeof(float), qkv->nb[1], v_offset * ggml_element_size(qkv)); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); + cb(Qcur, "attn_q_norm", il); + Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); + cb(Kcur, "attn_k_norm", il); + + Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, + ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, + ext_factor, attn_factor, beta_fast, beta_slow); + + const float attn_scale = 1.0f / sqrtf(float(head_dim_q)); + + cur = build_attn(inp_attn, + model.layers[il].wo, NULL, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, attn_scale, il); + cb(cur, "attn_out", il); + + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + residual = ggml_get_rows(ctx0, residual, inp_out_ids); + } + + cur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "attn_post_norm", il); + + cur = ggml_add(ctx0, cur, residual); + cb(cur, "attn_residual", il); + + residual = cur; + + cur = build_norm(cur, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + NULL, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SWIGLU, LLM_FFN_SEQ, il); + cb(cur, "ffn_out", il); + + cur = build_norm(cur, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "ffn_post_norm", il); + + cur = ggml_add(ctx0, cur, residual); + cb(cur, "ffn_residual", il); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} + +// Explicit template instantiations +template struct llm_build_plamo3; +template struct llm_build_plamo3; diff --git a/tools/completion/completion.cpp b/tools/completion/completion.cpp index c497a7097..45a80f36c 100644 --- a/tools/completion/completion.cpp +++ b/tools/completion/completion.cpp @@ -176,7 +176,10 @@ int main(int argc, char ** argv) { struct ggml_threadpool_params tpp = ggml_threadpool_params_from_cpu_params(params.cpuparams); - set_process_priority(params.cpuparams.priority); + if (!set_process_priority(params.cpuparams.priority)) { + LOG_ERR("%s: error: failed to set process priority\n", __func__); + return 1; + } struct ggml_threadpool * threadpool_batch = NULL; if (!ggml_threadpool_params_match(&tpp, &tpp_batch)) { diff --git a/tools/server/public/index.html.gz b/tools/server/public/index.html.gz index cf5c625b4..d1c10eed9 100644 Binary files a/tools/server/public/index.html.gz and b/tools/server/public/index.html.gz differ diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 1abbf6d6d..9726e0252 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2960,19 +2960,22 @@ std::unique_ptr server_routes::handle_completions_impl( // in streaming mode, the first error must be treated as non-stream response // this is to match the OAI API behavior // ref: https://github.com/ggml-org/llama.cpp/pull/16486#discussion_r2419657309 - server_task_result_ptr first_result = rd.next(req.should_stop); + auto first_result = rd.next(req.should_stop); if (first_result == nullptr) { + GGML_ASSERT(req.should_stop()); return res; // connection is closed - } else if (first_result->is_error()) { + } + + if (first_result->is_error()) { res->error(first_result->to_json()); return res; - } else { - GGML_ASSERT( - dynamic_cast(first_result.get()) != nullptr - || dynamic_cast(first_result.get()) != nullptr - ); } + GGML_ASSERT( + dynamic_cast(first_result.get()) != nullptr || + dynamic_cast (first_result.get()) != nullptr + ); + // next responses are streamed // to be sent immediately json first_result_json = first_result->to_json(); @@ -3028,6 +3031,7 @@ std::unique_ptr server_routes::handle_completions_impl( auto result = rd.next(req.should_stop); if (result == nullptr) { SRV_DBG("%s", "stopping streaming due to should_stop condition\n"); + GGML_ASSERT(req.should_stop()); return false; // should_stop condition met } @@ -3111,6 +3115,11 @@ void server_routes::init_routes() { // get the result auto result = res->rd.next(req.should_stop); + if (!result) { + // connection was closed + GGML_ASSERT(req.should_stop()); + return res; + } if (result->is_error()) { res->error(result->to_json()); @@ -3211,6 +3220,11 @@ void server_routes::init_routes() { // get the result auto result = res->rd.next(req.should_stop); + if (!result) { + // connection was closed + GGML_ASSERT(req.should_stop()); + return res; + } if (result->is_error()) { res->error(result->to_json()); @@ -3717,7 +3731,12 @@ void server_routes::init_routes() { } // get the result - server_task_result_ptr result = rd.next(req.should_stop); + auto result = rd.next(req.should_stop); + if (!result) { + // connection was closed + GGML_ASSERT(req.should_stop()); + return res; + } if (result->is_error()) { res->error(result->to_json()); @@ -3746,7 +3765,12 @@ void server_routes::init_routes() { } // get the result - server_task_result_ptr result = rd.next(req.should_stop); + auto result = rd.next(req.should_stop); + if (!result) { + // connection was closed + GGML_ASSERT(req.should_stop()); + return res; + } if (result->is_error()) { res->error(result->to_json()); @@ -3779,7 +3803,12 @@ std::unique_ptr server_routes::handle_slots_save(const ser rd.post_task(std::move(task)); } - server_task_result_ptr result = rd.next(req.should_stop); + auto result = rd.next(req.should_stop); + if (!result) { + // connection was closed + GGML_ASSERT(req.should_stop()); + return res; + } if (result->is_error()) { res->error(result->to_json()); @@ -3810,7 +3839,12 @@ std::unique_ptr server_routes::handle_slots_restore(const rd.post_task(std::move(task)); } - server_task_result_ptr result = rd.next(req.should_stop); + auto result = rd.next(req.should_stop); + if (!result) { + // connection was closed + GGML_ASSERT(req.should_stop()); + return res; + } if (result->is_error()) { res->error(result->to_json()); @@ -3832,7 +3866,12 @@ std::unique_ptr server_routes::handle_slots_erase(const se rd.post_task(std::move(task)); } - server_task_result_ptr result = rd.next(req.should_stop); + auto result = rd.next(req.should_stop); + if (!result) { + // connection was closed + GGML_ASSERT(req.should_stop()); + return res; + } if (result->is_error()) { res->error(result->to_json()); diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index cb7e70455..56e1dc46b 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -662,7 +662,10 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co req.path, req.headers, req.body, - req.should_stop); + req.should_stop, + base_params.timeout_read, + base_params.timeout_write + ); return proxy; } @@ -950,13 +953,18 @@ server_http_proxy::server_http_proxy( const std::string & path, const std::map & headers, const std::string & body, - const std::function should_stop) { + const std::function should_stop, + int32_t timeout_read, + int32_t timeout_write + ) { // shared between reader and writer threads auto cli = std::make_shared(host, port); auto pipe = std::make_shared>(); // setup Client cli->set_connection_timeout(0, 200000); // 200 milliseconds + cli->set_write_timeout(timeout_read, 0); // reversed for cli (client) vs srv (server) + cli->set_read_timeout(timeout_write, 0); this->status = 500; // to be overwritten upon response this->cleanup = [pipe]() { pipe->close_read(); diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 7e3353753..24ddc6566 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -183,7 +183,10 @@ public: const std::string & path, const std::map & headers, const std::string & body, - const std::function should_stop); + const std::function should_stop, + int32_t timeout_read, + int32_t timeout_write + ); ~server_http_proxy() { if (cleanup) { cleanup(); diff --git a/tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageAssistant.svelte b/tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageAssistant.svelte index 8997963f1..c1ef4dfd0 100644 --- a/tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageAssistant.svelte +++ b/tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageAssistant.svelte @@ -89,6 +89,7 @@ const fallbackToolCalls = $derived(typeof toolCallContent === 'string' ? toolCallContent : null); const processingState = useProcessingState(); + let currentConfig = $derived(config()); let isRouter = $derived(isRouterMode()); let displayedModel = $derived((): string | null => { @@ -116,6 +117,12 @@ } }); + $effect(() => { + if (isLoading() && !message?.content?.trim()) { + processingState.startMonitoring(); + } + }); + function formatToolCallBadge(toolCall: ApiChatCompletionToolCall, index: number) { const callNumber = index + 1; const functionName = toolCall.function?.name?.trim(); @@ -186,7 +193,7 @@
- {processingState.getProcessingMessage()} + {processingState.getPromptProgressText() ?? processingState.getProcessingMessage()}
@@ -263,6 +270,23 @@ predictedTokens={message.timings.predicted_n} predictedMs={message.timings.predicted_ms} /> + {:else if isLoading() && currentConfig.showMessageStats} + {@const liveStats = processingState.getLiveProcessingStats()} + {@const genStats = processingState.getLiveGenerationStats()} + {@const promptProgress = processingState.processingState?.promptProgress} + {@const isStillProcessingPrompt = + promptProgress && promptProgress.processed < promptProgress.total} + + {#if liveStats || genStats} + + {/if} {/if} {/if} diff --git a/tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageStatistics.svelte b/tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageStatistics.svelte index a39acb1d7..24fe5926b 100644 --- a/tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageStatistics.svelte +++ b/tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageStatistics.svelte @@ -5,21 +5,64 @@ import { ChatMessageStatsView } from '$lib/enums'; interface Props { - predictedTokens: number; - predictedMs: number; + predictedTokens?: number; + predictedMs?: number; promptTokens?: number; promptMs?: number; + // Live mode: when true, shows stats during streaming + isLive?: boolean; + // Whether prompt processing is still in progress + isProcessingPrompt?: boolean; + // Initial view to show (defaults to READING in live mode) + initialView?: ChatMessageStatsView; } - let { predictedTokens, predictedMs, promptTokens, promptMs }: Props = $props(); + let { + predictedTokens, + predictedMs, + promptTokens, + promptMs, + isLive = false, + isProcessingPrompt = false, + initialView = ChatMessageStatsView.GENERATION + }: Props = $props(); - let activeView: ChatMessageStatsView = $state(ChatMessageStatsView.GENERATION); + let activeView: ChatMessageStatsView = $state(initialView); + let hasAutoSwitchedToGeneration = $state(false); - let tokensPerSecond = $derived((predictedTokens / predictedMs) * 1000); - let timeInSeconds = $derived((predictedMs / 1000).toFixed(2)); + // In live mode: auto-switch to GENERATION tab when prompt processing completes + $effect(() => { + if (isLive) { + // Auto-switch to generation tab only when prompt processing is done (once) + if ( + !hasAutoSwitchedToGeneration && + !isProcessingPrompt && + predictedTokens && + predictedTokens > 0 + ) { + activeView = ChatMessageStatsView.GENERATION; + hasAutoSwitchedToGeneration = true; + } else if (!hasAutoSwitchedToGeneration) { + // Stay on READING while prompt is still being processed + activeView = ChatMessageStatsView.READING; + } + } + }); + + let hasGenerationStats = $derived( + predictedTokens !== undefined && + predictedTokens > 0 && + predictedMs !== undefined && + predictedMs > 0 + ); + + let tokensPerSecond = $derived(hasGenerationStats ? (predictedTokens! / predictedMs!) * 1000 : 0); + let timeInSeconds = $derived( + predictedMs !== undefined ? (predictedMs / 1000).toFixed(2) : '0.00' + ); let promptTokensPerSecond = $derived( - promptTokens !== undefined && promptMs !== undefined + promptTokens !== undefined && promptMs !== undefined && promptMs > 0 ? (promptTokens / promptMs) * 1000 : undefined ); @@ -34,11 +77,14 @@ promptTokensPerSecond !== undefined && promptTimeInSeconds !== undefined ); + + // In live mode, generation tab is disabled until we have generation stats + let isGenerationDisabled = $derived(isLive && !hasGenerationStats);
- {#if hasPromptStats} + {#if hasPromptStats || isLive} -

Generation (token output)

+

+ {isGenerationDisabled + ? 'Generation (waiting for tokens...)' + : 'Generation (token output)'} +

- {#if activeView === ChatMessageStatsView.GENERATION} + {#if activeView === ChatMessageStatsView.GENERATION && hasGenerationStats} (null); + let lastKnownProcessingStats = $state(null); // Derive processing state reactively from chatStore's direct state const processingState = $derived.by(() => { @@ -46,6 +64,34 @@ export function useProcessingState(): UseProcessingStateReturn { } }); + // Track last known processing stats for when promptProgress disappears + $effect(() => { + if (processingState?.promptProgress) { + const { processed, total, time_ms, cache } = processingState.promptProgress; + const actualProcessed = processed - cache; + const actualTotal = total - cache; + + if (actualProcessed > 0 && time_ms > 0) { + const tokensPerSecond = actualProcessed / (time_ms / 1000); + lastKnownProcessingStats = { + tokensProcessed: actualProcessed, + totalTokens: actualTotal, + timeMs: time_ms, + tokensPerSecond + }; + } + } + }); + + function getETASecs(done: number, total: number, elapsedMs: number): number | undefined { + const elapsedSecs = elapsedMs / 1000; + const progressETASecs = + done === 0 || elapsedSecs < 0.5 + ? undefined // can be the case for the 0% progress report + : elapsedSecs * (total / done - 1); + return progressETASecs; + } + function startMonitoring(): void { if (isMonitoring) return; isMonitoring = true; @@ -59,28 +105,25 @@ export function useProcessingState(): UseProcessingStateReturn { const currentConfig = config(); if (!currentConfig.keepStatsVisible) { lastKnownState = null; + lastKnownProcessingStats = null; } } function getProcessingMessage(): string { - const state = processingState; - if (!state) { + if (!processingState) { return 'Processing...'; } - switch (state.status) { + switch (processingState.status) { case 'initializing': return 'Initializing...'; case 'preparing': - if (state.progressPercent !== undefined) { - return `Processing (${state.progressPercent}%)`; + if (processingState.progressPercent !== undefined) { + return `Processing (${processingState.progressPercent}%)`; } return 'Preparing response...'; case 'generating': - if (state.tokensDecoded > 0) { - return `Generating... (${state.tokensDecoded} tokens)`; - } - return 'Generating...'; + return ''; default: return 'Processing...'; } @@ -131,8 +174,76 @@ export function useProcessingState(): UseProcessingStateReturn { } function shouldShowDetails(): boolean { - const state = processingState; - return state !== null && state.status !== 'idle'; + return processingState !== null && processingState.status !== 'idle'; + } + + /** + * Returns a short progress message with percent + */ + function getPromptProgressText(): string | null { + if (!processingState?.promptProgress) return null; + + const { processed, total, cache } = processingState.promptProgress; + + const actualProcessed = processed - cache; + const actualTotal = total - cache; + const percent = Math.round((actualProcessed / actualTotal) * 100); + const eta = getETASecs(actualProcessed, actualTotal, processingState.promptProgress.time_ms); + + if (eta !== undefined) { + const etaSecs = Math.ceil(eta); + return `Processing ${percent}% (ETA: ${etaSecs}s)`; + } + + return `Processing ${percent}%`; + } + + /** + * Returns live processing statistics for display (prompt processing phase) + * Returns last known stats when promptProgress becomes unavailable + */ + function getLiveProcessingStats(): LiveProcessingStats | null { + if (processingState?.promptProgress) { + const { processed, total, time_ms, cache } = processingState.promptProgress; + + const actualProcessed = processed - cache; + const actualTotal = total - cache; + + if (actualProcessed > 0 && time_ms > 0) { + const tokensPerSecond = actualProcessed / (time_ms / 1000); + + return { + tokensProcessed: actualProcessed, + totalTokens: actualTotal, + timeMs: time_ms, + tokensPerSecond + }; + } + } + + // Return last known stats if promptProgress is no longer available + return lastKnownProcessingStats; + } + + /** + * Returns live generation statistics for display (token generation phase) + */ + function getLiveGenerationStats(): LiveGenerationStats | null { + if (!processingState) return null; + + const { tokensDecoded, tokensPerSecond } = processingState; + + if (tokensDecoded <= 0) return null; + + // Calculate time from tokens and speed + const timeMs = + tokensPerSecond && tokensPerSecond > 0 ? (tokensDecoded / tokensPerSecond) * 1000 : 0; + + return { + tokensGenerated: tokensDecoded, + timeMs, + tokensPerSecond: tokensPerSecond || 0 + }; } return { @@ -141,6 +252,9 @@ export function useProcessingState(): UseProcessingStateReturn { }, getProcessingDetails, getProcessingMessage, + getPromptProgressText, + getLiveProcessingStats, + getLiveGenerationStats, shouldShowDetails, startMonitoring, stopMonitoring diff --git a/tools/server/webui/src/lib/services/chat.ts b/tools/server/webui/src/lib/services/chat.ts index 8074d2f5c..d78175c9a 100644 --- a/tools/server/webui/src/lib/services/chat.ts +++ b/tools/server/webui/src/lib/services/chat.ts @@ -117,7 +117,8 @@ export class ChatService { role: msg.role, content: msg.content })), - stream + stream, + return_progress: stream ? true : undefined }; // Include model in request if provided (required in ROUTER mode) @@ -271,7 +272,7 @@ export class ChatService { onReasoningChunk?: (chunk: string) => void, onToolCallChunk?: (chunk: string) => void, onModel?: (model: string) => void, - onTimings?: (timings: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void, + onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void, conversationId?: string, abortSignal?: AbortSignal ): Promise { @@ -370,11 +371,13 @@ export class ChatService { onModel?.(chunkModel); } - if (timings || promptProgress) { + if (promptProgress) { + ChatService.notifyTimings(undefined, promptProgress, onTimings); + } + + if (timings) { ChatService.notifyTimings(timings, promptProgress, onTimings); - if (timings) { - lastTimings = timings; - } + lastTimings = timings; } if (content) { @@ -772,10 +775,11 @@ export class ChatService { timings: ChatMessageTimings | undefined, promptProgress: ChatMessagePromptProgress | undefined, onTimingsCallback: - | ((timings: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void) + | ((timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void) | undefined ): void { - if (!timings || !onTimingsCallback) return; + if (!onTimingsCallback || (!timings && !promptProgress)) return; + onTimingsCallback(timings, promptProgress); } } diff --git a/tools/server/webui/src/lib/stores/chat.svelte.ts b/tools/server/webui/src/lib/stores/chat.svelte.ts index 010889452..67157e36a 100644 --- a/tools/server/webui/src/lib/stores/chat.svelte.ts +++ b/tools/server/webui/src/lib/stores/chat.svelte.ts @@ -303,11 +303,17 @@ class ChatStore { const currentConfig = config(); const outputTokensMax = currentConfig.max_tokens || -1; + // Note: for timings data, the n_prompt does NOT include cache tokens const contextUsed = promptTokens + cacheTokens + predictedTokens; const outputTokensUsed = predictedTokens; + // Note: for prompt progress, the "processed" DOES include cache tokens + // we need to exclude them to get the real prompt tokens processed count + const progressCache = promptProgress?.cache || 0; + const progressActualDone = (promptProgress?.processed ?? 0) - progressCache; + const progressActualTotal = (promptProgress?.total ?? 0) - progressCache; const progressPercent = promptProgress - ? Math.round((promptProgress.processed / promptProgress.total) * 100) + ? Math.round((progressActualDone / progressActualTotal) * 100) : undefined; return { @@ -324,6 +330,7 @@ class ChatStore { topP: currentConfig.top_p ?? 0.95, speculative: false, progressPercent, + promptProgress, promptTokens, promptMs, cacheTokens @@ -534,7 +541,7 @@ class ChatStore { conversationsStore.updateMessageAtIndex(idx, { toolCalls: streamedToolCallContent }); }, onModel: (modelName: string) => recordModel(modelName), - onTimings: (timings: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => { + onTimings: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => { const tokensPerSecond = timings?.predicted_ms && timings?.predicted_n ? (timings.predicted_n / timings.predicted_ms) * 1000 @@ -1032,7 +1039,7 @@ class ChatStore { }); }, - onTimings: (timings: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => { + onTimings: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => { const tokensPerSecond = timings?.predicted_ms && timings?.predicted_n ? (timings.predicted_n / timings.predicted_ms) * 1000 diff --git a/tools/server/webui/src/lib/types/api.d.ts b/tools/server/webui/src/lib/types/api.d.ts index e5fde24c7..c2ecc0282 100644 --- a/tools/server/webui/src/lib/types/api.d.ts +++ b/tools/server/webui/src/lib/types/api.d.ts @@ -186,6 +186,7 @@ export interface ApiChatCompletionRequest { }>; stream?: boolean; model?: string; + return_progress?: boolean; // Reasoning parameters reasoning_format?: string; // Generation parameters @@ -341,6 +342,7 @@ export interface ApiProcessingState { tokensPerSecond?: number; // Progress information from prompt_progress progressPercent?: number; + promptProgress?: ChatMessagePromptProgress; promptTokens?: number; promptMs?: number; cacheTokens?: number; diff --git a/tools/server/webui/src/lib/types/settings.d.ts b/tools/server/webui/src/lib/types/settings.d.ts index 40de98b70..e09f0f332 100644 --- a/tools/server/webui/src/lib/types/settings.d.ts +++ b/tools/server/webui/src/lib/types/settings.d.ts @@ -51,7 +51,7 @@ export interface SettingsChatServiceOptions { onReasoningChunk?: (chunk: string) => void; onToolCallChunk?: (chunk: string) => void; onModel?: (model: string) => void; - onTimings?: (timings: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void; + onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void; onComplete?: ( response: string, reasoningContent?: string,