diff --git a/common/chat.cpp b/common/chat.cpp index 6d854bcd2..ca8cd040f 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1662,11 +1662,12 @@ static common_chat_params common_chat_params_init_lfm2(const common_chat_templat data.thinking_start_tag = THINK_START; data.thinking_end_tag = THINK_END; - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); // Gate by reasoning format and whether the template supports auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE && tmpl.source().find(THINK_START) != std::string::npos; - auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); if (inputs.has_continuation()) { const auto & msg = inputs.continue_msg; @@ -1689,6 +1690,10 @@ static common_chat_params common_chat_params_init_lfm2(const common_chat_templat } if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { + if (has_response_format) { + auto response_format = p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)); + return generation_prompt + reasoning + response_format + end; + } return generation_prompt + reasoning + p.content(p.rest()) + end; } auto tool_calls = p.rule("tool-calls", @@ -1707,13 +1712,17 @@ static common_chat_params common_chat_params_init_lfm2(const common_chat_templat data.parser = parser.save(); if (include_grammar) { - data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; + data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); data.grammar = build_grammar([&](const common_grammar_builder & builder) { foreach_function(inputs.tools, [&](const json & tool) { const auto & function = tool.at("function"); auto schema = function.at("parameters"); builder.resolve_refs(schema); }); + if (has_response_format) { + auto schema = inputs.json_schema; + builder.resolve_refs(schema); + } parser.build_grammar(builder, data.grammar_lazy); }); diff --git a/common/speculative.cpp b/common/speculative.cpp index 1b990a024..2bfa794c7 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -843,7 +843,8 @@ struct common_speculative_impl_ngram_map_k : public common_speculative_impl { common_speculative_impl_ngram_map_k( const common_ngram_map & config, uint32_t n_seq) - : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, n_seq) + : common_speculative_impl(config.key_only ? COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K + : COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, n_seq) { for (uint32_t i = 0; i < n_seq; i++) { this->config.push_back(config); diff --git a/ggml/src/ggml-cuda/ssm-scan.cu b/ggml/src/ggml-cuda/ssm-scan.cu index 2e3f97c72..3022249c7 100644 --- a/ggml/src/ggml-cuda/ssm-scan.cu +++ b/ggml/src/ggml-cuda/ssm-scan.cu @@ -67,6 +67,7 @@ __global__ void __launch_bounds__(splitD, 1) __shared__ CubTempStorage cub_temp_storage; BlockLoad(cub_temp_storage.load_temp).Load(A_block, regA); + __syncthreads(); BlockLoad(cub_temp_storage.load_temp).Load(s0_block, regs0); #else const int stride_s0 = src0_nb2 / sizeof(float); @@ -105,6 +106,7 @@ __global__ void __launch_bounds__(splitD, 1) regs0[n] = state; } y_block[i * stride_y + threadIdx.x] = sumf; + __syncthreads(); } #ifdef USE_CUB @@ -249,9 +251,8 @@ static void ssm_scan_f32_cuda(const float * src0, const float * src1, const floa GGML_ASSERT(head_dim == 1); GGML_ASSERT(n_group == 1); const dim3 blocks(n_seq, (n_head + threads - 1) / threads, 1); - const int smem_size = (threads * (d_state + 1) * 2) * sizeof(float); if (d_state == 16) { - const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks, threads, smem_size, stream); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks, threads, 0, stream); switch (n_tok) { case 1: diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index a37483aa6..fd2f865a9 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1873,9 +1873,9 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const { res->t_inp_embd = cur; // For Granite architecture - // NOTE: Only apply scale to token inputs. Raw embeddings are assumed to be - // multimodal inputs that should not be scaled. - if (ubatch.token && hparams.f_embedding_scale != 0.0f) { + // NOTE: For deepstack models, only apply scale to token inputs (ie text-only input). + // Raw embeddings are assumed to be multimodal inputs that should not be scaled. + if (hparams.f_embedding_scale != 0.0f && (ubatch.token || hparams.n_deepstack_layers == 0)) { if (!ggml_is_contiguous(cur)) { cur = ggml_cont(ctx0, cur); } diff --git a/tools/mtmd/tests.sh b/tools/mtmd/tests.sh index 83416fb27..5da48d61b 100755 --- a/tools/mtmd/tests.sh +++ b/tools/mtmd/tests.sh @@ -91,7 +91,6 @@ add_test_vision "ggml-org/LightOnOCR-1B-1025-GGUF:Q8_0" add_test_vision "ggml-org/DeepSeek-OCR-GGUF:Q8_0" -p "Free OCR." --chat-template deepseek-ocr add_test_vision "ggml-org/dots.ocr-GGUF:Q8_0" -p "OCR" add_test_vision "ggml-org/HunyuanOCR-GGUF:Q8_0" -p "OCR" -add_test_vision "ggml-org/HunyuanVL-4B-GGUF:Q8_0" add_test_vision "ggml-org/gemma-4-E2B-it-GGUF:Q8_0" --jinja add_test_audio "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF:Q8_0" diff --git a/tools/ui/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigation.svelte b/tools/ui/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigation.svelte index ddaf4d5b8..1fa7722f2 100644 --- a/tools/ui/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigation.svelte +++ b/tools/ui/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigation.svelte @@ -1,7 +1,7 @@