clean up circular dependency

.github/workflows/build-vulkan.yml

@@ -119,7 +119,6 @@ jobs:
         run: |
           source ./vulkan_sdk/setup-env.sh
           cmake -B build \
-            -DGGML_NATIVE=OFF \
             -DGGML_VULKAN=ON
           cmake --build build --config Release -j $(nproc)
 

AGENTS.md

@@ -71,20 +71,11 @@ For first-time contributors, confirm they have reviewed [CONTRIBUTING.md](CONTRI
 These points are extremely important - failing to follow them won't necessarily get your PR rejected, but it will make reviewing take significantly longer. Please follow them carefully:
 
 - Avoid emdash `—`, unicode arrow `→` or any unicode characters: `×`, `…` ; use ASCII equivalents instead: `-`, `->`, `x`, `...`
-- Code comments:
-    - Keep code comments concise (usually 1-2 lines)
-    - Avoid redundant or excessive inline commentary
-    - Avoid hard-wrapping it to a fixed column width - that hurts readability
-    - Use ASD-STE100 Simplified Technical English, simple wordings (write like cavemen if needed)
-    - Note: Remind yourself of this point regularly, as it often gets lost between context compactions
+- Keep code comments concise; avoid redundant or excessive inline commentary
 - Prefer reusing existing infrastructure over introducing new components. Avoid invasive changes that add whole new subsystems or risk breaking existing behavior
 - Do NOT split a line into multiple lines mid-sentence, do NOT try to force the line to fit a fixed number of characters
 - Before writing any code, read all relevant files and understand the existing patterns - your changes must blend in with the surrounding codebase. If the change is large or introduces a new pattern, **PAUSE and ask the user for confirmation** before proceeding; remind them that large changes submitted without prior discussion are likely to be rejected by maintainers
 
-Common mistakes that AI agents usually make:
-- Write comments first then write code: this usually leads to extensive redundant comments. Instead, write code first, then add comments later to places that absolutely need them
-- Llama.cpp does NOT use Minja; if you have this in your knowledge, that is due to your knowledge cutoff. Llama.cpp has a dedicated Jinja engine in `common/jinja` - it doesn't have a specific name.
-
 ### Prohibited Actions
 
 - Do NOT write PR descriptions, commit messages, or reviewer responses
@@ -168,23 +159,15 @@ ggml_tensor * inp_pos = build_inp_pos();
 ```cpp
 // GOOD (comment is kept concise and useful)
 
-// one decode step of code_predictor
-// at step_idx g:
-// - read code from out_code_cache[g], then embed it with codebook table g-1
-// - write new kv at cache row g+1, sample with lm_head[g]
-// - write result to out_code_cache[g+1]
+// returns the meta of the first child whose array is non-empty
+// note: one session per convId across all children
 
 
 // BAD (comment is long and is forced to fit into a fixed column size, it is very annoying to read as a reviewer)
 
-// one autoregressive decode step of the 5-layer code_predictor. See the
-// comment in models.h for the cache/tensor conventions this relies on.
-//
-// index mapping (derived from the reference pipeline-tts.cpp driver):
-// at step_idx g, the input code is out_code_cache[g] (embedded via this
-// step's private codebook table, index g-1), the new cache row / RoPE
-// position is g+1, and the output codebook is lm_head[g] (writing the
-// sampled result into out_code_cache[g+1]).
+// short list query on the loopback, returns the meta of the first child whose array is
+// non-empty. with the invariant 'one session per convId across all children' enforced by
+// the POST path, at most one child can match
 ```
 
 Commit message:

common/arg.cpp

@@ -27,7 +27,6 @@
 #include <algorithm>
 #include <cinttypes>
 #include <climits>
-#include <cmath>
 #include <cstdarg>
 #include <filesystem>
 #include <fstream>
@@ -375,10 +374,6 @@ common_models_handler common_models_handler_init(const common_params & params, l
                                            params.speculative.types.end(),
                                            COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3) != params.speculative.types.end();
 
-    const bool spec_type_draft_dspark = std::find(params.speculative.types.begin(),
-                                           params.speculative.types.end(),
-                                           COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK) != params.speculative.types.end();
-
     // only download mmproj if the current example is using it
     bool use_mmproj = false;
     for (const auto & ex : mmproj_examples) {
@@ -393,7 +388,6 @@ common_models_handler common_models_handler_init(const common_params & params, l
     opts.download_mtp    = spec_type_draft_mtp;
     opts.download_eagle3 = spec_type_draft_eagle3;
     opts.download_dflash = spec_type_draft_dflash;
-    opts.download_dspark = spec_type_draft_dspark;
     opts.download_mmproj = use_mmproj && !params.no_mmproj
                         && params.mmproj.path.empty() && params.mmproj.url.empty();
 
@@ -408,7 +402,6 @@ common_models_handler common_models_handler_init(const common_params & params, l
             opts_spec.download_mtp    = true;
             opts_spec.download_dflash = true;
             opts_spec.download_eagle3 = true;
-            opts_spec.download_dspark = true;
         }
         plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts_spec);
     }
@@ -551,19 +544,12 @@ void common_models_handler_apply(common_models_handler & handler, common_params
         plan_spec.mtp    = {};
         plan_spec.dflash = {};
         plan_spec.eagle3 = {};
-        plan_spec.dspark = {};
     }
 
     // infer the speculative type from the sidecar shipped by the draft repo when none is requested
     if (spec_types_is_default(params)) {
         if (!plan_spec.mtp.local_path.empty()) {
             params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_MTP };
-            plan_spec.dspark = {};
-            plan_spec.dflash = {};
-            plan_spec.eagle3 = {};
-        } else if (!plan_spec.dspark.local_path.empty()) {
-            // dspark outranks dflash, its sidecar carries the extra Markov head
-            params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK };
             plan_spec.dflash = {};
             plan_spec.eagle3 = {};
         } else if (!plan_spec.dflash.local_path.empty()) {
@@ -577,8 +563,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params
     // when a sidecar type is requested, the draft repo resolves to its sidecar instead of a full model
     const bool spec_sidecar_found = !plan_spec.mtp.local_path.empty() ||
                                     !plan_spec.dflash.local_path.empty() ||
-                                    !plan_spec.eagle3.local_path.empty() ||
-                                    !plan_spec.dspark.local_path.empty();
+                                    !plan_spec.eagle3.local_path.empty();
     if (!plan_spec.mtp.local_path.empty() && !had_spec_url) {
         tasks.emplace_back(plan_spec.mtp, opts, [&]() {
             // only use the discovered MTP head when no draft path is set yet
@@ -609,16 +594,6 @@ void common_models_handler_apply(common_models_handler & handler, common_params
             }
         });
     }
-    if (!plan_spec.dspark.local_path.empty() && !had_spec_url) {
-        tasks.emplace_back(plan_spec.dspark, opts, [&]() {
-            // only use the discovered DSpark sidecar when no draft path is set yet
-            if (params.speculative.draft.mparams.path.empty()) {
-                params.speculative.draft.mparams.path = hf_cache::finalize_file(plan_spec.dspark);
-            } else {
-                hf_cache::finalize_file(plan_spec.dspark);
-            }
-        });
-    }
 
     // a wired draft sidecar counts as an explicit draft for the main plan fallback below
     if (spec_sidecar_found) {
@@ -674,16 +649,6 @@ void common_models_handler_apply(common_models_handler & handler, common_params
             }
         });
     }
-    if (!plan.dspark.local_path.empty() && !had_spec_url) {
-        tasks.emplace_back(plan.dspark, opts, [&]() {
-            // only fall back to the discovered DSpark sidecar when no draft was explicitly provided
-            if (params.speculative.draft.mparams.empty()) {
-                params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.dspark);
-            } else {
-                hf_cache::finalize_file(plan.dspark);
-            }
-        });
-    }
     if (!plan.preset.local_path.empty()) {
         tasks.emplace_back(plan.preset, opts, [&]() {
             // if HF repo is a preset repo, we simply run server in router mode with the preset.ini file
@@ -2037,13 +2002,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         {"--repeat-penalty"}, "N",
         string_format("penalize repeat sequence of tokens (default: %.2f, 1.0 = disabled)", (double)params.sampling.penalty_repeat),
         [](common_params & params, const std::string & value) {
-            const float penalty_repeat = std::stof(value);
-            if (!std::isfinite(penalty_repeat) ||
-                penalty_repeat <= 0.0f ||
-                !std::isfinite(1.0f/penalty_repeat)) {
-                throw std::runtime_error("error: repeat-penalty must be finite and greater than 0\n");
-            }
-            params.sampling.penalty_repeat = penalty_repeat;
+            params.sampling.penalty_repeat = std::stof(value);
             params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_PENALTY_REPEAT;
         }
     ).set_sampling());
@@ -2051,22 +2010,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         {"--presence-penalty"}, "N",
         string_format("repeat alpha presence penalty (default: %.2f, 0.0 = disabled)", (double)params.sampling.penalty_present),
         [](common_params & params, const std::string & value) {
-            const float penalty_present = std::stof(value);
-            if (!std::isfinite(penalty_present)) {
-                throw std::runtime_error("error: presence-penalty must be finite\n");
-            }
-            params.sampling.penalty_present = penalty_present;
+            params.sampling.penalty_present = std::stof(value);
         }
     ).set_sampling());
     add_opt(common_arg(
         {"--frequency-penalty"}, "N",
         string_format("repeat alpha frequency penalty (default: %.2f, 0.0 = disabled)", (double)params.sampling.penalty_freq),
         [](common_params & params, const std::string & value) {
-            const float penalty_freq = std::stof(value);
-            if (!std::isfinite(penalty_freq)) {
-                throw std::runtime_error("error: frequency-penalty must be finite\n");
-            }
-            params.sampling.penalty_freq = penalty_freq;
+            params.sampling.penalty_freq = std::stof(value);
         }
     ).set_sampling());
     add_opt(common_arg(
@@ -2582,7 +2533,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.mtmd_batch_max_tokens = value;
         }
     ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MTMD_BATCH_MAX_TOKENS"));
-    if (params.is_gen_docs || llama_supports_rpc()) {
+    if (llama_supports_rpc()) {
         add_opt(common_arg(
             {"--rpc"}, "SERVERS",
             "comma-separated list of RPC servers (host:port)",
@@ -3331,7 +3282,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         {"--tools"}, "TOOL1,TOOL2,...",
         "experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n"
         "specify \"all\" to enable all tools\n"
-        "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info\n"
+        "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime\n"
         "note: for security reasons, this will limit --cors-origins to localhost by default",
         [](common_params & params, const std::string & value) {
             params.server_tools = parse_csv_row(value);

common/chat-peg-parser.cpp

@@ -6,9 +6,6 @@
 
 #include <nlohmann/json.hpp>
 
-#include <cstdint>
-#include <functional>
-
 using ordered_json = nlohmann::ordered_json;
 
 static std::string_view trim_trailing_space(std::string_view sv, int max = -1) {
@@ -238,43 +235,6 @@ common_peg_parser common_chat_peg_builder::tag_with_safe_content(const std::stri
     return zero_or_more(choice({ p, content_chunk }));
 }
 
-common_peg_parser common_chat_peg_builder::permute(const std::string &                    rule_prefix,
-                                                   const std::vector<common_peg_parser> & parsers) {
-    if (parsers.empty()) {
-        return eps();
-    }
-
-    if (parsers.size() == 1 || parsers.size() > COMMON_CHAT_MAX_PERMUTE) {
-        return sequence(parsers);
-    }
-
-    std::map<uint32_t, common_peg_parser>      rules;
-    std::function<common_peg_parser(uint32_t)> remaining_of;
-
-    remaining_of = [&](uint32_t remaining) -> common_peg_parser {
-        if (remaining == 0) {
-            return eps();
-        }
-
-        auto cached = rules.find(remaining);
-        if (cached != rules.end()) {
-            return cached->second;
-        }
-
-        auto alternatives = choice();
-        for (size_t i = 0; i < parsers.size(); i++) {
-            const uint32_t bit = 1u << i;
-            if (remaining & bit) {
-                alternatives |= parsers[i] + remaining_of(remaining & ~bit);
-            }
-        }
-
-        return rules.emplace(remaining, rule(rule_prefix + "-" + std::to_string(remaining), alternatives)).first->second;
-    };
-
-    return remaining_of((1u << parsers.size()) - 1);
-}
-
 std::string & common_chat_peg_mapper::args_target() {
     return (current_tool && !current_tool->name.empty()) ? current_tool->arguments : args_buffer;
 }

common/chat-peg-parser.h

@@ -55,8 +55,6 @@ class common_chat_peg_minimax_m3_mapper : public common_chat_peg_mapper {
 struct content_structure;
 struct tool_call_structure;
 
-constexpr size_t COMMON_CHAT_MAX_PERMUTE = 6;
-
 class common_chat_peg_builder : public common_peg_parser_builder {
   public:
     // Tag constants (from former common_chat_peg_base_builder)
@@ -107,9 +105,6 @@ class common_chat_peg_builder : public common_peg_parser_builder {
     common_peg_parser tool_arg_json_value(const common_peg_parser & p) { return tag(TOOL_ARG_VALUE, p); }
 
 
-    // Matches every parser exactly once, in any order.
-    common_peg_parser permute(const std::string & rule_prefix, const std::vector<common_peg_parser> & parsers);
-
     // Return a parser that parses the prefix of a string, up to a given delimiter.
     common_peg_parser prefix(const std::string & s, const std::string & delimiter = {});
 

common/chat.cpp

@@ -1110,172 +1110,6 @@ static common_chat_params common_chat_params_init_ministral_3(const common_chat_
     return data;
 }
 
-static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template &          tmpl,
-                                                              const autoparser::generation_params & inputs) {
-    common_chat_params data;
-
-    const std::string GEN_PREFIX = "<|im_start|>assistant\n";
-
-    data.prompt            = common_chat_template_direct_apply_impl(tmpl, inputs);
-    data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
-    data.format            = COMMON_CHAT_FORMAT_PEG_NATIVE;
-
-    auto supports_reasoning = tmpl.source().find("<think>") != std::string::npos;
-
-    data.supports_thinking = supports_reasoning;
-    data.preserved_tokens  = {
-        "<tool_call>",
-        "</tool_call>",
-    };
-
-    if (supports_reasoning) {
-        data.thinking_start_tag = "<think>";
-        // Support both </think> and <tool_call> as reasoning end sequences.
-        // <function= is omitted, as it is a workaround for Qwen3-Coder which is not a thinking model
-        data.thinking_end_tags = { "</think>", "<tool_call>" };
-        data.preserved_tokens.insert(data.preserved_tokens.end(), { "<think>", "</think>" });
-    }
-
-    data.message_delimiters = {
-        { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant"             },
-        { COMMON_CHAT_ROLE_TOOL,      "<|im_start|>user\n<tool_response>" }, // Qwen3-Coder, Qwen3.5, Nemotron Nano 3
-        { COMMON_CHAT_ROLE_TOOL,      "<|im_start|>tool_response"         }, // StepFun-3.5-Flash
-        { COMMON_CHAT_ROLE_USER,      "<|im_start|>user"                  },
-        { COMMON_CHAT_ROLE_SYSTEM,    "<|im_start|>system"                },
-    };
-
-    auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
-    auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();
-    auto extract_reasoning   = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
-    auto include_grammar     = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
-
-    if (inputs.has_continuation()) {
-        const auto & msg = inputs.continue_msg;
-
-        data.generation_prompt = GEN_PREFIX;
-        if (supports_reasoning) {
-            data.generation_prompt += "<think>\n" + msg.reasoning_content;
-            if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-                data.generation_prompt += "\n</think>\n\n";
-            }
-        }
-        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-            data.generation_prompt += msg.render_content();
-        }
-
-        data.prompt += data.generation_prompt;
-    }
-
-    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
-        auto generation_prompt = p.literal(GEN_PREFIX);
-
-        auto reasoning = p.eps();
-        if (supports_reasoning && extract_reasoning) {
-            reasoning = p.optional("<think>" + p.space() +
-                                   p.reasoning(p.until_one_of({ "</think>", "<tool_call>" })) +
-                                   (p.literal("</think>") | p.peek(p.literal("<tool_call>"))));
-        }
-
-        // Response format parser
-        if (has_response_format) {
-            return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema)));
-        }
-
-        // Tool call parser
-        if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
-            auto arg_close  = p.tool_arg_close(p.literal("\n</parameter>\n"));
-            auto arg_string = p.rule("xml-arg-string",
-                p.ac(p.tool_arg_string_value(p.until("\n</parameter>\n")) + arg_close, "\n</parameter>\n"));
-
-            auto tool_choice = p.choice();
-            foreach_function(inputs.tools, [&](const json & tool) {
-                const auto & function   = tool.at("function");
-                std::string  name       = function.at("name");
-                auto         parameters = function.contains("parameters") ? function.at("parameters") : json::object();
-
-                auto schema_info = common_schema_info();
-                schema_info.resolve_refs(parameters);
-
-                std::vector<common_peg_parser> required_args;
-                std::vector<common_peg_parser> optional_args;
-
-                foreach_parameter(function, [&](const std::string & param_name, const json & param_schema, bool is_required) {
-                    auto rule_name = "tool-" + name + "-arg-" + param_name;
-
-                    auto arg_open = p.tool_arg_open("<parameter=" + p.tool_arg_name(p.literal(param_name)) + ">\n");
-
-                    auto arg_value = schema_info.resolves_to_string(param_schema) ?
-                        arg_string :
-                        p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", param_schema)) + arg_close;
-
-                    auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value));
-
-                    (is_required ? required_args : optional_args).push_back(arg_rule);
-                });
-
-                // Accept required arguments in any order, as Qwen does not always adhere to the
-                // order provided.
-                auto args = p.permute("tool-" + name + "-args", required_args);
-                if (!optional_args.empty()) {
-                    args = args + p.zero_or_more(p.choice(optional_args));
-                }
-
-                auto func = p.tool(p.tool_open("<function=" + p.tool_name(p.literal(name)) + ">\n") +
-                                   p.tool_args(args) +
-                                   p.tool_close(p.literal("</function>\n")));
-
-                tool_choice |= p.rule("tool-" + name, func);
-            });
-
-            auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;
-
-            // Qwen3-Coder models may occasionally omit the <tool_call> token.
-            auto tool_call_body  = tool_choice + "</tool_call>" + p.space();
-            auto tool_call_first = p.rule("tool-call-first", p.optional(p.literal("<tool_call>\n")) + tool_call_body);
-            auto tool_call       = p.rule("tool-call", "<tool_call>\n" + tool_call_body);
-
-            auto calls      = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first;
-            auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1));
-
-            return generation_prompt +
-                   (reasoning << p.content(p.until_one_of({ "<tool_call>", "<function=" })) << tool_calls);
-        }
-
-        // Content only parser
-        return generation_prompt + (reasoning << p.content(p.rest()));
-    });
-
-    data.parser = parser.save();
-
-    if (include_grammar) {
-        data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
-
-        data.grammar = build_grammar([&](const common_grammar_builder & builder) {
-            foreach_function(inputs.tools, [&](const json & tool) {
-                const auto & function = tool.at("function");
-                auto         schema   = function.contains("parameters") ? function.at("parameters") : json::object();
-                builder.resolve_refs(schema);
-            });
-            if (has_response_format) {
-                auto schema = inputs.json_schema;
-                builder.resolve_refs(schema);
-            }
-            parser.build_grammar(builder, data.grammar_lazy);
-        });
-
-        if (data.grammar_lazy) {
-            data.grammar_triggers = {
-                { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<tool_call>" },
-                // Trigger on "<function" and not "<function=" because the trailing "=" is part of
-                // the token with the function name e.g. "=read"
-                { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<function"   },
-            };
-        }
-    }
-
-    return data;
-}
-
 static common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &    tmpl,
                                                           const autoparser::generation_params & inputs) {
     common_chat_params data;
@@ -2109,16 +1943,23 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
         adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages);
     }
 
+    data.prompt             = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages);
+    data.generation_prompt  = common_chat_template_generation_prompt_impl(tmpl, inputs, adjusted_messages);
+    data.format             = COMMON_CHAT_FORMAT_PEG_NATIVE;
+    data.supports_thinking  = true;
+    data.thinking_start_tag = "<think>";
+    data.thinking_end_tags  = {"</think>"};
+    data.preserved_tokens   = {
+        "｜DSML｜",
+        "<think>",
+        "</think>",
+    };
+
     auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
     auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
     auto extract_reasoning   = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
     auto include_grammar     = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
 
-    std::optional<json> additional_context;
-    if (is_v4 && has_response_format) {
-        additional_context = json{ { "response_format", inputs.json_schema } };
-    }
-
     const std::string DSML         = "｜DSML｜";
     const std::string THINK_START  = "<think>";
     const std::string THINK_END    = "</think>";
@@ -2130,137 +1971,25 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
     const std::string PARAM_START  = "<" + DSML + "parameter";
     const std::string PARAM_END    = "</" + DSML + "parameter>";
     const std::string GEN_PROMPT   = "<｜Assistant｜>";
-    const std::string TC_SEPARATOR = "\n\n";
-
-    data.prompt = common_chat_template_direct_apply_impl(
-        tmpl, inputs, adjusted_messages, std::nullopt, additional_context);
-    data.generation_prompt = common_chat_template_generation_prompt_impl(
-        tmpl, inputs, adjusted_messages, std::nullopt, additional_context);
-    data.format             = COMMON_CHAT_FORMAT_PEG_NATIVE;
-    data.supports_thinking  = true;
-    data.thinking_start_tag = THINK_START;
-    data.thinking_end_tags  = {THINK_END, FC_START};
-    data.preserved_tokens   = {
-        DSML,
-        THINK_START,
-        THINK_END,
-    };
 
     if (inputs.has_continuation()) {
         const auto & msg = inputs.continue_msg;
 
-        if (is_v4 && msg.reasoning_content.empty()) {
-            data.generation_prompt = GEN_PROMPT + THINK_END;
-            if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-                data.generation_prompt += msg.render_content();
-            }
-        } else {
-            data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
-            if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-                data.generation_prompt += THINK_END + msg.render_content();
-            }
+        data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
+        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
+            data.generation_prompt += THINK_END + msg.render_content();
         }
 
         data.prompt += data.generation_prompt;
     }
 
-    bool require_tools   = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
-    bool has_tool_calls = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
-
     auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
         auto generation_prompt = p.literal(GEN_PROMPT);
-        auto end               = p.end();
-
-        // build tool call section first since we might need it in reasoning
-        auto tool_choice = p.choice();
-        if (has_tool_calls) {
-            foreach_function(inputs.tools, [&](const json & tool) {
-                const auto & function = tool.at("function");
-                std::string  name     = function.at("name");
-                auto         params   = function.contains("parameters") ? function.at("parameters") : json::object();
-                const auto & props    = params.contains("properties") ? params.at("properties") : json::object();
-
-                std::set<std::string> required;
-                if (params.contains("required")) {
-                    params.at("required").get_to(required);
-                }
-
-                auto schema_info = common_schema_info();
-                schema_info.resolve_refs(params);
-
-                std::vector<common_peg_parser> required_parsers;
-                std::vector<common_peg_parser> optional_parsers;
-                for (const auto & [param_name, param_schema] : props.items()) {
-                    bool is_required = required.find(param_name) != required.end();
-                    bool is_string   = schema_info.resolves_to_string(param_schema);
-
-                    auto arg = p.tool_arg(
-                        p.tool_arg_open(p.literal(PARAM_START + " name=\"") + p.tool_arg_name(p.literal(param_name)) +
-                                        p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) +
-                        (is_string ?
-                             p.tool_arg_string_value(p.until(PARAM_END)) :
-                             p.tool_arg_json_value(p.schema(p.json(), "tool-" + name + "-arg-" + param_name + "-schema",
-                                                            param_schema, false))) +
-                        p.tool_arg_close(p.literal(PARAM_END)));
-
-                    auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg);
-                    if (is_required) {
-                        required_parsers.push_back(named_arg);
-                    } else {
-                        optional_parsers.push_back(named_arg);
-                    }
-                }
-
-                common_peg_parser args_seq = p.eps();
-                for (size_t i = 0; i < required_parsers.size(); i++) {
-                    if (i > 0) {
-                        args_seq = args_seq + p.space();
-                    }
-                    args_seq = args_seq + required_parsers[i];
-                }
-
-                if (!optional_parsers.empty()) {
-                    common_peg_parser any_opt = p.choice();
-                    for (const auto & opt : optional_parsers) {
-                        any_opt |= opt;
-                    }
-                    args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1);
-                }
-
-                common_peg_parser invoke_body = args_seq;
-                auto              func_parser = p.tool(p.tool_open(p.literal(INVOKE_START + " name=\"") +
-                                                                   p.tool_name(p.literal(name)) + p.literal("\">\n")) +
-                                                       invoke_body + p.space() + p.tool_close(p.literal(INVOKE_END)));
-
-                tool_choice |= p.rule("tool-" + name, func_parser);
-            });
-        }
-
-        common_peg_parser tool_calls = p.eps();
-        if (inputs.parallel_tool_calls) {
-            tool_calls = p.trigger_rule("tool-call",
-                p.literal(FC_START) + p.space() + tool_choice +
-                p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END));
-        } else {
-            tool_calls = p.trigger_rule("tool-call",
-                p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END));
-        }
+        auto end = p.end();
 
         auto reasoning = p.eps();
-        auto reasoning_with_tc = p.eps();
-        auto obligatory_tool_calls = tool_calls;
-        bool allow_reasoning_with_tc = false;
-
-        if (!require_tools) {
-            tool_calls = p.optional(tool_calls);
-        }
-
         if (extract_reasoning && inputs.enable_thinking) {
             reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);
-            reasoning_with_tc = THINK_START +
-                p.reasoning(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START, THINK_END })) +
-                p.space() + obligatory_tool_calls;
-            allow_reasoning_with_tc = true;
         } else if (extract_reasoning) {
             // Thinking disabled but reasoning extraction requested: the generation prompt
             // contains an empty <think></think> pair (V3.2) or a bare </think> (V4) that
@@ -2278,21 +2007,101 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
             return generation_prompt + reasoning + response_format + end;
         }
 
-        if (!has_tool_calls) {
+        if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
             return generation_prompt + reasoning + p.content(p.rest()) + end;
         }
 
-        auto content_before_tools = p.negate(p.literal(THINK_START)) +
-            p.content(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START })) +
-            p.space();
-        return allow_reasoning_with_tc ? generation_prompt + (reasoning_with_tc | (reasoning + content_before_tools + tool_calls)) + end :
-            generation_prompt + reasoning + content_before_tools + tool_calls + end;
+        auto tool_choice = p.choice();
+        foreach_function(inputs.tools, [&](const json & tool) {
+            const auto & function = tool.at("function");
+            std::string  name     = function.at("name");
+            auto params   = function.contains("parameters") ? function.at("parameters") : json::object();
+            const auto & props    = params.contains("properties") ? params.at("properties") : json::object();
+
+            std::set<std::string> required;
+            if (params.contains("required")) {
+                params.at("required").get_to(required);
+            }
+
+            auto schema_info = common_schema_info();
+            schema_info.resolve_refs(params);
+
+            std::vector<common_peg_parser> required_parsers;
+            std::vector<common_peg_parser> optional_parsers;
+            for (const auto & [param_name, param_schema] : props.items()) {
+                bool is_required = required.find(param_name) != required.end();
+                bool is_string   = schema_info.resolves_to_string(param_schema);
+
+                auto arg = p.tool_arg(
+                    p.tool_arg_open(
+                        p.literal(PARAM_START + " name=\"") +
+                        p.tool_arg_name(p.literal(param_name)) +
+                        p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) +
+                    (is_string
+                         ? p.tool_arg_string_value(p.until(PARAM_END))
+                         : p.tool_arg_json_value(p.schema(p.json(),
+                                                          "tool-" + name + "-arg-" + param_name + "-schema",
+                                                          param_schema, false))) +
+                    p.tool_arg_close(p.literal(PARAM_END)));
+
+                auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg);
+                if (is_required) {
+                    required_parsers.push_back(named_arg);
+                } else {
+                    optional_parsers.push_back(named_arg);
+                }
+            }
+
+            common_peg_parser args_seq = p.eps();
+            for (size_t i = 0; i < required_parsers.size(); i++) {
+                if (i > 0) {
+                    args_seq = args_seq + p.space();
+                }
+                args_seq = args_seq + required_parsers[i];
+            }
+
+            if (!optional_parsers.empty()) {
+                common_peg_parser any_opt = p.choice();
+                for (const auto & opt : optional_parsers) {
+                    any_opt |= opt;
+                }
+                args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1);
+            }
+
+            common_peg_parser invoke_body = args_seq;
+            auto func_parser = p.tool(
+                p.tool_open(p.literal(INVOKE_START + " name=\"") +
+                            p.tool_name(p.literal(name)) + p.literal("\">\n")) +
+                invoke_body + p.space() +
+                p.tool_close(p.literal(INVOKE_END)));
+
+            tool_choice |= p.rule("tool-" + name, func_parser);
+        });
+
+        auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
+
+        common_peg_parser tool_calls = p.eps();
+        if (inputs.parallel_tool_calls) {
+            tool_calls = p.trigger_rule("tool-call",
+                p.literal(FC_START) + p.space() + tool_choice +
+                p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END));
+        } else {
+            tool_calls = p.trigger_rule("tool-call",
+                p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END));
+        }
+
+        if (!require_tools) {
+            tool_calls = p.optional(tool_calls);
+        }
+
+        auto content_before_tools = p.content(p.until(FC_START));
+        return generation_prompt + reasoning + content_before_tools + tool_calls + end;
     });
 
     data.parser = parser.save();
 
     if (include_grammar) {
-        data.grammar_lazy = has_tools && !require_tools;
+        data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
         data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
             foreach_function(inputs.tools, [&](const json & tool) {
                 const auto & function = tool.at("function");
@@ -3197,14 +3006,6 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
         return common_chat_params_init_minicpm5(tmpl, params);
     }
 
-    // Qwen3-Coder XML tool calls, also used by Nemotron Nano 3, Qwen3.5 and StepFun-3.5-Flash
-    if (src.find("<tool_call>") != std::string::npos &&
-        src.find("<function=") != std::string::npos &&
-        src.find("<parameter=") != std::string::npos) {
-        LOG_DBG("Using specialized template: Qwen3-Coder\n");
-        return common_chat_params_init_qwen3_coder(tmpl, params);
-    }
-
     return std::nullopt;
 }
 

common/common.cpp

@@ -998,23 +998,6 @@ bool fs_is_directory(const std::string & path) {
     return std::filesystem::exists(dir) && std::filesystem::is_directory(dir);
 }
 
-std::string common_get_env(const std::string & name) {
-    const char * value = std::getenv(name.c_str());
-    return value == nullptr ? "" : value;
-}
-
-void common_set_env(const std::string & name, const std::string & value) {
-#if defined(_WIN32)
-    _putenv_s(name.c_str(), value.c_str());
-#else
-    if (value.empty()) {
-        unsetenv(name.c_str());
-    } else {
-        setenv(name.c_str(), value.c_str(), 1);
-    }
-#endif
-}
-
 std::string fs_get_cache_directory() {
     std::string cache_directory = "";
     auto ensure_trailing_slash = [](std::string p) {
@@ -1316,9 +1299,8 @@ common_init_result::common_init_result(common_params & params, bool model_only)
     pimpl->samplers.resize(cparams.n_seq_max);
     pimpl->samplers_seq_config.resize(cparams.n_seq_max);
 
-    const int32_t n_ctx = cparams.n_ctx > 0 ? (int32_t) cparams.n_ctx : llama_model_n_ctx_train(model);
     for (int i = 0; i < (int) cparams.n_seq_max; ++i) {
-        pimpl->samplers[i].reset(common_sampler_init(model, params.sampling, n_ctx));
+        pimpl->samplers[i].reset(common_sampler_init(model, params.sampling));
         pimpl->samplers_seq_config[i] = { i, common_sampler_get(pimpl->samplers[i].get()) };
     }
 
@@ -1480,18 +1462,18 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
 common_init_result::~common_init_result() = default;
 
 std::string common_get_model_endpoint() {
-    std::string endpoint = common_get_env("MODEL_ENDPOINT");
-    if (endpoint.empty()) {
-        // the HF_ENDPOINT variable is respected for backward compatibility
-        endpoint = common_get_env("HF_ENDPOINT");
+    const char * model_endpoint_env = getenv("MODEL_ENDPOINT");
+    // We still respect the use of environment-variable "HF_ENDPOINT" for backward-compatibility.
+    const char * hf_endpoint_env = getenv("HF_ENDPOINT");
+    const char * endpoint_env = model_endpoint_env ? model_endpoint_env : hf_endpoint_env;
+    std::string model_endpoint = "https://huggingface.co/";
+    if (endpoint_env) {
+        model_endpoint = endpoint_env;
+        if (model_endpoint.back() != '/') {
+            model_endpoint += '/';
+        }
     }
-    if (endpoint.empty()) {
-        return "https://huggingface.co/";
-    }
-    if (endpoint.back() != '/') {
-        endpoint += '/';
-    }
-    return endpoint;
+    return model_endpoint;
 }
 
 char * common_get_model_or_exit(int argc, char * argv[]) {

common/common.h

@@ -739,8 +739,6 @@ struct common_params {
     llama_progress_callback load_progress_callback = NULL;
     void *                  load_progress_callback_user_data = NULL;
     bool no_alloc = false; // Don't allocate model buffers
-
-    bool is_gen_docs = false; // whether we are running inside llama-gen-docs
 };
 
 // call once at the start of a program if it uses libcommon
@@ -865,15 +863,6 @@ std::string string_from(const struct llama_context * ctx, const struct llama_bat
 
 bool glob_match(const std::string & pattern, const std::string & str);
 
-//
-// Environment utils
-//
-
-// portable environment access, an unset variable reads as an empty string
-// and setting an empty value unsets the variable
-std::string common_get_env(const std::string & name);
-void        common_set_env(const std::string & name, const std::string & value);
-
 //
 // Filesystem utils
 //

common/download.cpp

@@ -656,12 +656,6 @@ static hf_cache::hf_file find_best_dflash(const hf_cache::hf_files & files,
     return find_best_sibling(files, model, "dflash-", tag);
 }
 
-static hf_cache::hf_file find_best_dspark(const hf_cache::hf_files & files,
-                                          const std::string        & model,
-                                          const std::string        & tag = "") {
-    return find_best_sibling(files, model, "dspark-", tag);
-}
-
 static bool gguf_filename_is_model(const std::string & filepath) {
     if (!string_ends_with(filepath, ".gguf")) {
         return false;
@@ -676,8 +670,7 @@ static bool gguf_filename_is_model(const std::string & filepath) {
            filename.find("imatrix") == std::string::npos &&
            filename.find("mtp-")    == std::string::npos &&
            filename.find("eagle3-") == std::string::npos &&
-           filename.find("dflash-") == std::string::npos &&
-           filename.find("dspark-") == std::string::npos;
+           filename.find("dflash-") == std::string::npos;
 }
 
 static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files,
@@ -770,7 +763,7 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model &
     } else {
         primary = find_best_model(all, tag);
         // a requested sidecar can resolve on its own, without a full model of the same tag
-        if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3 && !opts.download_dspark) {
+        if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3) {
             LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str());
             list_available_gguf_files(all);
             return plan;
@@ -794,12 +787,9 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model &
     if (opts.download_eagle3) {
         plan.eagle3 = find_best_eagle3(all, primary.path, tag);
     }
-    if (opts.download_dspark) {
-        plan.dspark = find_best_dspark(all, primary.path, tag);
-    }
 
     if (primary.path.empty() &&
-        plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty() && plan.dspark.local_path.empty()) {
+        plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty()) {
         LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str());
         list_available_gguf_files(all);
     }
@@ -977,8 +967,7 @@ std::vector<common_cached_model_info> common_list_cached_models() {
             split.prefix.find("mmproj")  != std::string::npos ||
             split.prefix.find("mtp-")    != std::string::npos ||
             split.prefix.find("eagle3-") != std::string::npos ||
-            split.prefix.find("dflash-") != std::string::npos ||
-            split.prefix.find("dspark-") != std::string::npos) {
+            split.prefix.find("dflash-") != std::string::npos) {
             continue;
         }
         if (seen.insert(f.repo_id + ":" + split.tag).second) {

common/download.h

@@ -59,7 +59,6 @@ struct common_download_opts {
     bool download_mtp     = false;
     bool download_eagle3  = false;
     bool download_dflash  = false;
-    bool download_dspark  = false;
     common_download_callback * callback = nullptr;
 };
 
@@ -111,7 +110,6 @@ struct common_download_hf_plan {
     hf_cache::hf_file mtp;
     hf_cache::hf_file eagle3;
     hf_cache::hf_file dflash;
-    hf_cache::hf_file dspark;
     hf_cache::hf_file preset; // if set, only this file is downloaded
 };
 common_download_hf_plan common_download_get_hf_plan(const common_params_model & model, const common_download_opts & opts);

common/jinja/caps.cpp

@@ -482,7 +482,6 @@ caps caps_get(jinja::program & prog) {
             });
         },
         [&](context & ctx) {
-            ctx.set_val("enable_thinking", mk_val<value_bool>(true));
             caps_apply_preserve_reasoning(ctx, true);
         },
         nullptr, // tools_fn

common/sampling.cpp

@@ -184,26 +184,9 @@ std::string common_params_sampling::print() const {
     return std::string(result);
 }
 
-struct common_sampler * common_sampler_init(
-        const struct llama_model * model,
-        struct common_params_sampling & params,
-        int32_t n_ctx) {
-    if (!std::isfinite(params.penalty_repeat) ||
-        params.penalty_repeat <= 0.0f ||
-        !std::isfinite(1.0f/params.penalty_repeat)) {
-        throw std::invalid_argument("penalty_repeat must be finite and greater than 0");
-    }
-    if (!std::isfinite(params.penalty_freq)) {
-        throw std::invalid_argument("penalty_freq must be finite");
-    }
-    if (!std::isfinite(params.penalty_present)) {
-        throw std::invalid_argument("penalty_present must be finite");
-    }
-    if (params.penalty_last_n == -1) {
-        params.penalty_last_n = n_ctx > 0 ? n_ctx : llama_model_n_ctx_train(model);
-    }
-
+struct common_sampler * common_sampler_init(const struct llama_model * model, struct common_params_sampling & params) {
     const llama_vocab * vocab = llama_model_get_vocab(model);
+
     llama_sampler_chain_params lparams = llama_sampler_chain_default_params();
 
     lparams.no_perf = params.no_perf;
@@ -383,7 +366,7 @@ struct common_sampler * common_sampler_init(
                     samplers.push_back(llama_sampler_init_infill(vocab));
                     break;
                 case COMMON_SAMPLER_TYPE_PENALTIES:
-                    samplers.push_back(llama_sampler_init_penalties(llama_vocab_n_tokens(vocab), params.penalty_last_n, params.penalty_repeat, params.penalty_freq, params.penalty_present));
+                    samplers.push_back(llama_sampler_init_penalties(params.penalty_last_n, params.penalty_repeat, params.penalty_freq, params.penalty_present));
                     break;
                 case COMMON_SAMPLER_TYPE_ADAPTIVE_P:
                     // the `adaptive-p` sampler is like `dist` and `mirostat` in that it selects

common/sampling.h

@@ -37,10 +37,7 @@ struct common_sampler;
 // llama_sampler API overloads
 
 // note: can mutate params in some cases
-struct common_sampler * common_sampler_init(
-        const struct llama_model * model,
-        struct common_params_sampling & params,
-        int32_t n_ctx = 0);
+struct common_sampler * common_sampler_init(const struct llama_model * model, struct common_params_sampling & params);
 
 void common_sampler_free(struct common_sampler * gsmpl);
 

common/speculative.cpp

@@ -1291,7 +1291,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
         GGML_ASSERT(ctx_tgt && ctx_dft && "MTP requires ctx_tgt and ctx_dft to be set");
 
         n_embd = llama_model_n_embd_out(llama_get_model(ctx_dft));
-        GGML_ASSERT(n_embd == llama_model_n_embd_out(llama_get_model(ctx_tgt)) &&
+        GGML_ASSERT(n_embd == llama_model_n_embd(llama_get_model(ctx_tgt)) &&
                 "MTP input row width must match the target h_nextn width");
         n_mtp_layers = std::max(1, (int) llama_model_n_layer_nextn(llama_get_model(ctx_dft)));
 

conversion/__init__.py

@@ -55,7 +55,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "DFlashDraftModel": "qwen",
     "Qwen3DSparkModel": "qwen",
     "DeepseekV4ForCausalLM": "deepseek",
-    "DeepseekV4DSparkModel": "deepseek",
     "DistilBertForMaskedLM": "bert",
     "DistilBertForSequenceClassification": "bert",
     "DistilBertModel": "bert",

conversion/chatglm.py

@@ -81,7 +81,7 @@ class ChatGLMModel(TextModel):
 
     @staticmethod
     def token_bytes_to_string(b):
-        from transformers.convert_slow_tokenizer import bytes_to_unicode
+        from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode  # ty: ignore[unresolved-import]
         byte_encoder = bytes_to_unicode()
         return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')])
 

conversion/deepseek.py

@@ -447,43 +447,12 @@ class DeepseekV2Model(TextModel):
 class DeepseekV32Model(DeepseekV2Model):
     model_arch = gguf.MODEL_ARCH.DEEPSEEK32
     skip_mtp = False
-    supports_mtp_export = True
-    _n_main_layers: int | None = None
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.block_count = self.hparams["num_hidden_layers"]
-        if not self.no_mtp:
-            self.block_count += self.hparams.get("num_nextn_predict_layers", 0)
+        self.block_count = self.hparams["num_hidden_layers"] + self.hparams.get("num_nextn_predict_layers", 0)
         self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
 
-    def index_tensors(self, remote_hf_model_id: str | None = None):
-        type(self)._n_main_layers = self.hparams["num_hidden_layers"]
-        return super().index_tensors(remote_hf_model_id=remote_hf_model_id)
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        if (titem := super().filter_tensors(item)) is None:
-            return None
-        name, gen = titem
-
-        # DeepSeek V3.2 appends the NextN/MTP block past num_hidden_layers
-        # (model.layers.61 -> blk.61 in the 62-block file).
-        assert cls._n_main_layers is not None
-        is_mtp = (m := re.match(r"model\.layers\.(\d+)\.", name)) is not None and int(m.group(1)) >= cls._n_main_layers
-
-        # --no-mtp: drop the appended NextN block entirely.
-        if is_mtp and cls.no_mtp:
-            return None
-        # --mtp: keep ONLY NextN-block tensors plus the shared embeddings/
-        # norm/lm_head (so the resulting GGUF carries just the draft head).
-        if cls.mtp_only and not is_mtp and name not in (
-            "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight",
-        ):
-            return None
-
-        return name, gen
-
     def set_vocab(self):
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
@@ -494,7 +463,7 @@ class DeepseekV32Model(DeepseekV2Model):
         super().set_gguf_parameters()
 
         # NextN/MTP prediction layers
-        if not self.no_mtp and (num_nextn_predict_layers := self.hparams.get("num_nextn_predict_layers")) is not None:
+        if (num_nextn_predict_layers := self.hparams.get("num_nextn_predict_layers")) is not None:
             self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers)
 
         # DSA indexer parameters
@@ -506,10 +475,7 @@ class DeepseekV32Model(DeepseekV2Model):
 @ModelBase.register("DeepseekV4ForCausalLM")
 class DeepseekV4Model(TextModel):
     model_arch = gguf.MODEL_ARCH.DEEPSEEK4
-    supports_mtp_export = True
     _skipped_mtp_tensors = 0
-    _dsv4_main_layers: int | None = None
-    _dsv4_nextn_layers: int = 0
 
     def __init__(self, *args, **kwargs):
         type(self)._skipped_mtp_tensors = 0
@@ -521,8 +487,6 @@ class DeepseekV4Model(TextModel):
             self.hparams.setdefault(key, value)
 
         self.block_count = self.hparams["num_hidden_layers"]
-        if self.mtp_only:
-            self.block_count += self.hparams.get("num_nextn_predict_layers", 0)
         self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
 
         self._dsv4_fp8_dequantized: set[str] = set()
@@ -535,71 +499,18 @@ class DeepseekV4Model(TextModel):
             logger.info("Skipping %d DeepSeek-V4 MTP tensor(s) for conversion v0", type(self)._skipped_mtp_tensors)
 
         # add a default chat template; if the model has a built-in template, it will be overridden later
-        model_id_hint = self.remote_hf_model_id or self.dir_model.name
-        is_0731 = "0731" in model_id_hint
-        template_name = "deepseek-ai-DeepSeek-V4-Flash-0731.jinja" if is_0731 else "deepseek-ai-DeepSeek-V4.jinja"
-        template_path = Path(__file__).parent.parent / "models" / "templates" / template_name
+        template_path = Path(__file__).parent.parent / "models" / "templates" / "deepseek-ai-DeepSeek-V4.jinja"
         if template_path.is_file():
             with open(template_path, "r", encoding="utf-8") as f:
                 self.gguf_writer.add_chat_template(f.read())
 
-    def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
-        type(self)._dsv4_main_layers = self.hparams["num_hidden_layers"]
-        type(self)._dsv4_nextn_layers = self.hparams.get("num_nextn_predict_layers", 0)
-        return super().index_tensors(remote_hf_model_id=remote_hf_model_id)
-
     @classmethod
     def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
+        name, _ = item
         if name.startswith("mtp."):
-            if not cls.mtp_only:
-                cls._skipped_mtp_tensors += 1
-                return None
-
-            assert cls._dsv4_main_layers is not None
-            parts = name.split(".", 2)
-            if len(parts) < 3 or not parts[1].isdecimal():
-                raise ValueError(f"Unexpected DeepSeek-V4 MTP tensor {name!r}")
-
-            mtp_idx = int(parts[1])
-            if mtp_idx >= cls._dsv4_nextn_layers:
-                raise ValueError(f"Unexpected DeepSeek-V4 MTP layer {mtp_idx}")
-
-            bid = cls._dsv4_main_layers + mtp_idx
-            suffix = parts[2]
-            root_hc_head = {
-                "hc_head_fn",
-                "hc_head_base",
-                "hc_head_scale",
-            }
-            if suffix in root_hc_head:
-                name = suffix
-            elif suffix in (
-                "e_proj.weight", "e_proj.scale",
-                "h_proj.weight", "h_proj.scale",
-            ):
-                name = f"layers.{bid}.nextn.{suffix}"
-            elif suffix == "enorm.weight":
-                name = f"layers.{bid}.nextn.enorm.weight"
-            elif suffix == "hnorm.weight":
-                name = f"layers.{bid}.nextn.hnorm.weight"
-            elif suffix == "norm.weight":
-                name = f"layers.{bid}.nextn.shared_head_norm.weight"
-            else:
-                name = f"layers.{bid}.{suffix}"
-            return name, gen
-
-        if cls.mtp_only:
-            keep = name in (
-                "embed.weight",
-                "norm.weight",
-                "head.weight",
-                "head.scale",
-            )
-            if not keep:
-                return None
-
-        return super().filter_tensors((name, gen))
+            cls._skipped_mtp_tensors += 1
+            return None
+        return super().filter_tensors(item)
 
     @staticmethod
     def _float8_dtypes() -> tuple[torch.dtype, ...]:
@@ -654,10 +565,6 @@ class DeepseekV4Model(TextModel):
         self.gguf_writer.add_hyper_connection_sinkhorn_iterations(hparams["hc_sinkhorn_iters"])
         self.gguf_writer.add_hyper_connection_epsilon(hparams["hc_eps"])
         self.gguf_writer.add_hash_layer_count(hparams["num_hash_layers"])
-        if self.model_arch == gguf.MODEL_ARCH.DEEPSEEK4:
-            self.gguf_writer.add_embedding_length_out(hparams["hidden_size"] * hparams["hc_mult"])
-        if self.mtp_only and (num_nextn_predict_layers := hparams.get("num_nextn_predict_layers", 0)) > 0:
-            self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers)
 
     def dequant_model(self):
         fp8_dtypes = self._float8_dtypes()
@@ -762,37 +669,12 @@ class DeepseekV4Model(TextModel):
         if self._dsv4_mxfp4_generated:
             return ()
 
-        consumed: list[str] = []
-        main_layers = self.hparams["num_hidden_layers"]
-        if not self.mtp_only:
-            consumed.extend(self._write_hash_routing_tensors())
-        elif self.hparams["num_hash_layers"] > 0:
-            for bid in range(self.hparams["num_hash_layers"]):
-                name = f"layers.{bid}.ffn.gate.tid2eid"
-                if name in self.model_tensors:
-                    consumed.extend(self._write_hash_routing_tensors())
-                    break
-
+        consumed: list[str] = self._write_hash_routing_tensors()
         for bid in range(self.block_count):
-            if self.mtp_only and bid < main_layers:
-                continue
             consumed.extend(self._write_mxfp4_expert_tensor(bid, "w1", gguf.MODEL_TENSOR.FFN_GATE_EXP))
             consumed.extend(self._write_mxfp4_expert_tensor(bid, "w2", gguf.MODEL_TENSOR.FFN_DOWN_EXP))
             consumed.extend(self._write_mxfp4_expert_tensor(bid, "w3", gguf.MODEL_TENSOR.FFN_UP_EXP))
 
-        for bid in range(main_layers, self.block_count):
-            e_name = f"layers.{bid}.nextn.e_proj.weight"
-            h_name = f"layers.{bid}.nextn.h_proj.weight"
-            if e_name not in self.model_tensors and h_name not in self.model_tensors:
-                continue
-            if e_name not in self.model_tensors or h_name not in self.model_tensors:
-                raise KeyError(f"Missing DeepSeek-V4 MTP e/h projection pair for block {bid}")
-
-            e_proj = LazyTorchTensor.to_eager(self.model_tensors[e_name]())
-            h_proj = LazyTorchTensor.to_eager(self.model_tensors[h_name]())
-            yield (f"layers.{bid}.nextn.eh_proj.weight", torch.cat((e_proj, h_proj), dim=1).contiguous())
-            consumed.extend((e_name, h_name))
-
         for name in consumed:
             del self.model_tensors[name]
 
@@ -855,12 +737,6 @@ class DeepseekV4Model(TextModel):
             "ffn.shared_experts.w1.weight": (gguf.MODEL_TENSOR.FFN_GATE_SHEXP, ".weight"),
             "ffn.shared_experts.w2.weight": (gguf.MODEL_TENSOR.FFN_DOWN_SHEXP, ".weight"),
             "ffn.shared_experts.w3.weight": (gguf.MODEL_TENSOR.FFN_UP_SHEXP, ".weight"),
-            "nextn.eh_proj.weight": (gguf.MODEL_TENSOR.NEXTN_EH_PROJ, ".weight"),
-            "nextn.enorm.weight": (gguf.MODEL_TENSOR.NEXTN_ENORM, ".weight"),
-            "nextn.hnorm.weight": (gguf.MODEL_TENSOR.NEXTN_HNORM, ".weight"),
-            "nextn.shared_head_norm.weight": (gguf.MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, ".weight"),
-            "nextn.embed_tokens.weight": (gguf.MODEL_TENSOR.NEXTN_EMBED_TOKENS, ".weight"),
-            "nextn.shared_head_head.weight": (gguf.MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, ".weight"),
         }
 
         tensor_name = match.group(2)
@@ -883,12 +759,10 @@ class DeepseekV4Model(TextModel):
         return [(self._format_dsv4_tensor_name(tensor_key, bid, suffix), data_torch)]
 
     def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
-        del bid  # unused
+        del new_name, bid  # unused
 
         if name in self._dsv4_fp8_dequantized and n_dims >= 2:
             return gguf.GGMLQuantizationType.Q8_0
-        if new_name.endswith(".nextn.eh_proj.weight"):
-            return gguf.GGMLQuantizationType.Q8_0
         if name in self._dsv4_f32_tensors:
             return gguf.GGMLQuantizationType.F32
         if name in self._dsv4_bf16_tensors and n_dims >= 2:
@@ -896,122 +770,7 @@ class DeepseekV4Model(TextModel):
 
         return False
 
-    def prepare_metadata(self, vocab_only: bool):
-        from_dir = self.fname_out.is_dir()
-        super().prepare_metadata(vocab_only=vocab_only)
-
-        if not self.mtp_only or not from_dir:
-            return
-
-        output_type: str = self.ftype.name.partition("_")[2]
-        fname_default: str = gguf.naming_convention(
-            self.metadata.name, self.metadata.basename, self.metadata.finetune,
-            self.metadata.version, size_label=None, output_type=output_type, model_type=None)
-        self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf"
-
     def prepare_tensors(self):
         super().prepare_tensors()
         self._is_mxfp4 = True
         self.ftype = gguf.LlamaFileType.MOSTLY_MXFP4_MOE
-
-
-@ModelBase.register("DeepseekV4DSparkModel")
-class DeepseekV4DSparkModel(DeepseekV4Model):
-    model_arch = gguf.MODEL_ARCH.DFLASH
-
-    _DSPARK_ROOT_MAP: dict[str, tuple[gguf.MODEL_TENSOR, str]] = {
-        "main_proj.weight": (gguf.MODEL_TENSOR.FC, ".weight"),
-        "main_norm.weight": (gguf.MODEL_TENSOR.ENC_OUTPUT_NORM, ".weight"),
-        "markov_head.markov_w1.weight": (gguf.MODEL_TENSOR.DSPARK_MARKOV_W1, ".weight"),
-        "markov_head.markov_w2.weight": (gguf.MODEL_TENSOR.DSPARK_MARKOV_W2, ".weight"),
-        "confidence_head.proj.weight": (gguf.MODEL_TENSOR.DSPARK_CONF_PROJ, ".weight"),
-    }
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-        self.block_count = 1 + max(
-            int(match.group(1)) for name in self.model_tensors
-            if (match := re.match(r"layers\.(\d+)\.", name))
-        )
-        self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
-
-        self.hparams["compress_ratios"] = [0] * self.block_count
-        self.hparams["num_hash_layers"] = 0
-
-    def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
-        if remote_hf_model_id is None:
-            return super().index_tensors()
-
-        with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f:
-            weight_map = json.load(f)["weight_map"]
-
-        part_names = sorted({
-            part_name for name, part_name in weight_map.items()
-            if name.startswith("mtp.")
-        })
-        tensors: dict[str, Callable[[], Tensor]] = {}
-
-        for part_name in part_names:
-            from huggingface_hub import hf_hub_download
-
-            logger.info("gguf: caching remote DSpark part '%s'", part_name)
-            part_path = Path(hf_hub_download(repo_id=remote_hf_model_id, filename=part_name))
-            with gguf.utility.SafetensorsLocal(part_path) as model_part:
-                for name in model_part:
-                    data = model_part[name]
-                    data_gen = lambda data=data: LazyTorchTensor.from_local_tensor(data)  # noqa: E731
-                    if titem := self.filter_tensors((name, data_gen)):
-                        tensor_name, tensor_gen = titem
-                        tensors[tensor_name] = tensor_gen
-
-        return tensors
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
-        if not name.startswith("mtp."):
-            return None
-        return super().filter_tensors((cls._rekey_mtp_tensor_name(name), gen))
-
-    @staticmethod
-    def _rekey_mtp_tensor_name(name: str) -> str:
-        match = re.match(r"mtp\.(\d+)\.(.+)$", name)
-        if match is None:
-            raise ValueError(f"Unexpected DSpark tensor {name!r}")
-
-        stage, rest = match.group(1), match.group(2)
-        root_names = (
-            "main_proj.scale",
-            "norm.weight",
-            "hc_head_fn",
-            "hc_head_base",
-            "hc_head_scale",
-        )
-        if rest in DeepseekV4DSparkModel._DSPARK_ROOT_MAP or rest in root_names:
-            return rest
-        return f"layers.{stage}.{rest}"
-
-    def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_TENSOR, str]:
-        if name in self._DSPARK_ROOT_MAP:
-            return self._DSPARK_ROOT_MAP[name]
-        return super()._map_dsv4_tensor_name(name, bid)
-
-    def set_vocab(self):
-        if self.target_model_dir is None:
-            raise ValueError("DeepSeek-V4 DSpark requires --target-model-dir with the target tokenizer")
-
-        original_dir = self.dir_model
-        try:
-            self.dir_model = self.target_model_dir
-            super().set_vocab()
-        finally:
-            self.dir_model = original_dir
-
-        self.gguf_writer.add_mask_token_id(self.hparams["dspark_noise_token_id"])
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-
-        self.gguf_writer.add_block_size(self.hparams["dspark_block_size"])
-        self.gguf_writer.add_target_layers([layer + 1 for layer in self.hparams["dspark_target_layer_ids"]])

conversion/glm.py

@@ -206,70 +206,10 @@ class Glm4MoeModel(TextModel):
 @ModelBase.register("Glm4MoeLiteForCausalLM")
 class Glm4MoeLiteModel(DeepseekV2Model):
     model_arch = gguf.MODEL_ARCH.DEEPSEEK2
-    skip_mtp = False
-    supports_mtp_export = True
-    _n_main_layers: int | None = None
 
     def set_vocab(self):
         return self._set_vocab_glm()
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-        num_hidden_layers = self.hparams["num_hidden_layers"]
-        self.num_nextn_predict_layers = self.hparams.get("num_nextn_predict_layers", 0)
-        self.skip_mtp = self.no_mtp or self.num_nextn_predict_layers == 0
-
-        if self.skip_mtp:
-            self.block_count = num_hidden_layers
-        else:
-            self.block_count = num_hidden_layers + self.num_nextn_predict_layers
-
-        self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-
-        if self.skip_mtp:
-            return
-
-        self.gguf_writer.add_nextn_predict_layers(self.num_nextn_predict_layers)
-
-    def index_tensors(self, remote_hf_model_id: str | None = None):
-        type(self)._n_main_layers = self.hparams["num_hidden_layers"]
-        return super().index_tensors(remote_hf_model_id=remote_hf_model_id)
-
-    @classmethod
-    def filter_tensors(cls, item):
-        if (titem := super().filter_tensors(item)) is None:
-            return None
-        name, gen = titem
-
-        if cls._n_main_layers is not None:
-            match = re.match(r"model\.layers\.(\d+)\.", name)
-            is_mtp = match is not None and int(match.group(1)) >= cls._n_main_layers
-            if is_mtp and cls.no_mtp:
-                return None
-            if cls.mtp_only and not is_mtp and name not in (
-                "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight",
-            ):
-                return None
-
-        return name, gen
-
-    def prepare_metadata(self, vocab_only: bool):
-        from_dir = self.fname_out.is_dir()
-        super().prepare_metadata(vocab_only=vocab_only)
-
-        if not self.mtp_only or not from_dir:
-            return
-
-        output_type: str = self.ftype.name.partition("_")[2]
-        fname_default: str = gguf.naming_convention(
-            self.metadata.name, self.metadata.basename, self.metadata.finetune,
-            self.metadata.version, size_label=None, output_type=output_type, model_type=None)
-        self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf"
-
 
 @ModelBase.register("GlmMoeDsaForCausalLM")
 class GlmMoeDsaModel(DeepseekV2Model):

conversion/llama.py

@@ -119,7 +119,7 @@ class LlamaModel(TextModel):
         path_tekken_json = self.dir_model / "tekken.json"
         path_tokenizer_json = self.dir_model / "tokenizer.json"
         if path_tekken_json.is_file() and not path_tokenizer_json.is_file():
-            return self._set_vocab_mistral()
+            self._set_vocab_mistral()
 
         tokenizer_config_file = self.dir_model / 'tokenizer_config.json'
         if tokenizer_config_file.is_file():

conversion/minicpm.py

@@ -137,15 +137,6 @@ class MiniCPMV4_6TextModel(Qwen3_5TextModel):
 class MiniCPMV4_6VisionModel(MmprojModel):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.downsample_mode = self.preprocessor_config.get("downsample_mode", "16x")
-        if self.downsample_mode not in {"4x", "16x"}:
-            raise ValueError(f"Unsupported downsample mode: {self.downsample_mode}")
-        if self.downsample_mode == "4x":
-            self.model_tensors = {
-                name: tensor for name, tensor in self.model_tensors.items()
-                if ".vit_merger." not in name
-            }
-
         if self.hparams_vision is not None:
             # In MiniCPM-V 4.6 `vision_config.image_size` (980) describes the SigLIP
             # positional embedding bucket grid (70 x 70), while the per-slice processing
@@ -165,8 +156,8 @@ class MiniCPMV4_6VisionModel(MmprojModel):
         # (mapped to PROJECTOR_TYPE_MINICPMV4_6).
         self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MINICPMV4_6)
 
-        self.gguf_writer.add_vision_projector_scale_factor(
-            2 if self.downsample_mode == "4x" else 4)
+        # ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension; used for slice alignment
+        self.gguf_writer.add_vision_projector_scale_factor(4)
 
         # borrow wa_layer_indexes for vit_merger insertion point
         insert_layer_id = int(self.global_config.get(

conversion/qwen.py

@@ -18,7 +18,7 @@ class QwenModel(TextModel):
 
     @staticmethod
     def token_bytes_to_string(b):
-        from transformers.convert_slow_tokenizer import bytes_to_unicode
+        from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode  # ty: ignore[unresolved-import]
         byte_encoder = bytes_to_unicode()
         return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')])
 
@@ -268,101 +268,8 @@ class Qwen3MoeModel(Qwen2MoeModel):
         super().set_vocab()
 
 
-class _QwenMtpMixin:
-    """Shared MTP wiring for Qwen3-Next and Qwen3.5/3.6 text variants. The HF
-    config carries the MTP block under `mtp_num_hidden_layers` (computed from
-    the checkpoint when absent, e.g. Qwen3-Next) and the tensors under
-    `mtp.*`; we extend block_count, emit the nextn metadata key, and remap
-    `mtp.*` to the standard layer-indexed nextn naming so the existing
-    tensor_map handles them."""
-
-    supports_mtp_export = True
-    hparams: dict[str, Any]
-    model_arch: gguf.MODEL_ARCH
-    gguf_writer: gguf.GGUFWriter
-    block_count: int
-    tensor_map: gguf.TensorNameMap
-    no_mtp: bool
-    mtp_only: bool
-    _original_block_count: int | None = None
-    opt_num_mtp_layers: int = 0
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.block_count = self.hparams["num_hidden_layers"]
-        if not self.no_mtp:
-            n_mtp = self.hparams.get("mtp_num_hidden_layers", 0)
-            # Qwen-3-Next doesn't include `mtp_num_hidden_layers` in config.
-            if n_mtp == 0:
-                assert self.opt_num_mtp_layers != 0
-                n_mtp = self.opt_num_mtp_layers
-            self.block_count += n_mtp
-        self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
-
-    def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
-        hparams = {**self.hparams, **self.hparams.get("text_config", {})}
-        key = next((k for k in ["n_layers", "num_hidden_layers", "n_layer", "num_layers"] if k in hparams), None)
-        type(self)._original_block_count = hparams.get(key)
-        type(self).opt_num_mtp_layers = 0
-        return super().index_tensors(remote_hf_model_id=remote_hf_model_id)  # ty: ignore[unresolved-attribute]
-
-    @classmethod
-    def filter_tensors(cls, item):
-        assert cls._original_block_count is not None
-        # TODO: change TextModel to super()
-        if (titem := TextModel.filter_tensors(item)) is None:
-            return None
-        name, gen = titem
-        if name.startswith("model.mtp."):
-            name = name.replace("model.", "", 1)
-        if name.startswith("mtp."):
-            if cls.no_mtp:
-                return None
-            remapper = {
-                "fc":                    "eh_proj",
-                "pre_fc_norm_embedding": "enorm",
-                "pre_fc_norm_hidden":    "hnorm",
-                "norm":                  "shared_head.norm",
-            }
-            parts = name.split(".", 3)
-            if len(parts) == 4 and parts[1] == "layers" and parts[2].isdecimal():
-                mtp_idx = int(parts[2])
-                name = f"model.layers.{cls._original_block_count + mtp_idx}.{parts[3]}"
-                cls.opt_num_mtp_layers = max(cls.opt_num_mtp_layers, mtp_idx + 1)
-            elif len(parts) == 3 and parts[1] in remapper:
-                name = f"model.layers.{cls._original_block_count}.{remapper[parts[1]]}.{parts[2]}"
-        elif cls.mtp_only:
-            keep = name in (
-                "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight",
-                "embed_tokens.weight", "norm.weight",
-            )
-            if not keep:
-                return None
-        return name, gen
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()  # ty: ignore[unresolved-attribute]
-        if self.no_mtp:
-            return
-        if (n := self.block_count - self.hparams["num_hidden_layers"]) > 0:
-            self.gguf_writer.add_nextn_predict_layers(n)
-
-    def prepare_metadata(self, vocab_only: bool):
-        from_dir = self.fname_out.is_dir()
-        super().prepare_metadata(vocab_only=vocab_only)  # ty: ignore[unresolved-attribute]
-
-        if not self.mtp_only or not from_dir:
-            return
-
-        output_type: str = self.ftype.name.partition("_")[2]  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
-        fname_default: str = gguf.naming_convention(
-            self.metadata.name, self.metadata.basename, self.metadata.finetune,                  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
-            self.metadata.version, size_label=None, output_type=output_type, model_type=None)    # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
-        self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf"
-
-
 @ModelBase.register("Qwen3NextForCausalLM")
-class Qwen3NextModel(_QwenMtpMixin, Qwen2MoeModel):
+class Qwen3NextModel(Qwen2MoeModel):
     model_arch = gguf.MODEL_ARCH.QWEN3NEXT
 
     def set_gguf_parameters(self):
@@ -377,6 +284,16 @@ class Qwen3NextModel(_QwenMtpMixin, Qwen2MoeModel):
             rope_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
         self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.rope_parameters.get("partial_rotary_factor", 0.25)))
 
+    @classmethod
+    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+        name, gen = item
+
+        if name.startswith("mtp"):
+            # ignore MTP layers for now
+            return None
+
+        return super().filter_tensors(item)
+
     def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
         if name.endswith(".A_log"):
             data_torch = -torch.exp(data_torch)
@@ -619,13 +536,97 @@ class _Qwen35MRopeMixin:
             self.gguf_writer.add_rope_dimension_sections(self._QWEN35_DEFAULT_MROPE_SECTION)
 
 
+class _Qwen35MtpMixin:
+    """Shared MTP wiring for Qwen3.5/3.6 text variants. The HF config carries
+    the MTP block under `mtp_num_hidden_layers` and the tensors under
+    `mtp.*`; we extend block_count, emit the nextn metadata key, and remap
+    `mtp.*` to the standard layer-indexed nextn naming so the existing
+    tensor_map handles them."""
+
+    supports_mtp_export = True
+    hparams: dict[str, Any]
+    model_arch: gguf.MODEL_ARCH
+    gguf_writer: gguf.GGUFWriter
+    block_count: int
+    tensor_map: gguf.TensorNameMap
+    no_mtp: bool
+    mtp_only: bool
+    _original_block_count: int | None = None
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.block_count = self.hparams["num_hidden_layers"]
+        if not self.no_mtp:
+            self.block_count += self.hparams.get("mtp_num_hidden_layers", 0)
+        self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
+
+    def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
+        hparams = {**self.hparams, **self.hparams.get("text_config", {})}
+        key = next((k for k in ["n_layers", "num_hidden_layers", "n_layer", "num_layers"] if k in hparams), None)
+        type(self)._original_block_count = hparams.get(key)
+        return super().index_tensors(remote_hf_model_id=remote_hf_model_id)  # ty: ignore[unresolved-attribute]
+
+    @classmethod
+    def filter_tensors(cls, item):
+        assert cls._original_block_count is not None
+        # TODO: change TextModel to super()
+        if (titem := TextModel.filter_tensors(item)) is None:
+            return None
+        name, gen = titem
+        if name.startswith("model.mtp."):
+            name = name.replace("model.", "", 1)
+        if name.startswith("mtp."):
+            if cls.no_mtp:
+                return None
+            remapper = {
+                "fc":                    "eh_proj",
+                "pre_fc_norm_embedding": "enorm",
+                "pre_fc_norm_hidden":    "hnorm",
+                "norm":                  "shared_head.norm",
+            }
+            parts = name.split(".", 3)
+            if len(parts) == 4 and parts[1] == "layers" and parts[2].isdecimal():
+                mtp_idx = int(parts[2])
+                name = f"model.layers.{cls._original_block_count + mtp_idx}.{parts[3]}"
+            elif len(parts) == 3 and parts[1] in remapper:
+                name = f"model.layers.{cls._original_block_count}.{remapper[parts[1]]}.{parts[2]}"
+        elif cls.mtp_only:
+            keep = name in (
+                "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight",
+                "embed_tokens.weight", "norm.weight",
+            )
+            if not keep:
+                return None
+        return name, gen
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()  # ty: ignore[unresolved-attribute]
+        if self.no_mtp:
+            return
+        if (n := self.hparams.get("mtp_num_hidden_layers", 0)) > 0:
+            self.gguf_writer.add_nextn_predict_layers(n)
+
+    def prepare_metadata(self, vocab_only: bool):
+        from_dir = self.fname_out.is_dir()
+        super().prepare_metadata(vocab_only=vocab_only)  # ty: ignore[unresolved-attribute]
+
+        if not self.mtp_only or not from_dir:
+            return
+
+        output_type: str = self.ftype.name.partition("_")[2]  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+        fname_default: str = gguf.naming_convention(
+            self.metadata.name, self.metadata.basename, self.metadata.finetune,                  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+            self.metadata.version, size_label=None, output_type=output_type, model_type=None)    # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+        self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf"
+
+
 @ModelBase.register("Qwen3_5ForConditionalGeneration", "Qwen3_5ForCausalLM")
-class Qwen3_5TextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
+class Qwen3_5TextModel(_Qwen35MtpMixin, _Qwen35MRopeMixin, _LinearAttentionVReorderBase):
     model_arch = gguf.MODEL_ARCH.QWEN35
 
 
 @ModelBase.register("Qwen3_5MoeForConditionalGeneration", "Qwen3_5MoeForCausalLM")
-class Qwen3_5MoeTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
+class Qwen3_5MoeTextModel(_Qwen35MtpMixin, _Qwen35MRopeMixin, _LinearAttentionVReorderBase):
     model_arch = gguf.MODEL_ARCH.QWEN35MOE
 
 

convert_hf_to_gguf.py

@@ -122,12 +122,8 @@ def parse_args() -> argparse.Namespace:
         help="Export only the multi-token prediction (MTP) head as a separate GGUF, suitable for use as a speculative draft. An 'mtp-' prefix will be added to the output file name.",
     )
     parser.add_argument(
-        "--no-nextn", "--no-mtp", dest="no_mtp", action="store_true",
-        help="Exclude NextN speculative draft tensors from the converted GGUF. Pair with --mtp or --dspark on a second run to publish target and draft as two files.",
-    )
-    parser.add_argument(
-        "--dspark", action="store_true",
-        help="Export only the DeepSeek-V4 DSpark draft tensors as a separate GGUF.",
+        "--no-mtp", action="store_true",
+        help="Exclude the multi-token prediction (MTP) head from the converted GGUF. Pair with --mtp on a second run to publish trunk and MTP as two files. Note: the split form duplicates embeddings, but even though the bundled default is more space-efficient overall, this allows differing quantization which may be more performant.",
     )
     parser.add_argument(
         "--mistral-format", action="store_true",
@@ -258,20 +254,13 @@ def main() -> None:
             from conversion.mistral import MistralModel
             model_class = MistralModel
 
-        if sum((args.mtp, args.no_mtp, args.dspark)) > 1:
-            logger.error("--mtp, --no-nextn, and --dspark are mutually exclusive")
+        if args.mtp and args.no_mtp:
+            logger.error("--mtp and --no-mtp are mutually exclusive")
             sys.exit(1)
 
-        if args.dspark:
-            if is_mistral_format or model_architecture != "DeepseekV4ForCausalLM":
-                logger.error("--dspark is only supported for DeepseekV4ForCausalLM")
-                sys.exit(1)
-            from conversion.deepseek import DeepseekV4DSparkModel
-            model_class = DeepseekV4DSparkModel
-
         if args.mtp or args.no_mtp:
             if not model_class.supports_mtp_export:
-                logger.error("--mtp / --no-nextn are not supported for %s", model_architecture)
+                logger.error("--mtp / --no-mtp are not supported for %s", model_architecture)
                 sys.exit(1)
             if args.no_mtp:
                 model_class.no_mtp = True

docs/ops.md

@@ -23,16 +23,16 @@ Legend:
 |                           ARGMAX | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                          ARGSORT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             CEIL | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                            CLAMP | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                        COL2IM_1D | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                            CLAMP | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | 🟡 | ✅ | 🟡 | ✅ | ❌ | ❌ |
+|                        COL2IM_1D | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
 |                           CONCAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
 |                             CONT | ❌ | 🟡 | ✅ | ✅ | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
-|                          CONV_2D | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
+|                          CONV_2D | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                       CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                          CONV_3D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                          CONV_3D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
 |                CONV_TRANSPOSE_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                CONV_TRANSPOSE_2D | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                              COS | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              COS | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ |
 |                      COUNT_EQUAL | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                              CPY | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
 |               CROSS_ENTROPY_LOSS | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
@@ -51,8 +51,8 @@ Legend:
 |                             FILL | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                   FLASH_ATTN_EXT | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
 |                            FLOOR | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                  GATED_DELTA_NET | ❌ | ❌ | ✅ | ❌ | ✅ | 🟡 | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                GATED_LINEAR_ATTN | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                  GATED_DELTA_NET | ❌ | ❌ | ✅ | ❌ | ✅ | 🟡 | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ |
+|                GATED_LINEAR_ATTN | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
 |                            GEGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                        GEGLU_ERF | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                      GEGLU_QUICK | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
@@ -60,14 +60,14 @@ Legend:
 |                         GELU_ERF | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                       GELU_QUICK | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                         GET_ROWS | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
-|                    GET_ROWS_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ |
+|                    GET_ROWS_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
 |                       GROUP_NORM | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                      HARDSIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                        HARDSWISH | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                           IM2COL | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                        IM2COL_3D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                          L2_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ✅ | ✅ | 🟡 | ❌ | ❌ |
-|                       LEAKY_RELU | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                       LEAKY_RELU | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | ❌ | ✅ | 🟡 | ❌ | ❌ | ❌ |
 |                LIGHTNING_INDEXER | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
 |                              LOG | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
@@ -76,13 +76,13 @@ Legend:
 |                 MUL_MAT_HADAMARD | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                       MUL_MAT_ID | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ❌ |
 |                              NEG | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ |
+|                             NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ❌ |
 |                   OPT_STEP_ADAMW | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
 |                     OPT_STEP_SGD | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
-|                         OUT_PROD | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | 🟡 |
+|                         OUT_PROD | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ | 🟡 |
 |                              PAD | ❌ | 🟡 | ✅ | 🟡 | ❌ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
 |                   PAD_REFLECT_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
-|                          POOL_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                          POOL_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
 |                          POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                            REGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             RELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
@@ -103,13 +103,13 @@ Legend:
 |                          SIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             SILU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                        SILU_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
-|                              SIN | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              SIN | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ |
 |                         SOFTPLUS | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                         SOFT_MAX | ❌ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                    SOFT_MAX_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ |
 |                        SOLVE_TRI | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ |
-|                              SQR | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             SQRT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              SQR | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
+|                             SQRT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
 |                         SSM_CONV | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                         SSM_SCAN | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ |
 |                             STEP | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |

examples/gen-docs/gen-docs.cpp

@@ -70,8 +70,6 @@ static void write_table(std::ostringstream & ss, std::vector<common_arg *> & opt
 
 static void write_help(std::ostringstream & ss, const md_file & md) {
     common_params params;
-    params.is_gen_docs = true;
-
     auto ctx_arg = common_params_parser_init(params, md.ex);
 
     std::vector<common_arg *> common_options;

ggml/CMakeLists.txt

@@ -5,7 +5,7 @@ project("ggml" C CXX ASM)
 ### GGML Version
 set(GGML_VERSION_MAJOR 0)
 set(GGML_VERSION_MINOR 18)
-set(GGML_VERSION_PATCH 1)
+set(GGML_VERSION_PATCH 0)
 set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/")

ggml/src/ggml-backend.cpp

@@ -765,9 +765,8 @@ struct ggml_backend_sched_split {
     int backend_id;
     int i_start;
     int i_end;
-    struct ggml_tensor ** inputs;
+    struct ggml_tensor * inputs[GGML_SCHED_MAX_SPLIT_INPUTS];
     int n_inputs;
-    int inputs_capacity;
     // graph view of this split
     struct ggml_cgraph graph;
 };
@@ -806,9 +805,8 @@ struct ggml_backend_sched {
     int cur_copy;
     int next_copy;
     ggml_backend_event_t events[GGML_SCHED_MAX_BACKENDS][GGML_SCHED_MAX_COPIES];
-    struct ggml_tensor ** graph_inputs;
+    struct ggml_tensor * graph_inputs[GGML_SCHED_MAX_SPLIT_INPUTS];
     int n_graph_inputs;
-    int graph_inputs_capacity;
 
     struct ggml_context * ctx;
 
@@ -834,36 +832,6 @@ struct ggml_backend_sched {
 #define tensor_id_copy(id, backend_id, copy_id) sched->hv_tensor_copies[(id) * sched->n_backends * sched->n_copies + (backend_id) * sched->n_copies + (copy_id)]
 #define tensor_copy(tensor, backend_id, copy_id) tensor_id_copy(hash_id(tensor), backend_id, copy_id)
 
-static void ggml_backend_sched_split_inputs_grow(struct ggml_backend_sched_split * split) {
-    int new_cap = GGML_SCHED_MAX_SPLIT_INPUTS;
-    if (split->inputs_capacity > 0) {
-        new_cap = 2*split->inputs_capacity;
-        GGML_LOG_WARN("%s: increasing split inputs capacity from %d to %d\n", __func__, split->inputs_capacity, new_cap);
-    }
-    auto * pnew = (struct ggml_tensor **) realloc((void *) split->inputs, new_cap * sizeof(struct ggml_tensor *));
-    if (pnew == NULL) {
-        GGML_LOG_ERROR("%s: failed to allocate %zu bytes\n", __func__, new_cap * sizeof(struct ggml_tensor *));
-        GGML_ABORT("failed to grow split inputs container");
-    }
-    split->inputs = pnew;
-    split->inputs_capacity = new_cap;
-}
-
-static void ggml_backend_sched_graph_inputs_grow(ggml_backend_sched_t sched) {
-    int new_cap = GGML_SCHED_MAX_SPLIT_INPUTS;
-    if (sched->graph_inputs_capacity > 0) {
-        new_cap = 2*sched->graph_inputs_capacity;
-        GGML_LOG_WARN("%s: increasing graph inputs capacity from %d to %d\n", __func__, sched->graph_inputs_capacity, new_cap);
-    }
-    auto * pnew = (struct ggml_tensor **) realloc((void *) sched->graph_inputs, new_cap * sizeof(struct ggml_tensor *));
-    if (pnew == NULL) {
-        GGML_LOG_ERROR("%s: failed to allocate %zu bytes\n", __func__, new_cap * sizeof(struct ggml_tensor *));
-        GGML_ABORT("failed to grow graph inputs container");
-    }
-    sched->graph_inputs = pnew;
-    sched->graph_inputs_capacity = new_cap;
-}
-
 // returns the priority of the backend, lower id is higher priority
 static int ggml_backend_sched_backend_id(ggml_backend_sched_t sched, ggml_backend_t backend) {
     for (int i = 0; i < sched->n_backends; i++) {
@@ -1329,7 +1297,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
                     }
                     // check if the split has too many inputs
                     // FIXME: count the number of inputs instead of only checking when full
-                    if (split->n_inputs >= split->inputs_capacity) {
+                    if (split->n_inputs == GGML_SCHED_MAX_SPLIT_INPUTS) {
                         const size_t id = hash_id(src);
                         int src_backend_id = sched->hv_tensor_backend_ids[id];
                         bool supported = ggml_backend_sched_buffer_supported(sched, src, cur_backend_id);
@@ -1345,14 +1313,10 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
                 split->i_end = i;
                 i_split++;
                 if (i_split >= sched->splits_capacity) {
-                    int old_cap = sched->splits_capacity;
                     sched->splits_capacity *= 2;
                     sched->splits = (ggml_backend_sched_split *)
                         realloc(sched->splits, sched->splits_capacity * sizeof(struct ggml_backend_sched_split));
                     GGML_ASSERT(sched->splits != NULL);
-                    for (int k = old_cap; k < sched->splits_capacity; k++) {
-                        memset(&sched->splits[k], 0, sizeof(struct ggml_backend_sched_split));
-                    }
                 }
                 split = &sched->splits[i_split];
                 split->backend_id = node_backend_id;
@@ -1389,9 +1353,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
                             SET_CAUSE(tensor_copy, "4.cpy");
                         }
                         int n_graph_inputs = sched->n_graph_inputs++;
-                        if (n_graph_inputs >= sched->graph_inputs_capacity) {
-                            ggml_backend_sched_graph_inputs_grow(sched);
-                        }
+                        GGML_ASSERT(n_graph_inputs < GGML_SCHED_MAX_SPLIT_INPUTS);
                         sched->graph_inputs[n_graph_inputs] = src;
                     }
                 }
@@ -1411,9 +1373,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
                             SET_CAUSE(tensor_copy, "4.cpy");
                         }
                         int n_inputs = split->n_inputs++;
-                        if (n_inputs >= split->inputs_capacity) {
-                            ggml_backend_sched_split_inputs_grow(split);
-                        }
+                        GGML_ASSERT(n_inputs < GGML_SCHED_MAX_SPLIT_INPUTS);
                         split->inputs[n_inputs] = src;
                     }
                     node->src[j] = tensor_id_copy(src_id, cur_backend_id, sched->cur_copy);
@@ -1439,11 +1399,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
         sched->prev_leaf_backend_ids = tmp;
     }
 
-    int total_inputs = sched->n_graph_inputs;
-    for (int i = 0; i < sched->n_splits; i++) {
-        total_inputs += sched->splits[i].n_inputs;
-    }
-    int graph_size = std::max(graph->n_nodes, graph->n_leafs) + total_inputs * 2 * sched->n_copies;
+    int graph_size = std::max(graph->n_nodes, graph->n_leafs) + sched->n_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2*sched->n_copies;
 
     // remember the actual graph_size for performing reallocation checks later [GGML_SCHED_DEBUG_REALLOC]
     sched->debug_prev_graph_size = sched->debug_graph_size;
@@ -1826,9 +1782,6 @@ ggml_backend_sched_t ggml_backend_sched_new(
     sched->splits = (ggml_backend_sched_split *) calloc(initial_splits_capacity, sizeof(sched->splits[0]));
     sched->splits_capacity = initial_splits_capacity;
 
-    sched->graph_inputs_capacity = GGML_SCHED_MAX_SPLIT_INPUTS;
-    sched->graph_inputs = (struct ggml_tensor **) calloc(sched->graph_inputs_capacity, sizeof(struct ggml_tensor *));
-
     for (int b = 0; b < n_backends; b++) {
         sched->backends[b] = backends[b];
         sched->bufts[b] = bufts ? bufts[b] : ggml_backend_get_default_buffer_type(backends[b]);
@@ -1861,11 +1814,7 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) {
     ggml_gallocr_free(sched->galloc);
     ggml_free(sched->ctx);
     ggml_hash_set_free(&sched->hash_set);
-    for (int i = 0; i < sched->splits_capacity; i++) {
-        free(sched->splits[i].inputs);
-    }
     free(sched->splits);
-    free(sched->graph_inputs);
     free(sched->hv_tensor_backend_ids);
     free(sched->hv_tensor_copies);
     free(sched->node_backend_ids);

ggml/src/ggml-cuda/common.cuh

@@ -627,8 +627,7 @@ template <typename T> struct block_reduce_policy<block_reduce_method::MAX, T> {
 };
 
 template <block_reduce_method reduce_method_t, const unsigned int block_size_template = 0, typename T>
-static __device__ T block_reduce(T val, [[maybe_unused]] T * shared_vals) {
-    // for multi-warp reductions, callers must not reuse shared_vals until all reads from this invocation have completed
+static __device__ T block_reduce(T val, T * shared_vals) {
     val                           = block_reduce_policy<reduce_method_t, T>::reduce(val);
     const unsigned int block_size = block_size_template == 0 ? blockDim.x : block_size_template;
     if (block_size > WARP_SIZE) {

ggml/src/ggml-cuda/norm.cu

@@ -64,7 +64,7 @@ static __global__ void group_norm_f32(const float * x, float * dst, const int gr
         tmp += xi * xi;
     }
 
-    tmp = block_reduce<block_reduce_method::SUM, block_size>(tmp, s_sum + 32);
+    tmp = block_reduce<block_reduce_method::SUM, block_size>(tmp, s_sum);
 
     const float variance = tmp / group_size;
     const float scale = rsqrtf(variance + eps);
@@ -297,7 +297,7 @@ static void group_norm_f32_cuda(
         group_norm_f32<WARP_SIZE><<<num_groups, block_dims, 0, stream>>>(x, dst, group_size, ne_elements, eps);
     } else {
         const dim3 block_dims(1024, 1, 1);
-        group_norm_f32<1024><<<num_groups, block_dims, block_dims.x > WARP_SIZE ? 2 * 32 * sizeof(float): 0, stream>>>(x, dst, group_size, ne_elements, eps);
+        group_norm_f32<1024><<<num_groups, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream>>>(x, dst, group_size, ne_elements, eps);
     }
 }
 

ggml/src/ggml-cuda/softmax.cu

@@ -116,11 +116,6 @@ static __global__ void soft_max_f32(
         vals[col] = val;
     }
 
-    if (block_size > WARP_SIZE) {
-        // sync is needed as we reuse buf_iw across block_reduce invocations, see #26385
-        // for block_size <= WARP_SIZE, block_reduce does not access buf_iw
-        __syncthreads();
-    }
     // find the sum of exps in the block
     tmp = block_reduce<block_reduce_method::SUM, block_size_template>(tmp, buf_iw);
 
@@ -147,8 +142,6 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __
                                                                 float * __restrict__ dst,
                                                                 float * __restrict__ tmp_maxs,
                                                                 float * __restrict__ tmp_sums,
-                                                                float * shared_vals_max,
-                                                                float * shared_vals_sum,
                                                                 const soft_max_params p) {
     namespace cg = cooperative_groups;
 
@@ -161,6 +154,7 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __
     float     local_vals[n_elem_per_thread] = { -INFINITY, -INFINITY, -INFINITY, -INFINITY };
     float     local_max                     = -INFINITY;
     const int step_size                     = gridDim.x * blockDim.x;
+    __shared__ float shared_vals[32];
 
     // Compute thread-local max
     for (int col = col_start; col < p.ncols;) {
@@ -177,7 +171,7 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __
     }
 
     // Compute CTA-level max
-    local_max = block_reduce<block_reduce_method::MAX>(local_max, shared_vals_max);
+    local_max = block_reduce<block_reduce_method::MAX>(local_max, shared_vals);
 
     // Store CTA-level max to GMEM
     if (tid == 0) {
@@ -192,7 +186,7 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __
     } else {
         local_max = -INFINITY;
     }
-    local_max = block_reduce<block_reduce_method::MAX>(local_max, shared_vals_max);
+    local_max = block_reduce<block_reduce_method::MAX>(local_max, shared_vals);
 
     // Compute softmax dividends, accumulate divisor
     float tmp_expf = 0.0f;
@@ -215,7 +209,7 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __
     }
 
     // Reduce divisor within CTA
-    tmp_expf = block_reduce<block_reduce_method::SUM>(tmp_expf, shared_vals_sum);
+    tmp_expf = block_reduce<block_reduce_method::SUM>(tmp_expf, shared_vals);
 
     // Store CTA-level sum to GMEM
     if (tid == 0) {
@@ -229,7 +223,7 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __
     } else {
         tmp_expf = 0.0f;
     }
-    tmp_expf = block_reduce<block_reduce_method::SUM>(tmp_expf, shared_vals_sum);
+    tmp_expf = block_reduce<block_reduce_method::SUM>(tmp_expf, shared_vals);
 
     // Divide dividend by global sum + store data
     for (int col = col_start; col < p.ncols;) {
@@ -316,11 +310,9 @@ __launch_bounds__(8*WARP_SIZE, 1) static __global__ void soft_max_f32_paralleliz
 // https://docs.nvidia.com/cuda/cuda-programming-guide/05-appendices/device-callable-apis.html#grid-synchronization
 // https://docs.nvidia.com/cuda/cuda-programming-guide/05-appendices/device-callable-apis.html#class-cluster-group
 {
-    __shared__ float shared_vals[2][32];
-
     for (int rowx = 0; rowx < p.ne01 * p.ne02 * p.ne03; rowx++) {
         soft_max_f32_parallelize_cols_single_row(x + int64_t(rowx) * p.ncols, dst + int64_t(rowx) * p.ncols, tmp_maxs,
-                                                 tmp_sums, shared_vals[0], shared_vals[1], p);
+                                                 tmp_sums, p);
     }
 }
 

ggml/src/ggml-metal/ggml-metal-device.cpp

@@ -477,41 +477,6 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_soft_max(ggml_me
     return res;
 }
 
-ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_lightning_indexer(
-        ggml_metal_library_t lib,
-        const ggml_tensor * op) {
-    GGML_ASSERT(op->op == GGML_OP_LIGHTNING_INDEXER);
-
-    char name[256];
-
-    snprintf(name, 256, "kernel_lightning_indexer_%s", ggml_type_name(op->src[1]->type));
-
-    ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
-    if (!res.pipeline) {
-        res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);
-    }
-
-    return res;
-}
-
-ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc(ggml_metal_library_t lib, ggml_op op) {
-    const char * name = nullptr;
-
-    switch (op) {
-        case GGML_OP_DSV4_HC_COMB: name = "kernel_dsv4_hc_comb_f32"; break;
-        case GGML_OP_DSV4_HC_PRE:  name = "kernel_dsv4_hc_pre_f32";  break;
-        case GGML_OP_DSV4_HC_POST: name = "kernel_dsv4_hc_post_f32"; break;
-        default: GGML_ABORT("fatal error");
-    }
-
-    ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
-    if (!res.pipeline) {
-        res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);
-    }
-
-    return res;
-}
-
 ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv(ggml_metal_library_t lib, const ggml_tensor * op) {
     GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32);
     GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32);
@@ -2152,23 +2117,6 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_opt_step_sgd(ggm
     return res;
 }
 
-ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_silu_back(ggml_metal_library_t lib, const ggml_tensor * op) {
-    assert(op->op == GGML_OP_SILU_BACK);
-
-    char base[256];
-    char name[256];
-
-    snprintf(base, 256, "kernel_silu_back_%s", ggml_type_name(op->src[0]->type));
-    snprintf(name, 256, "%s", base);
-
-    ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
-    if (!res.pipeline) {
-        res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);
-    }
-
-    return res;
-}
-
 ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_memset(ggml_metal_library_t lib, const ggml_tensor *  op) {
     GGML_ASSERT(op->type == GGML_TYPE_I64);
 

ggml/src/ggml-metal/ggml-metal-device.h

@@ -117,7 +117,6 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_diag
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_repeat            (ggml_metal_library_t lib, enum ggml_type tsrc);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_concat            (ggml_metal_library_t lib, enum ggml_type tsrc);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_unary             (ggml_metal_library_t lib, const struct ggml_tensor * op);
-struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_silu_back         (ggml_metal_library_t lib, const struct ggml_tensor * op);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_glu               (ggml_metal_library_t lib, const struct ggml_tensor * op);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_sum               (ggml_metal_library_t lib, const struct ggml_tensor * op);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_sum_rows          (ggml_metal_library_t lib, const struct ggml_tensor * op);
@@ -125,8 +124,6 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cumsum_bl
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cumsum_add        (ggml_metal_library_t lib, const struct ggml_tensor * op);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_tri               (ggml_metal_library_t lib, const struct ggml_tensor * op);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_soft_max          (ggml_metal_library_t lib, const struct ggml_tensor * op);
-struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_lightning_indexer (ggml_metal_library_t lib, const struct ggml_tensor * op);
-struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc           (ggml_metal_library_t lib, enum ggml_op op);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv          (ggml_metal_library_t lib, const struct ggml_tensor * op);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched  (ggml_metal_library_t lib, const struct ggml_tensor * op, int ssm_conv_bs);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_scan          (ggml_metal_library_t lib, const struct ggml_tensor * op);

ggml/src/ggml-metal/ggml-metal-device.m

@@ -2,7 +2,6 @@
 
 #import "ggml-impl.h"
 #import "ggml-backend-impl.h"
-#import "ggml-metal-impl.h"
 
 #include <Foundation/Foundation.h>
 
@@ -1138,14 +1137,6 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
                 default:
                     return false;
             }
-        case GGML_OP_SILU_BACK:
-            return (op->src[0]->type == GGML_TYPE_F32) &&
-                (op->src[1]->type == GGML_TYPE_F32) &&
-                (op->type == GGML_TYPE_F32) &&
-                ggml_is_contiguous(op->src[0]) &&
-                ggml_is_contiguous(op->src[1]) &&
-                ggml_is_contiguous(op) &&
-                ggml_are_same_shape(op->src[0], op->src[1]);
         case GGML_OP_GLU:
             switch (ggml_get_glu_op(op)) {
                 case GGML_GLU_OP_REGLU:
@@ -1190,7 +1181,6 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
         case GGML_OP_MUL:
         case GGML_OP_DIV:
         case GGML_OP_ADD_ID:
-            return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && (op->src[0]->type == op->src[1]->type);
         case GGML_OP_ACC:
             return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]) && op->src[0]->type == GGML_TYPE_F32;
         case GGML_OP_REPEAT:
@@ -1309,72 +1299,6 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
                     return false;
             }
             return has_simdgroup_mm; // TODO: over-restricted for vec-kernels
-        case GGML_OP_LIGHTNING_INDEXER:
-            if (op->src[0]->ne[0] != OP_LIGHTNING_INDEXER_DK ||
-                op->src[0]->ne[1] != OP_LIGHTNING_INDEXER_NH) {
-                return false;
-            }
-            if (!has_simdgroup_mm ||
-                op->src[0]->type != GGML_TYPE_F32 ||
-                op->src[2]->type != GGML_TYPE_F32 ||
-                op->src[3]->type != GGML_TYPE_F16 ||
-                op->type         != GGML_TYPE_F32 ||
-                !ggml_is_contiguous_rows(op->src[0]) ||
-                !ggml_is_contiguous_rows(op->src[1]) ||
-                !ggml_is_contiguous_rows(op->src[2]) ||
-                !ggml_is_contiguous_rows(op->src[3])) {
-                return false;
-            }
-            switch (op->src[1]->type) {
-                case GGML_TYPE_F32:
-                case GGML_TYPE_F16:
-                case GGML_TYPE_Q4_0:
-                case GGML_TYPE_Q4_1:
-                case GGML_TYPE_Q5_0:
-                case GGML_TYPE_Q5_1:
-                case GGML_TYPE_Q8_0:
-                    return true;
-                case GGML_TYPE_BF16:
-                    return has_bfloat;
-                default:
-                    return false;
-            }
-        case GGML_OP_DSV4_HC_COMB:
-            return has_simdgroup_reduction &&
-                op->src[0]->type == GGML_TYPE_F32 &&
-                op->src[1]->type == GGML_TYPE_F32 &&
-                op->src[2]->type == GGML_TYPE_F32 &&
-                op->type         == GGML_TYPE_F32 &&
-                op->src[0]->ne[0] == 24 &&
-                op->src[1]->ne[0] >= 3 &&
-                op->src[2]->ne[0] == 24 &&
-                ggml_is_contiguous_rows(op->src[0]) &&
-                ggml_is_contiguous_rows(op->src[1]) &&
-                ggml_is_contiguous_rows(op->src[2]);
-        case GGML_OP_DSV4_HC_PRE:
-            return has_simdgroup_reduction &&
-                op->src[0]->type == GGML_TYPE_F32 &&
-                op->src[1]->type == GGML_TYPE_F32 &&
-                op->type         == GGML_TYPE_F32 &&
-                op->src[0]->ne[1] == 4 &&
-                op->src[1]->ne[0] == 4 &&
-                ggml_is_contiguous_rows(op->src[0]) &&
-                ggml_is_contiguous_rows(op->src[1]);
-        case GGML_OP_DSV4_HC_POST:
-            return has_simdgroup_reduction &&
-                op->src[0]->type == GGML_TYPE_F32 &&
-                op->src[1]->type == GGML_TYPE_F32 &&
-                op->src[2]->type == GGML_TYPE_F32 &&
-                op->src[3]->type == GGML_TYPE_F32 &&
-                op->type         == GGML_TYPE_F32 &&
-                op->src[1]->ne[1] == 4 &&
-                op->src[2]->ne[0] == 4 &&
-                op->src[3]->ne[0] == 4 &&
-                op->src[3]->ne[1] == 4 &&
-                ggml_is_contiguous_rows(op->src[0]) &&
-                ggml_is_contiguous_rows(op->src[1]) &&
-                ggml_is_contiguous_rows(op->src[2]) &&
-                ggml_is_contiguous_rows(op->src[3]);
         case GGML_OP_SSM_CONV:
         case GGML_OP_SSM_SCAN:
             return has_simdgroup_reduction;

ggml/src/ggml-metal/ggml-metal-impl.h

@@ -112,13 +112,6 @@
 #define OP_FLASH_ATTN_EXT_VEC_NQPSG 1
 #define OP_FLASH_ATTN_EXT_VEC_NCPSG 32
 
-#define OP_LIGHTNING_INDEXER_DK    128
-#define OP_LIGHTNING_INDEXER_NH     64
-#define OP_LIGHTNING_INDEXER_NHPTG   8
-#define OP_LIGHTNING_INDEXER_NKPSG   8
-#define OP_LIGHTNING_INDEXER_NSG     8
-#define OP_LIGHTNING_INDEXER_NBPTG   8
-
 #define OP_UNARY_NUM_SCALE      10
 #define OP_UNARY_NUM_FILL       11
 #define OP_UNARY_NUM_CLAMP      12
@@ -1178,66 +1171,6 @@ typedef struct {
     int64_t val;
 } ggml_metal_kargs_memset;
 
-typedef struct {
-    int32_t  n_kv;
-    int32_t  n_batch;
-    int32_t  mask_ne3;
-    uint64_t nb1;
-    uint64_t nb3;
-    uint64_t nbq1;
-    uint64_t nbq2;
-    uint64_t nbq3;
-    uint64_t nbk2;
-    uint64_t nbk3;
-    uint64_t nbw1;
-    uint64_t nbw3;
-    uint64_t nbm1;
-    uint64_t nbm3;
-} ggml_metal_kargs_lightning_indexer;
-
-typedef struct {
-    int32_t  n_tokens;
-    int32_t  n_iter;
-    uint64_t nb_m0;
-    uint64_t nb_m1;
-    uint64_t nb_s0;
-    uint64_t nb_b0;
-    uint64_t nb_d0;
-    uint64_t nb_d1;
-    uint64_t nb_d2;
-    float    eps;
-} ggml_metal_kargs_dsv4_hc_comb;
-
-typedef struct {
-    int32_t  n_embd;
-    int32_t  n_tokens;
-    uint64_t nb_x0;
-    uint64_t nb_x1;
-    uint64_t nb_x2;
-    uint64_t nb_w0;
-    uint64_t nb_w1;
-    uint64_t nb_d0;
-    uint64_t nb_d1;
-} ggml_metal_kargs_dsv4_hc_pre;
-
-typedef struct {
-    int32_t  n_embd;
-    int32_t  n_tokens;
-    uint64_t nb_x0;
-    uint64_t nb_x1;
-    uint64_t nb_r0;
-    uint64_t nb_r1;
-    uint64_t nb_r2;
-    uint64_t nb_p0;
-    uint64_t nb_p1;
-    uint64_t nb_c0;
-    uint64_t nb_c1;
-    uint64_t nb_c2;
-    uint64_t nb_d0;
-    uint64_t nb_d1;
-    uint64_t nb_d2;
-} ggml_metal_kargs_dsv4_hc_post;
-
 typedef struct {
     int32_t  ne00;
     int32_t  ne01;
@@ -1289,8 +1222,4 @@ typedef struct {
     int64_t  np;
 } ggml_metal_kargs_opt_step_sgd;
 
-typedef struct {
-    int64_t ne;
-} ggml_metal_kargs_silu_back;
-
 #endif // GGML_METAL_IMPL

ggml/src/ggml-metal/ggml-metal-ops.cpp

@@ -299,10 +299,6 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) {
             {
                 n_fuse = ggml_metal_op_unary(ctx, idx);
             } break;
-        case GGML_OP_SILU_BACK:
-            {
-                n_fuse = ggml_metal_op_silu_back(ctx, idx);
-            } break;
         case GGML_OP_GLU:
             {
                 n_fuse = ggml_metal_op_glu(ctx, idx);
@@ -320,16 +316,6 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) {
             {
                 n_fuse = ggml_metal_op_cumsum(ctx, idx);
             } break;
-        case GGML_OP_LIGHTNING_INDEXER:
-            {
-                n_fuse = ggml_metal_op_lightning_indexer(ctx, idx);
-            } break;
-        case GGML_OP_DSV4_HC_COMB:
-        case GGML_OP_DSV4_HC_PRE:
-        case GGML_OP_DSV4_HC_POST:
-            {
-                n_fuse = ggml_metal_op_dsv4_hc(ctx, idx);
-            } break;
         case GGML_OP_SOFT_MAX:
             {
                 n_fuse = ggml_metal_op_soft_max(ctx, idx);
@@ -1311,203 +1297,6 @@ int ggml_metal_op_diag(ggml_metal_op_t ctx, int idx) {
     return 1;
 }
 
-int ggml_metal_op_lightning_indexer(ggml_metal_op_t ctx, int idx) {
-    ggml_tensor * op = ctx->node(idx);
-
-    ggml_metal_encoder_t enc = ctx->enc;
-
-    GGML_ASSERT(op->op == GGML_OP_LIGHTNING_INDEXER);
-
-    const ggml_tensor * q = op->src[0];
-    const ggml_tensor * k = op->src[1];
-    const ggml_tensor * w = op->src[2];
-    const ggml_tensor * m = op->src[3];
-
-    GGML_ASSERT(q->type == GGML_TYPE_F32);
-    GGML_ASSERT(k->type == GGML_TYPE_F32  ||
-                k->type == GGML_TYPE_F16  ||
-                k->type == GGML_TYPE_BF16 ||
-                k->type == GGML_TYPE_Q4_0 ||
-                k->type == GGML_TYPE_Q4_1 ||
-                k->type == GGML_TYPE_Q5_0 ||
-                k->type == GGML_TYPE_Q5_1 ||
-                k->type == GGML_TYPE_Q8_0);
-    GGML_ASSERT(w->type == GGML_TYPE_F32);
-    GGML_ASSERT(m->type == GGML_TYPE_F16);
-    GGML_ASSERT(op->type == GGML_TYPE_F32);
-
-    GGML_ASSERT(q->ne[0] == OP_LIGHTNING_INDEXER_DK);
-    GGML_ASSERT(q->ne[1] == OP_LIGHTNING_INDEXER_NH);
-
-    ggml_metal_kargs_lightning_indexer args = {
-        /*.n_kv      =*/ (int32_t) k->ne[2],
-        /*.n_batch   =*/ (int32_t) q->ne[2],
-        /*.mask_ne3  =*/ (int32_t) m->ne[3],
-        /*.nb1       =*/ op->nb[1],
-        /*.nb3       =*/ op->nb[3],
-        /*.nbq1      =*/ q->nb[1],
-        /*.nbq2      =*/ q->nb[2],
-        /*.nbq3      =*/ q->nb[3],
-        /*.nbk2      =*/ k->nb[2],
-        /*.nbk3      =*/ k->nb[3],
-        /*.nbw1      =*/ w->nb[1],
-        /*.nbw3      =*/ w->nb[3],
-        /*.nbm1      =*/ m->nb[1],
-        /*.nbm3      =*/ m->nb[3],
-    };
-
-    ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(q),  1);
-    ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(k),  2);
-    ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(w),  3);
-    ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(m),  4);
-    ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 5);
-
-    const int nsg   = OP_LIGHTNING_INDEXER_NSG;
-    const int nkptg = OP_LIGHTNING_INDEXER_NKPSG*nsg;
-    const int nbptg = OP_LIGHTNING_INDEXER_NBPTG;
-
-    auto pipeline = ggml_metal_library_get_pipeline_lightning_indexer(ctx->lib, op);
-    ggml_metal_encoder_set_pipeline(enc, pipeline);
-    ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0);
-    ggml_metal_encoder_dispatch_threadgroups(enc,
-            (k->ne[2] + nkptg - 1)/nkptg,
-            (q->ne[2] + nbptg - 1)/nbptg,
-            q->ne[3], 32, nsg, 1);
-
-    return 1;
-}
-
-int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) {
-    ggml_tensor * op = ctx->node(idx);
-
-    ggml_metal_encoder_t enc = ctx->enc;
-    auto pipeline = ggml_metal_library_get_pipeline_dsv4_hc(ctx->lib, op->op);
-
-    ggml_metal_encoder_set_pipeline(enc, pipeline);
-
-    switch (op->op) {
-        case GGML_OP_DSV4_HC_COMB:
-            {
-                const ggml_tensor * mixes = op->src[0];
-                const ggml_tensor * scale = op->src[1];
-                const ggml_tensor * base  = op->src[2];
-
-                GGML_ASSERT(mixes->type == GGML_TYPE_F32);
-                GGML_ASSERT(scale->type == GGML_TYPE_F32);
-                GGML_ASSERT(base->type  == GGML_TYPE_F32);
-                GGML_ASSERT(op->type    == GGML_TYPE_F32);
-                GGML_ASSERT(mixes->ne[0] == 24);
-                GGML_ASSERT(op->ne[0] == 4 && op->ne[1] == 4);
-
-                ggml_metal_kargs_dsv4_hc_comb args = {
-                    /*.n_tokens =*/ (int32_t) mixes->ne[1],
-                    /*.n_iter   =*/ ggml_get_op_params_i32(op, 1),
-                    /*.nb_m0    =*/ mixes->nb[0],
-                    /*.nb_m1    =*/ mixes->nb[1],
-                    /*.nb_s0    =*/ scale->nb[0],
-                    /*.nb_b0    =*/ base->nb[0],
-                    /*.nb_d0    =*/ op->nb[0],
-                    /*.nb_d1    =*/ op->nb[1],
-                    /*.nb_d2    =*/ op->nb[2],
-                    /*.eps      =*/ ggml_get_op_params_f32(op, 0),
-                };
-
-                ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
-                ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(mixes), 1);
-                ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(scale), 2);
-                ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(base),  3);
-                ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op),    4);
-
-                // One SIMDgroup owns one 4x4 Sinkhorn matrix. Packing up to four
-                // independent tokens per threadgroup keeps both decode and prompt
-                // dispatches compact without any threadgroup-memory synchronization.
-                const int nsg = std::min(4, args.n_tokens);
-                ggml_metal_encoder_dispatch_threadgroups(
-                        enc, (args.n_tokens + nsg - 1)/nsg, 1, 1, 32, nsg, 1);
-            } break;
-        case GGML_OP_DSV4_HC_PRE:
-            {
-                const ggml_tensor * x       = op->src[0];
-                const ggml_tensor * weights = op->src[1];
-
-                GGML_ASSERT(x->type       == GGML_TYPE_F32);
-                GGML_ASSERT(weights->type == GGML_TYPE_F32);
-                GGML_ASSERT(op->type      == GGML_TYPE_F32);
-                GGML_ASSERT(x->ne[1] == 4);
-
-                ggml_metal_kargs_dsv4_hc_pre args = {
-                    /*.n_embd   =*/ (int32_t) x->ne[0],
-                    /*.n_tokens =*/ (int32_t) x->ne[2],
-                    /*.nb_x0    =*/ x->nb[0],
-                    /*.nb_x1    =*/ x->nb[1],
-                    /*.nb_x2    =*/ x->nb[2],
-                    /*.nb_w0    =*/ weights->nb[0],
-                    /*.nb_w1    =*/ weights->nb[1],
-                    /*.nb_d0    =*/ op->nb[0],
-                    /*.nb_d1    =*/ op->nb[1],
-                };
-
-                ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
-                ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(x),       1);
-                ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(weights), 2);
-                ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op),      3);
-
-                const int n_tiles = (args.n_embd + 31)/32;
-                const int nsg = std::min(4, n_tiles);
-                ggml_metal_encoder_dispatch_threadgroups(
-                        enc, (n_tiles + nsg - 1)/nsg, args.n_tokens, 1, 32, nsg, 1);
-            } break;
-        case GGML_OP_DSV4_HC_POST:
-            {
-                const ggml_tensor * x        = op->src[0];
-                const ggml_tensor * residual = op->src[1];
-                const ggml_tensor * post     = op->src[2];
-                const ggml_tensor * comb     = op->src[3];
-
-                GGML_ASSERT(x->type        == GGML_TYPE_F32);
-                GGML_ASSERT(residual->type == GGML_TYPE_F32);
-                GGML_ASSERT(post->type     == GGML_TYPE_F32);
-                GGML_ASSERT(comb->type     == GGML_TYPE_F32);
-                GGML_ASSERT(op->type       == GGML_TYPE_F32);
-                GGML_ASSERT(residual->ne[1] == 4);
-
-                ggml_metal_kargs_dsv4_hc_post args = {
-                    /*.n_embd   =*/ (int32_t) x->ne[0],
-                    /*.n_tokens =*/ (int32_t) x->ne[1],
-                    /*.nb_x0    =*/ x->nb[0],
-                    /*.nb_x1    =*/ x->nb[1],
-                    /*.nb_r0    =*/ residual->nb[0],
-                    /*.nb_r1    =*/ residual->nb[1],
-                    /*.nb_r2    =*/ residual->nb[2],
-                    /*.nb_p0    =*/ post->nb[0],
-                    /*.nb_p1    =*/ post->nb[1],
-                    /*.nb_c0    =*/ comb->nb[0],
-                    /*.nb_c1    =*/ comb->nb[1],
-                    /*.nb_c2    =*/ comb->nb[2],
-                    /*.nb_d0    =*/ op->nb[0],
-                    /*.nb_d1    =*/ op->nb[1],
-                    /*.nb_d2    =*/ op->nb[2],
-                };
-
-                ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
-                ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(x),        1);
-                ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(residual), 2);
-                ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(post),     3);
-                ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(comb),     4);
-                ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op),       5);
-
-                const int n_tiles = (args.n_embd + 31)/32;
-                const int nsg = std::min(4, n_tiles);
-                ggml_metal_encoder_dispatch_threadgroups(
-                        enc, (n_tiles + nsg - 1)/nsg, args.n_tokens, 1, 32, nsg, 1);
-            } break;
-        default:
-            GGML_ABORT("fatal error");
-    }
-
-    return 1;
-}
-
 int ggml_metal_op_soft_max(ggml_metal_op_t ctx, int idx) {
     ggml_tensor * op = ctx->node(idx);
 
@@ -3408,6 +3197,9 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) {
     GGML_TENSOR_LOCALS( int32_t, ne,  op,         ne);
     GGML_TENSOR_LOCALS(uint64_t, nb,  op,         nb);
 
+    GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32);
+    GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32);
+
     GGML_ASSERT(ggml_is_contiguous_rows(op->src[0]));
     GGML_ASSERT(ggml_is_contiguous_rows(op->src[1]));
 
@@ -3547,36 +3339,6 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) {
     return n_fuse;
 }
 
-int ggml_metal_op_silu_back(ggml_metal_op_t ctx, int idx) {
-    ggml_tensor * op = ctx->node(idx);
-
-    ggml_metal_library_t lib = ctx->lib;
-    ggml_metal_encoder_t enc = ctx->enc;
-
-    auto pipeline = ggml_metal_library_get_pipeline_silu_back(lib, op);
-
-    const int64_t ne = ggml_nelements(op);
-
-    ggml_metal_kargs_silu_back args = {
-        /*.ne =*/ ne,
-    };
-
-    int arg_idx{0};
-
-    ggml_metal_encoder_set_pipeline(enc, pipeline);
-    ggml_metal_encoder_set_bytes   (enc, &args, sizeof(args), arg_idx++);
-    ggml_metal_encoder_set_buffer  (enc, ggml_metal_get_buffer_id(op->src[0]), arg_idx++);
-    ggml_metal_encoder_set_buffer  (enc, ggml_metal_get_buffer_id(op->src[1]), arg_idx++);
-    ggml_metal_encoder_set_buffer  (enc, ggml_metal_get_buffer_id(op), arg_idx++);
-
-    const int nth = std::min<int64_t>(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne);
-    const int64_t n = (ne + nth - 1) / nth;
-
-    ggml_metal_encoder_dispatch_threadgroups(enc, n, 1, 1, nth, 1, 1);
-
-    return 1;
-}
-
 int ggml_metal_op_l2_norm(ggml_metal_op_t ctx, int idx) {
     ggml_tensor * op = ctx->node(idx);
 

ggml/src/ggml-metal/ggml-metal-ops.h

@@ -54,8 +54,6 @@ int ggml_metal_op_cumsum            (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_get_rows          (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_set_rows          (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_diag              (ggml_metal_op_t ctx, int idx);
-int ggml_metal_op_lightning_indexer (ggml_metal_op_t ctx, int idx);
-int ggml_metal_op_dsv4_hc           (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_soft_max          (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_ssm_conv          (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_ssm_scan          (ggml_metal_op_t ctx, int idx);
@@ -72,7 +70,6 @@ int ggml_metal_op_mul_mat_id        (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_add_id            (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_flash_attn_ext    (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_bin               (ggml_metal_op_t ctx, int idx);
-int ggml_metal_op_silu_back         (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_l2_norm           (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_group_norm        (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_norm              (ggml_metal_op_t ctx, int idx);

ggml/src/ggml-metal/ggml-metal.metal

@@ -1255,20 +1255,6 @@ template [[host_name("kernel_unary_f32_f32_4")]] kernel kernel_unary_t kernel_un
 template [[host_name("kernel_unary_f16_f16")]]   kernel kernel_unary_t kernel_unary_impl<half,   half,   float>;
 template [[host_name("kernel_unary_f16_f16_4")]] kernel kernel_unary_t kernel_unary_impl<half4,  half4,  float4>;
 
-kernel void kernel_silu_back_f32(
-        constant ggml_metal_kargs_silu_back & args,
-        device const float * dy,
-        device const float * x,
-        device       float * dx,
-        uint gid [[thread_position_in_grid]]) {
-    if (gid >= args.ne) {
-        return;
-    }
-
-    const float s = 1.0f / (1.0f + exp(-x[gid]));
-    dx[gid] = dy[gid] * s * (1.0f + x[gid] * (1.0f - s));
-}
-
 // OP: 0 - add, 1 - sub, 2 - mul, 3 - div
 constant short FC_bin_op [[function_constant(FC_BIN + 0)]];
 constant short FC_bin_f  [[function_constant(FC_BIN + 1)]];
@@ -1432,8 +1418,6 @@ typedef decltype(kernel_bin_fuse_impl<float, float, float>) kernel_bin_fuse_t;
 
 template [[host_name("kernel_bin_fuse_f32_f32_f32")]]   kernel kernel_bin_fuse_t kernel_bin_fuse_impl<float,  float,  float>;
 template [[host_name("kernel_bin_fuse_f32_f32_f32_4")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl<float4, float4, float4>;
-template [[host_name("kernel_bin_fuse_f16_f16_f16")]]   kernel kernel_bin_fuse_t kernel_bin_fuse_impl<half,   half,   half>;
-template [[host_name("kernel_bin_fuse_f16_f16_f16_4")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl<half4,  half4,  half4>;
 
 kernel void kernel_add_id(
         constant ggml_metal_kargs_add_id & args,
@@ -11294,310 +11278,3 @@ kernel void kernel_count_equal(
 typedef decltype(kernel_count_equal<int32_t>) kernel_count_equal_t;
 
 template [[host_name("kernel_count_equal_i32")]] kernel kernel_count_equal_t kernel_count_equal<int32_t>;
-
-template<
-    typename kd4x4_t,
-    short nl_k,
-    void (*deq_k)(device const kd4x4_t *, short, thread half4x4 &)>
-kernel void kernel_lightning_indexer(
-        constant ggml_metal_kargs_lightning_indexer & args,
-        device const char * q,
-        device const char * k,
-        device const char * w,
-        device const char * m,
-        device       char * dst,
-        uint3  tgpig[[threadgroup_position_in_grid]],
-        ushort tiitg[[thread_index_in_threadgroup]],
-        ushort tiisg[[thread_index_in_simdgroup]],
-        ushort sgitg[[simdgroup_index_in_threadgroup]]) {
-    constexpr short DK    = OP_LIGHTNING_INDEXER_DK;
-    constexpr short NH    = OP_LIGHTNING_INDEXER_NH;
-    constexpr short NHPTG = OP_LIGHTNING_INDEXER_NHPTG;
-    constexpr short NKPSG = OP_LIGHTNING_INDEXER_NKPSG;
-    constexpr short NSG   = OP_LIGHTNING_INDEXER_NSG;
-    constexpr short NBPTG = OP_LIGHTNING_INDEXER_NBPTG;
-
-    constexpr short DK4  = DK/4;
-    constexpr short DK8  = DK/8;
-    constexpr short DK16 = DK/16;
-
-    constexpr short NK  = NKPSG*NSG; // keys    per threadgroup
-    constexpr short NTG = 32*NSG;    // threads per threadgroup
-
-    const int i_stream = tgpig.z;
-    const int i_kv_0   = tgpig.x*NK;            // first key of this threadgroup
-    const int i_kv     = i_kv_0 + sgitg*NKPSG;  // first key of this simdgroup
-
-    threadgroup half4x4   sk4x4[NK*DK16];
-    threadgroup half    * sk = (threadgroup half *) sk4x4;
-
-    for (short i = tiitg; i < NK*DK16; i += NTG) {
-        const short ik  = i/DK16;
-        const short i16 = i%DK16;
-
-        half4x4 tmp;
-
-        if (i_kv_0 + ik < args.n_kv) {
-            device const kd4x4_t * kr = (device const kd4x4_t *) (k + (i_kv_0 + ik)*args.nbk2 + i_stream*args.nbk3);
-
-            deq_k(kr + i16/nl_k, i16%nl_k, tmp);
-        } else {
-            FOR_UNROLL (short j = 0; j < 4; ++j) {
-                tmp[j] = half4(0.0h);
-            }
-        }
-
-        sk4x4[i] = tmp;
-    }
-
-    threadgroup_barrier(mem_flags::mem_threadgroup);
-
-    // K tile of this simdgroup, transposed to [DK, NKPSG]
-    simdgroup_half8x8 mk[DK8];
-
-    FOR_UNROLL (short i = 0; i < DK8; ++i) {
-        simdgroup_load(mk[i], sk + sgitg*NKPSG*DK + 8*i, DK, 0, true);
-    }
-
-    threadgroup half4   sq4[NHPTG*DK4];
-    threadgroup half  * sq = (threadgroup half *) sq4;
-
-    threadgroup float sw [NHPTG];
-    threadgroup float sqk[NSG*NHPTG*NKPSG];
-
-    const int i_batch_0 = tgpig.y*NBPTG;
-    const int n_batch   = min((int) NBPTG, args.n_batch - i_batch_0);
-
-    for (short ib = 0; ib < n_batch; ++ib) {
-        const int i_batch = i_batch_0 + ib;
-
-        device const char * pq = q + i_batch*args.nbq2 + i_stream*args.nbq3;
-        device const char * pw = w + i_batch*args.nbw1 + i_stream*args.nbw3;
-
-        float score = 0.0f;
-
-        FOR_UNROLL (short i_head = 0; i_head < NH; i_head += NHPTG) {
-            // stage the Q tile [DK, NHPTG] and the (prescaled) head weights
-            for (short i = tiitg; i < NHPTG*DK4; i += NTG) {
-                const short ih = i/DK4;
-                const short i4 = i%DK4;
-
-                device const float4 * q4 = (device const float4 *) (pq + (i_head + ih)*args.nbq1);
-
-                sq4[ih*DK4 + i4] = half4(q4[i4]);
-            }
-
-            if (tiitg < NHPTG) {
-                sw[tiitg] = ((device const float *) pw)[i_head + tiitg];
-            }
-
-            threadgroup_barrier(mem_flags::mem_threadgroup);
-
-            simdgroup_float8x8 mqk = make_filled_simdgroup_matrix<float, 8>(0.0f);
-
-            FOR_UNROLL (short i = 0; i < DK8; ++i) {
-                simdgroup_half8x8 mq;
-
-                simdgroup_load(mq, sq + 8*i, DK, 0, false);
-                simdgroup_multiply_accumulate(mqk, mq, mk[i], mqk);
-            }
-
-            threadgroup float * pqk = sqk + sgitg*NHPTG*NKPSG;
-
-            simdgroup_store(mqk, pqk, NKPSG, 0, false);
-            simdgroup_barrier(mem_flags::mem_threadgroup);
-
-            // one lane per key: ReLU, apply the head weight and accumulate over the head tile
-            if (tiisg < NKPSG) {
-                FOR_UNROLL (short ih = 0; ih < NHPTG; ++ih) {
-                    score += max(pqk[ih*NKPSG + tiisg], 0.0f)*sw[ih];
-                }
-            }
-
-            threadgroup_barrier(mem_flags::mem_threadgroup);
-        }
-
-        if (tiisg < NKPSG) {
-            const int ik = i_kv + tiisg;
-            if (ik < args.n_kv) {
-                device const half  * pm = (device const half  *) (m   + i_batch*args.nbm1 + (i_stream % args.mask_ne3)*args.nbm3);
-                device       float * pd = (device       float *) (dst + i_batch*args.nb1  + i_stream*args.nb3);
-
-                pd[ik] = score + (float) pm[ik];
-            }
-        }
-    }
-}
-
-typedef decltype(kernel_lightning_indexer<half4x4, 1, dequantize_f16>) kernel_lightning_indexer_t;
-
-template [[host_name("kernel_lightning_indexer_f32")]]  kernel kernel_lightning_indexer_t kernel_lightning_indexer<float4x4, 1, dequantize_f32>;
-template [[host_name("kernel_lightning_indexer_f16")]]  kernel kernel_lightning_indexer_t kernel_lightning_indexer<half4x4,  1, dequantize_f16>;
-
-#if defined(GGML_METAL_HAS_BF16)
-template [[host_name("kernel_lightning_indexer_bf16")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer<bfloat4x4, 1, dequantize_bf16>;
-#endif
-
-template [[host_name("kernel_lightning_indexer_q4_0")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer<block_q4_0, 2, dequantize_q4_0>;
-template [[host_name("kernel_lightning_indexer_q4_1")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer<block_q4_1, 2, dequantize_q4_1>;
-template [[host_name("kernel_lightning_indexer_q5_0")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer<block_q5_0, 2, dequantize_q5_0>;
-template [[host_name("kernel_lightning_indexer_q5_1")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer<block_q5_1, 2, dequantize_q5_1>;
-template [[host_name("kernel_lightning_indexer_q8_0")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer<block_q8_0, 2, dequantize_q8_0>;
-
-kernel void kernel_dsv4_hc_comb_f32(
-        constant ggml_metal_kargs_dsv4_hc_comb & args,
-        device const char * mixes,
-        device const char * scale,
-        device const char * base,
-        device       char * dst,
-        uint3   tgpig[[threadgroup_position_in_grid]],
-        ushort  tiisg[[thread_index_in_simdgroup]],
-        ushort  sgitg[[simdgroup_index_in_threadgroup]],
-        ushort3   ntg[[threads_per_threadgroup]]) {
-    constexpr ushort hc = 4;
-    constexpr ushort comb_offset = 2*hc;
-
-    const int it = tgpig.x*ntg.y + sgitg;
-    if (it >= args.n_tokens) {
-        return;
-    }
-
-    float scale_lane = 0.0f;
-    if (tiisg == 0) {
-        scale_lane = *(device const float *) (scale + 2*args.nb_s0);
-    }
-    const float scale_comb = simd_shuffle(scale_lane, 0);
-
-    float v = 0.0f;
-    if (tiisg < hc*hc) {
-        v = *(device const float *) (mixes + (comb_offset + tiisg)*args.nb_m0 + it*args.nb_m1)*scale_comb
-          + *(device const float *) (base   + (comb_offset + tiisg)*args.nb_b0);
-    }
-
-    // Softmax across destinations (the four contiguous lanes for each source).
-    float vmax = max(v, simd_shuffle_xor(v, 1));
-    vmax = max(vmax, simd_shuffle_xor(vmax, 2));
-    v = exp(v - vmax);
-
-    float sum = v + simd_shuffle_xor(v, 1);
-    sum += simd_shuffle_xor(sum, 2);
-    v = v/sum + args.eps;
-
-    // Normalize columns: equal destination indices are four lanes apart.
-    sum = v + simd_shuffle_xor(v, 4);
-    sum += simd_shuffle_xor(sum, 8);
-    v /= sum + args.eps;
-
-    for (int i = 1; i < args.n_iter; ++i) {
-        sum = v + simd_shuffle_xor(v, 1);
-        sum += simd_shuffle_xor(sum, 2);
-        v /= sum + args.eps;
-
-        sum = v + simd_shuffle_xor(v, 4);
-        sum += simd_shuffle_xor(sum, 8);
-        v /= sum + args.eps;
-    }
-
-    if (tiisg < hc*hc) {
-        const ushort idst = tiisg & 3;
-        const ushort isrc = tiisg >> 2;
-        *(device float *) (dst + idst*args.nb_d0 + isrc*args.nb_d1 + it*args.nb_d2) = v;
-    }
-}
-
-kernel void kernel_dsv4_hc_pre_f32(
-        constant ggml_metal_kargs_dsv4_hc_pre & args,
-        device const char * x,
-        device const char * weights,
-        device       char * dst,
-        uint3   tgpig[[threadgroup_position_in_grid]],
-        ushort  tiisg[[thread_index_in_simdgroup]],
-        ushort  sgitg[[simdgroup_index_in_threadgroup]],
-        ushort3   ntg[[threads_per_threadgroup]]) {
-    constexpr ushort hc = 4;
-
-    const int it = tgpig.y;
-    const int i0 = ((int) tgpig.x*ntg.y + sgitg)*32 + tiisg;
-
-    float weight_lane = 0.0f;
-    if (tiisg < hc) {
-        weight_lane = *(device const float *) (weights + tiisg*args.nb_w0 + it*args.nb_w1);
-    }
-
-    float w[hc];
-    FOR_UNROLL (ushort ih = 0; ih < hc; ++ih) {
-        w[ih] = simd_shuffle(weight_lane, ih);
-    }
-
-    if (i0 >= args.n_embd) {
-        return;
-    }
-
-    device const char * xb = x + i0*args.nb_x0 + it*args.nb_x2;
-    float result = 0.0f;
-    FOR_UNROLL (ushort ih = 0; ih < hc; ++ih) {
-        result = fma(*(device const float *) (xb + ih*args.nb_x1), w[ih], result);
-    }
-
-    *(device float *) (dst + i0*args.nb_d0 + it*args.nb_d1) = result;
-}
-
-kernel void kernel_dsv4_hc_post_f32(
-        constant ggml_metal_kargs_dsv4_hc_post & args,
-        device const char * x,
-        device const char * residual,
-        device const char * post,
-        device const char * comb,
-        device       char * dst,
-        uint3   tgpig[[threadgroup_position_in_grid]],
-        ushort  tiisg[[thread_index_in_simdgroup]],
-        ushort  sgitg[[simdgroup_index_in_threadgroup]],
-        ushort3   ntg[[threads_per_threadgroup]]) {
-    constexpr ushort hc = 4;
-
-    const int it = tgpig.y;
-    const int i0 = ((int) tgpig.x*ntg.y + sgitg)*32 + tiisg;
-
-    float coeff_lane = 0.0f;
-    if (tiisg < hc) {
-        coeff_lane = *(device const float *) (post + tiisg*args.nb_p0 + it*args.nb_p1);
-    } else if (tiisg < hc + hc*hc) {
-        const ushort idx  = tiisg - hc;
-        const ushort idst = idx & 3;
-        const ushort isrc = idx >> 2;
-        coeff_lane = *(device const float *) (comb + idst*args.nb_c0 + isrc*args.nb_c1 + it*args.nb_c2);
-    }
-
-    float post_reg[hc];
-    float comb_reg[hc][hc];
-    FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) {
-        post_reg[idst] = simd_shuffle(coeff_lane, idst);
-    }
-    FOR_UNROLL (ushort isrc = 0; isrc < hc; ++isrc) {
-        FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) {
-            comb_reg[isrc][idst] = simd_shuffle(coeff_lane, hc + idst + hc*isrc);
-        }
-    }
-
-    if (i0 >= args.n_embd) {
-        return;
-    }
-
-    const float xv = *(device const float *) (x + i0*args.nb_x0 + it*args.nb_x1);
-    float result[hc];
-    FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) {
-        result[idst] = xv*post_reg[idst];
-    }
-
-    device const char * rb = residual + i0*args.nb_r0 + it*args.nb_r2;
-    FOR_UNROLL (ushort isrc = 0; isrc < hc; ++isrc) {
-        const float rv = *(device const float *) (rb + isrc*args.nb_r1);
-        FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) {
-            result[idst] = fma(rv, comb_reg[isrc][idst], result[idst]);
-        }
-    }
-
-    FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) {
-        *(device float *) (dst + i0*args.nb_d0 + idst*args.nb_d1 + it*args.nb_d2) = result[idst];
-    }
-}

ggml/src/ggml-opencl/ggml-opencl.cpp

@@ -7065,7 +7065,7 @@ static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_tensor *tensor) {
     return tensor->ne[1] >= 32768 && tensor->ne[2] == 1 && tensor->ne[3] == 1;
 }
 
-static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
+static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_tensor *tensor) {
     // gemv_noshuffle variant perf drops for large M, use flat variant for large M.
     // threshold is well above typical hidden/FFN dims, but below typical vocab sizes.
     // q6_K flat gemv is worse for smaller K; 2048 seems to be a reasonable threshold.
@@ -7083,15 +7083,7 @@ static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_backend_opencl_cont
     if ((tensor->ne[1] % 128 != 0) && tensor->ne[2] == 1 && tensor->ne[3] == 1) {
         return true;
     }
-
-    // The gemv_noshuffle slowdown tracks TOTAL weight size, not ne0 alone; ne0 >= 2048 is a
-    // proxy for "large weight" that misses a narrow-hidden vocab-scale lm_head.
-    // Add a direct size escape so such weights also take the flat path, without changing
-    // which weights ne0 >= 2048 already routes there.
-    // The size escape is not taken on the A7X since its compiler miscompiles the flat K-quant GEMV
-    return tensor->ne[1] >= 32768
-        && (tensor->ne[0] >= 2048 || (backend_ctx->adreno_gen != ADRENO_GPU_GEN::A7X && ggml_nbytes(tensor) >= (256ull << 20)))
-        && tensor->ne[2] == 1 && tensor->ne[3] == 1;
+    return tensor->ne[1] >= 32768 && tensor->ne[0] >= 2048 && tensor->ne[2] == 1 && tensor->ne[3] == 1;
 }
 
 static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) {
@@ -7503,7 +7495,6 @@ static ggml_backend_i ggml_backend_opencl_i = {
 ggml_backend_t ggml_backend_opencl_init(void) {
     ggml_backend_dev_t dev = ggml_backend_reg_dev_get(ggml_backend_opencl_reg(), 0);
     ggml_backend_opencl_context *backend_ctx = ggml_cl_init(dev);
-    backend_ctx->ref_count++;
 
     ggml_backend_t backend = new ggml_backend {
         /* .guid    = */ ggml_backend_opencl_guid(),
@@ -9411,7 +9402,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
         cl_kernel kernel;
 #ifdef GGML_OPENCL_USE_ADRENO_KERNELS
         kernel = backend_ctx->kernel_convert_block_q6_K;
-        if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor)) {
+        if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(tensor)) {
             kernel = backend_ctx->kernel_convert_block_q6_K_noshuffle;
         }
 #else
@@ -9444,7 +9435,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
         tensor->extra  = extra;
 
 #ifdef GGML_OPENCL_USE_ADRENO_KERNELS
-        if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor)) {
+        if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(tensor)) {
             cl_int M = tensor->ne[1];   // ne01
             cl_int K = tensor->ne[0];   // ne00
 
@@ -10481,7 +10472,7 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer,
             CL_CHECK(clReleaseMemObject(data_device));
             return;
         }
-        if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor)) {
+        if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(tensor)) {
             static ggml_cl_buffer buf_trans_ql;
             static ggml_cl_buffer buf_trans_qh;
             static ggml_cl_buffer buf_trans_s;
@@ -18903,7 +18894,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
         }
 
         // q6_K x fp32
-        if (src0t == GGML_TYPE_Q6_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q6_K(backend_ctx, src0)) {
+        if (src0t == GGML_TYPE_Q6_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q6_K(src0)) {
             ggml_cl_mul_mat_q6_K_f32_adreno(backend, src0, src1, dst);
             return;
         }
@@ -24268,7 +24259,7 @@ static void ggml_cl_glu(ggml_backend_t backend, const ggml_tensor * src0, const
     }
 
     const size_t nrows = ggml_nrows(src0);
-    size_t nth = backend_ctx->max_workgroup_size < 512 ? backend_ctx->max_workgroup_size : 512;
+    size_t nth = 512;
     size_t global_work_size[] = {nrows*nth, 1, 1};
     size_t local_work_size[] = {nth, 1, 1};
 

ggml/src/ggml-sycl/common.hpp

@@ -235,7 +235,6 @@ struct sycl_device_info {
     int max_wg_per_cu; // max work groups per compute unit - refer to
                        // cudaOccupancyMaxActiveBlocksPerMultiprocessor
     bool    vmm;                // virtual memory support
-    bool    l0_device_type_valid;
     bool    l0_discrete_gpu;    // Level Zero backend and not an integrated GPU
     size_t  vmm_granularity;    // granularity of virtual memory
     size_t  total_vram;

ggml/src/ggml-sycl/concat.cpp

@@ -127,15 +127,7 @@ static void concat_T_sycl_non_cont(
     int64_t ne2, int64_t ne3, uint64_t nb0, uint64_t nb1, uint64_t nb2,
     uint64_t nb3, int32_t dim) {
   sycl::range<3> gridDim(ne3, ne2, ne1);
-
-  // Avoid oversubscribing device when there is not enough elements along the innermost dim to
-  // fill a full SYCL_CONCAT_BLOCK_SIZE. For larger # of elements, the full SYCL_CONCAT_BLOCK_SIZE
-  // is used.
-  const int64_t ne0_pad   = GGML_PAD(ne0, WARP_SIZE);
-  const int64_t block_ne0 = ne0_pad < SYCL_CONCAT_BLOCK_SIZE ? ne0_pad : (int64_t) SYCL_CONCAT_BLOCK_SIZE;
-  sycl::range<3> blockDim(1, 1, block_ne0);
-
-  stream->parallel_for(sycl::nd_range<3>(gridDim * blockDim, blockDim), [=](sycl::nd_item<3> item_ct1) {
+  stream->parallel_for(sycl::nd_range<3>(gridDim, sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) {
       int64_t i3 = item_ct1.get_group(0);
       int64_t i2 = item_ct1.get_group(1);
       int64_t i1 = item_ct1.get_group(2);

ggml/src/ggml-sycl/fattn-onednn.cpp

@@ -2,13 +2,11 @@
 #include <cstdio>
 #include <cstring>
 #include <string>
-#include <optional>
 #include <unordered_map>
 #include <vector>
 
 #include "fattn-onednn.hpp"
 #include "fattn-tile.hpp"
-#include "convert.hpp"
 
 // set minimum query length to treat as prefill (32)
 #define GGML_SYCL_FA_ONEDNN_MIN_Q 32
@@ -35,30 +33,10 @@ bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst) {
     const ggml_tensor * mask  = dst->src[3];
     const ggml_tensor * sinks = dst->src[4];
 
-    // F16 KV: native SDPA at any KV length.
-    // Non-F16: dequant to F16 then SDPA at prefill lengths. Only the
-    // standard quantized KV cache types (Q4_0-Q8_0) and F32 are accepted
-    // because their to_fp16_sycl conversion is verified. BF16 and IQ*
-    // are excluded: BF16 needs a strided conversion kernel that does not
-    // exist yet; IQ types are model-weight-only quants with no dequant
-    // registration and are never used as KV caches.
+    // gate for f16 KV only for now
+    // need to implement quantized KV
     if (K->type != GGML_TYPE_F16 || V->type != GGML_TYPE_F16) {
-        auto kt = K->type, vt = V->type;
-        bool k_ok = kt == GGML_TYPE_F32 || kt == GGML_TYPE_Q4_0 || kt == GGML_TYPE_Q4_1 ||
-                    kt == GGML_TYPE_Q5_0 || kt == GGML_TYPE_Q5_1 || kt == GGML_TYPE_Q8_0;
-        bool v_ok = vt == GGML_TYPE_F32 || vt == GGML_TYPE_Q4_0 || vt == GGML_TYPE_Q4_1 ||
-                    vt == GGML_TYPE_Q5_0 || vt == GGML_TYPE_Q5_1 || vt == GGML_TYPE_Q8_0;
-        if (!k_ok || !v_ok) {
-            return false;
-        }
-        if (Q->ne[1] < 32 || K->ne[1] < 1024) {
-            return false;
-        }
-        for (const ggml_tensor * t : {K, V}) {
-            if (t->type == GGML_TYPE_F16 && t->nb[1] % (t->ne[0] * 2) != 0) {
-                return false;
-            }
-        }
+        return false;
     }
     // Optional KV-length ceiling (GGML_SYCL_FA_ONEDNN_MAX_KV, 0 = unlimited). Escape hatch:
     // very long sequences make the fused SDPA slow enough to risk the xe driver watchdog on
@@ -227,101 +205,13 @@ void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tenso
     dnnl::engine    eng    = ctx.engine_dnnl(stream);
     dnnl::stream    strm   = ctx.stream_dnnl(stream);
 
-    // Q: always f32 -- copy to dense f16.
-    ggml_sycl_pool_alloc<sycl::half> Qf(ctx.pool(), (size_t) H * q * d);
-    cont_to_f16_sycl<float>((const char *) Q->data, Qf.get(), d, q, H, mb, Q->nb[1], Q->nb[2], Q->nb[3], stream);
-
-    // K/V: use pool-alloc for both F16 and dequant paths.
-    sycl::half * K_ptr = nullptr;
-    sycl::half * V_ptr = nullptr;
-    std::optional<ggml_sycl_pool_alloc<sycl::half>> Kf_pool;
-    std::optional<ggml_sycl_pool_alloc<sycl::half>> Vf_pool;
-
-    if (K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16) {
-        Kf_pool.emplace(ctx.pool(), (size_t) Hkv * seq * d);
-        Vf_pool.emplace(ctx.pool(), (size_t) Hkv * seq * d);
-        cont_to_f16_sycl<sycl::half>((const char *) K->data, Kf_pool->get(), d, seq, Hkv, mb, K->nb[1], K->nb[2], K->nb[3], stream);
-        cont_to_f16_sycl<sycl::half>((const char *) V->data, Vf_pool->get(), d, seq, Hkv, mb, V->nb[1], V->nb[2], V->nb[3], stream);
-        K_ptr = Kf_pool->get();
-        V_ptr = Vf_pool->get();
-    } else if (ggml_is_quantized(K->type)) {
-        // Quantized K/V: dequant to dense F16 using pool, same lifetime as F16 path.
-        Kf_pool.emplace(ctx.pool(), ggml_nelements(K));
-        K_ptr = Kf_pool->get();
-        {
-            const char * K_data = (const char *)K->data;
-            const bool k_non_dense = ((int64_t)K->ne[1] * K->nb[1] != K->nb[2]) && K->ne[2] > 1;
-            const bool k_gemma = k_non_dense &&
-                ((int64_t)K->nb[2] < (int64_t)K->ne[1] * (int64_t)K->nb[1]);
-            if (ggml_is_contiguously_allocated(K) && !k_non_dense) {
-                to_fp16_sycl_t to_fp16 = ggml_get_to_fp16_sycl(K->type, dst);
-                to_fp16(K_data, K_ptr, ggml_nelements(K), stream);
-            } else {
-                const size_t bs = ggml_blck_size(K->type);
-                const size_t ts = ggml_type_size(K->type);
-                to_fp16_nc_sycl_t to_fp16 = ggml_get_to_fp16_nc_sycl(K->type);
-                int64_t s01, s02, s03;
-                if (k_gemma) {
-                    const int64_t blk_per_row = (int64_t)K->ne[0] / bs;
-                    s01 = (int64_t)Hkv * blk_per_row;
-                    s02 = blk_per_row;
-                    s03 = (int64_t)K->ne[1] * s01;
-                } else {
-                    s01 = (int64_t)K->nb[1] / ts;
-                    s02 = (int64_t)K->nb[2] / ts;
-                    s03 = (int64_t)K->nb[3] / ts;
-                }
-                to_fp16(K_data, K_ptr,
-                        K->ne[0], K->ne[1], K->ne[2], K->ne[3],
-                        s01, s02, s03, stream);
-            }
-        }
-        // Quantized V: always dequant separately. Even when K and V share
-        // the same underlying allocation (V is a view of K with the same
-        // data pointer), their logical values differ because the quantized
-        // elements at different positions/offsets represent different K/V
-        // data. Master's F16 path also never aliases K and V.
-        Vf_pool.emplace(ctx.pool(), ggml_nelements(V));
-        V_ptr = Vf_pool->get();
-        {
-            const char * V_data = (const char *)V->data;
-            const bool v_non_dense = ((int64_t)V->ne[1] * V->nb[1] != V->nb[2]) && V->ne[2] > 1;
-            const bool v_gemma = v_non_dense &&
-                ((int64_t)V->nb[2] < (int64_t)V->ne[1] * (int64_t)V->nb[1]);
-            if (ggml_is_contiguously_allocated(V) && !v_non_dense) {
-                to_fp16_sycl_t to_fp16 = ggml_get_to_fp16_sycl(V->type, dst);
-                to_fp16(V_data, V_ptr, ggml_nelements(V), stream);
-            } else {
-                const size_t bs = ggml_blck_size(V->type);
-                const size_t ts = ggml_type_size(V->type);
-                to_fp16_nc_sycl_t to_fp16 = ggml_get_to_fp16_nc_sycl(V->type);
-                int64_t s01, s02, s03;
-                if (v_gemma) {
-                    const int64_t blk_per_row = (int64_t)V->ne[0] / bs;
-                    s01 = (int64_t)V->ne[2] * blk_per_row;
-                    s02 = blk_per_row;
-                    s03 = (int64_t)V->ne[1] * s01;
-                } else {
-                    s01 = (int64_t)V->nb[1] / ts;
-                    s02 = (int64_t)V->nb[2] / ts;
-                    s03 = (int64_t)V->nb[3] / ts;
-                }
-                to_fp16(V_data, V_ptr,
-                        V->ne[0], V->ne[1], V->ne[2], V->ne[3],
-                        s01, s02, s03, stream);
-            }
-        }
-    } else {
-        // F32: strided copy to dense F16 via cont_to_f16_sycl<float>.
-        Kf_pool.emplace(ctx.pool(), ggml_nelements(K));
-        K_ptr = Kf_pool->get();
-        cont_to_f16_sycl<float>((const char *) K->data, K_ptr, K->ne[0], K->ne[1], K->ne[2], K->ne[3],
-                                K->nb[1], K->nb[2], K->nb[3], stream);
-        Vf_pool.emplace(ctx.pool(), ggml_nelements(V));
-        V_ptr = Vf_pool->get();
-        cont_to_f16_sycl<float>((const char *) V->data, V_ptr, V->ne[0], V->ne[1], V->ne[2], V->ne[3],
-                                V->nb[1], V->nb[2], V->nb[3], stream);
-    }
+    // cont/cast inputs to contiguous f16 (head-major) -- the layout the fast systolic path wants.
+    ggml_sycl_pool_alloc<sycl::half> Qf(ctx.pool(), (size_t) H   * q   * d);
+    ggml_sycl_pool_alloc<sycl::half> Kf(ctx.pool(), (size_t) Hkv * seq * d);
+    ggml_sycl_pool_alloc<sycl::half> Vf(ctx.pool(), (size_t) Hkv * seq * d);
+    cont_to_f16_sycl<float>     ((const char *) Q->data, Qf.get(), d, q,   H,   mb, Q->nb[1], Q->nb[2], Q->nb[3], stream);
+    cont_to_f16_sycl<sycl::half>((const char *) K->data, Kf.get(), d, seq, Hkv, mb, K->nb[1], K->nb[2], K->nb[3], stream);
+    cont_to_f16_sycl<sycl::half>((const char *) V->data, Vf.get(), d, seq, Hkv, mb, V->nb[1], V->nb[2], V->nb[3], stream);
 
     // divide-by-(1/scale) reproduces ggml's score *= kq_scale on the proven probe graph.
     //
@@ -354,8 +244,8 @@ void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tenso
 
     auto id2ptr = [&](size_t r) -> void * {
         if (r == E.id_q)     return Qf.get();
-        if (r == E.id_k)     return K_ptr;
-        if (r == E.id_v)     return V_ptr;
+        if (r == E.id_k)     return Kf.get();
+        if (r == E.id_v)     return Vf.get();
         if (r == E.id_scale) return scale_dev;
         if (r == E.id_mask)  return (void *) mask->data;
         return nullptr;

ggml/src/ggml-sycl/fattn.cpp

@@ -97,7 +97,7 @@ static void ggml_sycl_flash_attn_ext_vec(ggml_backend_sycl_context & ctx, ggml_t
 enum best_fattn_kernel {
     BEST_FATTN_KERNEL_NONE     =   0,
     BEST_FATTN_KERNEL_VEC      = 100,
-    BEST_FATTN_KERNEL_ONEDNN   = 150, // oneDNN SDPA: native F16 (PR #25222)
+    BEST_FATTN_KERNEL_ONEDNN   = 150, // added enum for onednn==150
     BEST_FATTN_KERNEL_TILE     = 200,
     BEST_FATTN_KERNEL_MKL      = 300,
 };
@@ -130,14 +130,6 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const
 
     bool gqa_opt_applies = gqa_ratio >= 2 && mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0;
 
-    // XMX-accelerated path: oneDNN SDPA (native F16 and dequant+non-F16).
-    // ONEDNN requires min 32 query tokens — short-circuit decode to avoid
-    // calling _supported() on every decode FA call.
-    if (Q->ne[1] >= 32
-        && ggml_sycl_flash_attn_ext_onednn_supported(dst)) {
-        return BEST_FATTN_KERNEL_ONEDNN;
-    }
-
     // MKL path: XMX-accelerated GEMM for prompt processing (all KV cache types).
     // The MKL kernel converts non-F16 K/V to F16 via to_fp16_sycl before GEMM,
     // so quantized, F16, BF16, and F32 caches all benefit from XMX acceleration.
@@ -175,6 +167,7 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const
             return BEST_FATTN_KERNEL_MKL;
         }
     }
+
     for (const ggml_tensor * t : {Q, K, V, mask}) {
         if (t == nullptr || ggml_is_quantized(t->type)) {
             continue;
@@ -222,7 +215,6 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const
     switch (K->type) {
         case GGML_TYPE_F32:
         case GGML_TYPE_F16:
-        case GGML_TYPE_BF16:
             break;
         case GGML_TYPE_Q4_1:
         case GGML_TYPE_Q5_0:
@@ -241,11 +233,8 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const
         return BEST_FATTN_KERNEL_NONE;
     }
 
-    // For small batch sizes the vector kernel may be preferable over the kernels optimized for large batch sizes.
-    // BF16 is excluded: the VEC kernel has no BF16 template (it needs GGML_SYCL_FA_ALL_QUANTS for non-F16/Q4_0/Q8_0).
-    const bool has_bf16 = (K->type == GGML_TYPE_BF16 || V->type == GGML_TYPE_BF16);
-    const bool can_use_vector_kernel = Q->ne[0] <= 512 && Q->ne[0] % 64 == 0 && K->ne[1] % FATTN_KQ_STRIDE == 0
-        && !has_bf16;
+    // For small batch sizes the vector kernel may be preferable over the kernels optimized for large batch sizes:
+    const bool can_use_vector_kernel = Q->ne[0] <= 512 && Q->ne[0] % 64 == 0 && K->ne[1] % FATTN_KQ_STRIDE == 0;
 
     // Fused-XMX path: oneDNN Graph SDPA (flash attention). Strictly
     // additive -- taken only when statically supported, otherwise falls through to VEC/TILE below.
@@ -287,7 +276,6 @@ void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst
         const char * kname = "TILE";
         best_fattn_kernel k = ggml_sycl_get_best_fattn_kernel(ctx.device, dst);
         if (k == BEST_FATTN_KERNEL_MKL)  kname = "MKL";
-        if (k == BEST_FATTN_KERNEL_ONEDNN)  kname = "ONEDNN";
         if (k == BEST_FATTN_KERNEL_VEC)  kname = "VEC";
         int64_t delta = 0;
         if (Dk == 256) {
@@ -304,8 +292,7 @@ void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst
                 (long long)V_dbg->ne[1]);
     }
 
-    const best_fattn_kernel fk = ggml_sycl_get_best_fattn_kernel(ggml_sycl_get_device(), dst);
-    switch (fk) {
+    switch (ggml_sycl_get_best_fattn_kernel(ggml_sycl_get_device(), dst)) {
         case BEST_FATTN_KERNEL_NONE:
             GGML_ABORT("Not support Flash-Attention");
         case BEST_FATTN_KERNEL_ONEDNN:
@@ -344,7 +331,6 @@ void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst
             q->wait();
             const char * kname = "???";
             best_fattn_kernel kb = ggml_sycl_get_best_fattn_kernel(ctx.device, dst);
-            if (kb == BEST_FATTN_KERNEL_ONEDNN) kname = "ONEDNN";
             if (kb == BEST_FATTN_KERNEL_MKL) kname = "MKL";
             if (kb == BEST_FATTN_KERNEL_TILE) kname = "TILE";
             if (kb == BEST_FATTN_KERNEL_VEC) kname = "VEC";
@@ -368,7 +354,6 @@ void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst
             }
         }
     }
-
 }
 
 bool ggml_sycl_flash_attn_ext_supported(int device, const ggml_tensor * dst) {

ggml/src/ggml-sycl/ggml-sycl.cpp

@@ -167,10 +167,7 @@ static ggml_sycl_device_info ggml_sycl_init() {
             ze_device_properties_t props = {};
             props.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES;
             ze_result_t r = zeDeviceGetProperties(ze_dev, &props);
-            if (r == ZE_RESULT_SUCCESS) {
-                info.devices[i].l0_device_type_valid = true;
-                info.devices[i].l0_discrete_gpu = !(props.flags & ZE_DEVICE_PROPERTY_FLAG_INTEGRATED);
-            }
+            info.devices[i].l0_discrete_gpu = r == ZE_RESULT_SUCCESS && !(props.flags & ZE_DEVICE_PROPERTY_FLAG_INTEGRATED);
         }
 #endif
     }
@@ -5609,11 +5606,7 @@ static void ggml_backend_sycl_device_get_memory(ggml_backend_dev_t dev, size_t *
 }
 
 static enum ggml_backend_dev_type ggml_backend_sycl_device_get_type(ggml_backend_dev_t dev) {
-    ggml_backend_sycl_device_context * ctx = (ggml_backend_sycl_device_context *)dev->context;
-    const sycl_device_info & info = ggml_sycl_info().devices[ctx->device];
-    if (info.l0_device_type_valid && !info.l0_discrete_gpu) {
-        return GGML_BACKEND_DEVICE_TYPE_IGPU;
-    }
+    GGML_UNUSED(dev);
     return GGML_BACKEND_DEVICE_TYPE_GPU;
 }
 

ggml/src/ggml-vulkan/ggml-vulkan.cpp

@@ -186,13 +186,22 @@ static bool is_pow2(uint32_t x) { return x > 1 && (x & (x-1)) == 0; }
 
 #define VK_DEVICE_DESCRIPTOR_POOL_SIZE 256
 
-#define VK_CHECK(err, msg)                                          \
+#define VK_CHECK(err, msg, dev)                                     \
     do {                                                            \
-        vk::Result err_ = (err);                                    \
+        vk::Result err_;                                            \
+        try {                                                       \
+            err_ = (err);                                           \
+        } catch (vk::DeviceLostError &) {                           \
+            ggml_vk_print_device_lost_info(dev);                    \
+            GGML_LOG_ERROR("ggml_vulkan: %s at %s:%d\n",            \
+                #err, __FILE__, __LINE__);                          \
+            throw;                                                  \
+        }                                                           \
         if (err_ != vk::Result::eSuccess) {                         \
-            fprintf(stderr, "ggml_vulkan: %s error %s at %s:%d\n",  \
+            GGML_LOG_ERROR("ggml_vulkan: %s error %s at %s:%d\n",   \
                 #err, to_string(err_).c_str(), __FILE__, __LINE__); \
-            exit(1);                                                \
+            throw vk::SystemError(vk::make_error_code(err_),        \
+                "ggml_vulkan: " msg);                               \
         }                                                           \
     } while (0)
 
@@ -302,9 +311,13 @@ struct vk_command_pool {
     }
 };
 
+static void ggml_vk_print_device_fault_info(const vk_device& device);
+static void ggml_vk_print_device_lost_info(const vk_device& device);
+
 // Prevent simultaneous submissions to the same queue.
 struct vk_queue_handle {
     vk::Queue queue;
+    vk_device_ref device;
     virtual void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) = 0;
     virtual void lock()   {}   // no-op by default (internally synchronized case)
     virtual void unlock() {}
@@ -315,7 +328,14 @@ struct vk_queue_handle_synchronized : vk_queue_handle {
     std::mutex mutex;
     void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) override {
         std::lock_guard<std::mutex> guard(mutex);
-        queue.submit(submits, fence);
+        try {
+            queue.submit(submits, fence);
+        } catch (vk::DeviceLostError &) {
+            if (auto dev = device.lock()) {
+                ggml_vk_print_device_lost_info(dev);
+            }
+            throw;
+        }
     }
     void lock()   override { mutex.lock(); }
     void unlock() override { mutex.unlock(); }
@@ -324,7 +344,14 @@ struct vk_queue_handle_synchronized : vk_queue_handle {
 struct vk_queue_handle_unsynchronized : vk_queue_handle {
     void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) override {
         // Driver guarantees internal synchronization via VK_KHR_internally_synchronized_queues
-        queue.submit(submits, fence);
+        try {
+            queue.submit(submits, fence);
+        } catch (vk::DeviceLostError &) {
+            if (auto dev = device.lock()) {
+                ggml_vk_print_device_lost_info(dev);
+            }
+            throw;
+        }
     }
     // lock()/unlock() inherited no-ops
 };
@@ -610,13 +637,6 @@ static constexpr std::initializer_list<ggml_op> topk_moe_sigmoid_norm_bias{ GGML
                                                                             GGML_OP_RESHAPE,  GGML_OP_SUM_ROWS, GGML_OP_CLAMP,
                                                                             GGML_OP_DIV,      GGML_OP_RESHAPE };
 
-static constexpr std::initializer_list<ggml_op> topk_moe_sqrt_softplus_norm_bias{ GGML_OP_UNARY,    GGML_OP_SQRT,
-                                                                                  GGML_OP_RESHAPE,  GGML_OP_ADD,
-                                                                                  GGML_OP_ARGSORT,  GGML_OP_VIEW,
-                                                                                  GGML_OP_GET_ROWS, GGML_OP_RESHAPE,
-                                                                                  GGML_OP_SUM_ROWS, GGML_OP_CLAMP,
-                                                                                  GGML_OP_DIV,      GGML_OP_RESHAPE };
-
 static constexpr std::initializer_list<ggml_op> topk_moe_early_softmax     { GGML_OP_SOFT_MAX, GGML_OP_RESHAPE,  GGML_OP_ARGSORT,
                                                                              GGML_OP_VIEW,     GGML_OP_GET_ROWS };
 
@@ -680,22 +700,6 @@ static constexpr std::initializer_list<std::array<int, 3>> topk_moe_sigmoid_norm
     {10, 0, 9 }, // reshape->src[0]  == div
 };
 
-static constexpr std::initializer_list<std::array<int, 3>> topk_moe_sqrt_softplus_norm_bias_edges {
-    { 1, 0, 0 }, // sqrt->src[0]     == softplus
-    { 2, 0, 1 }, // reshape->src[0]  == sqrt
-    { 3, 0, 1 }, // add->src[0]      == sqrt
-    { 4, 0, 3 }, // argsort->src[0]  == add
-    { 5, 0, 4 }, // view->src[0]     == argsort
-    { 6, 0, 2 }, // get_rows->src[0] == reshape
-    { 6, 1, 5 }, // get_rows->src[1] == view
-    { 7, 0, 6 }, // reshape->src[0]  == get_rows
-    { 8, 0, 7 }, // sum_rows->src[0] == reshape
-    { 9, 0, 8 }, // clamp->src[0]    == sum_rows
-    {10, 0, 7 }, // div->src[0]      == reshape
-    {10, 1, 9 }, // div->src[1]      == clamp
-    {11, 0,10 }, // reshape->src[0]  == div
-};
-
 // same as early_softmax_norm but ending after the get_rows
 static constexpr std::initializer_list<std::array<int, 3>> topk_moe_early_softmax_edges {
     { 1, 0, 0 }, // reshape->src[0]  == softmax
@@ -724,7 +728,6 @@ enum topk_moe_mode {
     TOPK_MOE_EARLY_SOFTMAX_NORM,
     TOPK_MOE_LATE_SOFTMAX,
     TOPK_MOE_SIGMOID_NORM_BIAS,
-    TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS,
     TOPK_MOE_COUNT,
 };
 
@@ -835,6 +838,15 @@ struct vk_device_struct {
 
     bool pipeline_executable_properties_support {};
 
+    bool device_fault {};
+    PFN_vkGetDeviceFaultInfoEXT pfn_vkGetDeviceFaultInfoEXT {};
+
+    bool serialize_submissions {};
+
+    const ggml_cgraph * diag_cgraph {};
+    int diag_prev_start = -1;
+    int diag_prev_end = -1;
+
     size_t idx;
 
     bool mul_mat_l[GGML_TYPE_COUNT];
@@ -1022,11 +1034,9 @@ struct vk_device_struct {
     vk_pipeline pipeline_snake_f32;
     vk_pipeline pipeline_snake_f16;
     vk_pipeline pipeline_snake_bf16;
-    vk_pipeline pipeline_pool1d_f32;
     vk_pipeline pipeline_pool2d_f32;
     vk_pipeline pipeline_rwkv_wkv6_f32;
     vk_pipeline pipeline_rwkv_wkv7_f32;
-    vk_pipeline pipeline_gated_linear_attn_f32;
     // [size_idx][kda] where size_idx: 0=d16, 1=d32, 2=d64, 3=d128
     vk_pipeline pipeline_gated_delta_net[4][2];
     vk_pipeline pipeline_ssm_scan_f32_d128;
@@ -1118,6 +1128,57 @@ void vk_command_pool::destroy(vk::Device& device) {
     cmd_buffers.clear();
 }
 
+static void ggml_vk_print_device_fault_info(const vk_device& device) {
+    if (!device->device_fault || !device->pfn_vkGetDeviceFaultInfoEXT) {
+        return;
+    }
+
+    VkDeviceFaultCountsEXT fault_counts {};
+    fault_counts.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_COUNTS_EXT;
+    VkResult res = device->pfn_vkGetDeviceFaultInfoEXT(device->device, &fault_counts, nullptr);
+    if (res != VK_SUCCESS) {
+        GGML_LOG_ERROR("ggml_vulkan: vkGetDeviceFaultInfoEXT (counts) failed: %d\n", res);
+        return;
+    }
+
+    std::vector<VkDeviceFaultAddressInfoEXT> address_infos(fault_counts.addressInfoCount);
+    std::vector<VkDeviceFaultVendorInfoEXT> vendor_infos(fault_counts.vendorInfoCount);
+
+    VkDeviceFaultInfoEXT fault_info {};
+    fault_info.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_INFO_EXT;
+    fault_info.pAddressInfos = address_infos.data();
+    fault_info.pVendorInfos = vendor_infos.data();
+
+    res = device->pfn_vkGetDeviceFaultInfoEXT(device->device, &fault_counts, &fault_info);
+    if (res != VK_SUCCESS) {
+        GGML_LOG_ERROR("ggml_vulkan: vkGetDeviceFaultInfoEXT (info) failed: %d\n", res);
+        return;
+    }
+
+    if (fault_counts.addressInfoCount == 0 && fault_counts.vendorInfoCount == 0 && fault_info.description[0] == '\0') {
+        return;
+    }
+
+    if (fault_info.description[0] != '\0') {
+        GGML_LOG_ERROR("ggml_vulkan: device fault on %s: %s\n", device->name.c_str(), fault_info.description);
+    }
+
+    for (uint32_t i = 0; i < fault_counts.addressInfoCount; i++) {
+        const auto& info = address_infos[i];
+        GGML_LOG_CONT("  address fault %u: type=%d address=0x%llx precision=0x%llx\n",
+                i, (int)info.addressType,
+                (unsigned long long)info.reportedAddress,
+                (unsigned long long)info.addressPrecision);
+    }
+    for (uint32_t i = 0; i < fault_counts.vendorInfoCount; i++) {
+        const auto& info = vendor_infos[i];
+        GGML_LOG_CONT("  vendor fault %u: %s (code=0x%llx data=0x%llx)\n",
+                i, info.description,
+                (unsigned long long)info.vendorFaultCode,
+                (unsigned long long)info.vendorFaultData);
+    }
+}
+
 struct vk_buffer_struct {
     vk::Buffer buffer = VK_NULL_HANDLE;
     vk::DeviceMemory device_memory = VK_NULL_HANDLE;
@@ -1713,17 +1774,6 @@ struct vk_op_snake_push_constants {
     uint32_t ne1;
 };
 
-struct vk_op_pool1d_push_constants {
-    uint32_t IL;
-    uint32_t OL;
-    uint32_t OC;
-    uint32_t pelements;
-    uint32_t op;
-    int32_t k0;
-    int32_t s0;
-    int32_t p0;
-};
-
 struct vk_op_pool2d_push_constants {
     uint32_t IW; uint32_t IH;
     uint32_t OW; uint32_t OH;
@@ -1748,13 +1798,6 @@ struct vk_op_rwkv_wkv7_push_constants {
     uint32_t C;
     uint32_t H;
 };
-struct vk_op_gated_linear_attn_push_constants {
-    uint32_t B;
-    uint32_t T;
-    uint32_t C;
-    uint32_t H;
-    float scale;
-};
 struct vk_op_gated_delta_net_push_constants {
     uint32_t H;
     uint32_t n_tokens;
@@ -1978,7 +2021,6 @@ struct ggml_vk_garbage_collector {
 static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_context subctx);
 static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested = nullptr);
 static void ggml_pipeline_allocate_descriptor_sets(ggml_backend_vk_context * ctx);
-static bool ggml_vk_intel_windows_driver_equals_or_newer_than(uint32_t driver_version, uint32_t threshold_major, uint32_t threshold_minor);
 
 static bool vk_memory_logger_enabled = false;
 
@@ -2059,6 +2101,36 @@ static uint64_t ggml_vk_get_node_flops(const ggml_tensor * node) {
     return 0;
 }
 
+static void ggml_vk_print_node_list(const ggml_cgraph * cgraph, int start, int end) {
+    uint64_t total_flops = 0;
+    int n_ops = 0;
+    for (int j = start; j <= end && j < cgraph->n_nodes; j++) {
+        uint64_t flops = ggml_vk_get_node_flops(cgraph->nodes[j]);
+        total_flops += flops;
+        n_ops++;
+        if (flops > 0) {
+            GGML_LOG_CONT("  node %d: %s (%s) [%.2f GFLOP]\n",
+                    j, cgraph->nodes[j]->name, ggml_op_name(cgraph->nodes[j]->op),
+                    flops / 1e9);
+        } else {
+            GGML_LOG_CONT("  node %d: %s (%s)\n",
+                    j, cgraph->nodes[j]->name, ggml_op_name(cgraph->nodes[j]->op));
+        }
+    }
+    GGML_LOG_CONT("  total: %d ops, %.2f GFLOP\n", n_ops, total_flops / 1e9);
+}
+
+static void ggml_vk_print_device_lost_info(const vk_device& device) {
+    ggml_vk_print_device_fault_info(device);
+    if (device->serialize_submissions && device->diag_cgraph != nullptr && device->diag_prev_start >= 0) {
+        GGML_LOG_ERROR("ggml_vulkan: device lost on %s, likely caused by previous submission (nodes %d to %d):\n",
+                device->name.c_str(), device->diag_prev_start, device->diag_prev_end);
+        ggml_vk_print_node_list(device->diag_cgraph, device->diag_prev_start, device->diag_prev_end);
+    } else {
+        GGML_LOG_ERROR("ggml_vulkan: device lost on %s\n", device->name.c_str());
+    }
+}
+
 class vk_perf_logger {
   public:
     void print_timings(bool force = false) {
@@ -2471,17 +2543,27 @@ static void ggml_vk_wait_for_fence(ggml_backend_vk_context * ctx) {
     // Use waitForFences while most of the graph executes. Hopefully the CPU can sleep
     // during this wait.
     if (ctx->almost_ready_fence_pending) {
-        VK_CHECK(ctx->device->device.waitForFences({ ctx->almost_ready_fence }, true, UINT64_MAX), "almost_ready_fence");
+        VK_CHECK(ctx->device->device.waitForFences({ ctx->almost_ready_fence }, true, UINT64_MAX), "almost_ready_fence", ctx->device);
         ctx->device->device.resetFences({ ctx->almost_ready_fence });
         ctx->almost_ready_fence_pending = false;
     }
 
     // Spin (w/pause) waiting for the graph to finish executing.
     vk::Result result;
-    while ((result = ctx->device->device.getFenceStatus(ctx->fence)) != vk::Result::eSuccess) {
+    for (;;) {
+        try {
+            result = ctx->device->device.getFenceStatus(ctx->fence);
+        } catch (vk::DeviceLostError &) {
+            ggml_vk_print_device_lost_info(ctx->device);
+            GGML_LOG_ERROR("ggml_vulkan: getFenceStatus at %s:%d\n", __FILE__, __LINE__);
+            throw;
+        }
+        if (result == vk::Result::eSuccess) {
+            break;
+        }
         if (result != vk::Result::eNotReady) {
-            fprintf(stderr, "ggml_vulkan: error %s at %s:%d\n", to_string(result).c_str(), __FILE__, __LINE__);
-            exit(1);
+            GGML_LOG_ERROR("ggml_vulkan: error %s at %s:%d\n", to_string(result).c_str(), __FILE__, __LINE__);
+            throw vk::SystemError(vk::make_error_code(result), "ggml_vulkan: getFenceStatus");
         }
         for (uint32_t i = 0; i < 100; ++i) {
             YIELD();
@@ -3172,6 +3254,7 @@ static std::unique_ptr<vk_queue> ggml_vk_create_queue(vk_device& device, uint32_
     }
 
     h->queue = device->device.getQueue2(queue_info2);
+    h->device = device;
     q->handle = h;
 
     q->cmd_pool.init(device, q.get());
@@ -5597,11 +5680,8 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
     ggml_vk_create_pipeline(device, device->pipeline_argmax_f32, "argmax_f32", argmax_f32_len, argmax_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1);
 
     ggml_vk_create_pipeline(device, device->pipeline_sum_rows_f32, "sum_rows_f32", sum_rows_f32_len, sum_rows_f32_data, "main", 2, sizeof(vk_op_sum_rows_push_constants), {1, 1, 1}, { device->subgroup_size }, 1);
-    // Intel Windows driver older than 32.0.101.8860 will crash when using fwht kernels on Xe2+ GPUS so we gate that here
-    const bool can_use_fwht = device->driver_id != vk::DriverId::eIntelProprietaryWindows ||
-        device->architecture != vk_device_architecture::INTEL_XE2 ||
-        (device->architecture == vk_device_architecture::INTEL_XE2 && ggml_vk_intel_windows_driver_equals_or_newer_than(device->properties.driverVersion, 101, 8860));
-    if (can_use_fwht && device->subgroup_basic && device->subgroup_shuffle) {
+    // Intel Arc B390 was observed segfaulting with this shader.
+    if (device->subgroup_basic && device->subgroup_shuffle && device->vendor_id != VK_VENDOR_ID_INTEL) {
         int idx = 0;
         for (uint32_t n : {64, 128, 256, 512}) {
             if (device->subgroup_size <= n) {
@@ -5609,7 +5689,8 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
             }
             ++idx;
         }
-    } else if (can_use_fwht) {
+    } else if (device->driver_id != vk::DriverId::eIntelProprietaryWindows) {
+        // Disabled on Intel Windows due to a driver bug: https://github.com/ggml-org/llama.cpp/pull/23964#issuecomment-4598226147
         int idx = 0;
         for (uint32_t n : {64, 128, 256, 512}) {
             const uint32_t block_size = std::min(device->subgroup_size, n);
@@ -5666,15 +5747,12 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
     ggml_vk_create_pipeline(device, device->pipeline_snake_f16,  "snake_f16",  snake_f16_len,  snake_f16_data,  "main", 4, sizeof(vk_op_snake_push_constants), {256, 1, 1}, {}, 1);
     ggml_vk_create_pipeline(device, device->pipeline_snake_bf16, "snake_bf16", snake_bf16_len, snake_bf16_data, "main", 4, sizeof(vk_op_snake_push_constants), {256, 1, 1}, {}, 1);
 
-    ggml_vk_create_pipeline(device, device->pipeline_pool1d_f32, "pool1d_f32", pool1d_f32_len, pool1d_f32_data, "main", 2, sizeof(vk_op_pool1d_push_constants), {512, 1, 1}, {}, 1);
     ggml_vk_create_pipeline(device, device->pipeline_pool2d_f32, "pool2d_f32", pool2d_f32_len, pool2d_f32_data, "main", 2, sizeof(vk_op_pool2d_push_constants), {512, 1, 1}, {}, 1);
 
     ggml_vk_create_pipeline(device, device->pipeline_rwkv_wkv6_f32, "rwkv_wkv6_f32", rwkv_wkv6_f32_len, rwkv_wkv6_f32_data, "main", 7, sizeof(vk_op_rwkv_wkv6_push_constants), {1, 1, 1}, {device->subgroup_size}, 1);
 
     ggml_vk_create_pipeline(device, device->pipeline_rwkv_wkv7_f32, "rwkv_wkv7_f32", rwkv_wkv7_f32_len, rwkv_wkv7_f32_data, "main", 8, sizeof(vk_op_rwkv_wkv7_push_constants), {1, 1, 1}, {device->subgroup_size}, 1);
 
-    ggml_vk_create_pipeline(device, device->pipeline_gated_linear_attn_f32, "gated_linear_attn_f32", gated_linear_attn_f32_len, gated_linear_attn_f32_data, "main", 6, sizeof(vk_op_gated_linear_attn_push_constants), {1, 1, 1}, {}, 1);
-
     {
         const uint32_t gdn_sizes[] = {16, 32, 64, 128};
         const char * gdn_names[][2] = {
@@ -6117,6 +6195,8 @@ static vk_device ggml_vk_get_device(size_t idx) {
 #endif
             } else if (strcmp(VK_KHR_INTERNALLY_SYNCHRONIZED_QUEUES_EXTENSION_NAME, properties.extensionName) == 0) {
                 internally_sync_support = true;
+            } else if (strcmp("VK_EXT_device_fault", properties.extensionName) == 0) {
+                device->device_fault = true;
             }
         }
 
@@ -6471,8 +6551,18 @@ static vk_device ggml_vk_get_device(size_t idx) {
         }
 #endif
 
+        VkPhysicalDeviceFaultFeaturesEXT fault_features {};
+        fault_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FAULT_FEATURES_EXT;
+        if (device->device_fault) {
+            last_struct->pNext = (VkBaseOutStructure *)&fault_features;
+            last_struct = (VkBaseOutStructure *)&fault_features;
+            device_extensions.push_back("VK_EXT_device_fault");
+        }
+
         vkGetPhysicalDeviceFeatures2(device->physical_device, &device_features2);
 
+        device->device_fault = device->device_fault && fault_features.deviceFault;
+
         device->has_internally_synchronized_queues = internally_synchronized_queues_features.internallySynchronizedQueues;
 
         // Build queue create infos only after querying whether internally synchronized queues are enabled.
@@ -6771,6 +6861,11 @@ static vk_device ggml_vk_get_device(size_t idx) {
         device_create_info.setPNext(&device_features2);
         device->device = device->physical_device.createDevice(device_create_info);
 
+        if (device->device_fault) {
+            device->pfn_vkGetDeviceFaultInfoEXT = (PFN_vkGetDeviceFaultInfoEXT)
+                vkGetDeviceProcAddr(device->device, "vkGetDeviceFaultInfoEXT");
+        }
+
         // Queues
         device->compute_queue = ggml_vk_create_queue(device, compute_queue_family_index, 0, { vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer }, false);
 
@@ -6893,6 +6988,8 @@ static vk_device ggml_vk_get_device(size_t idx) {
 
         device->idx = idx;
 
+        device->serialize_submissions = getenv("GGML_VK_SERIALIZE_SUBMISSIONS") != nullptr;
+
         device->disable_fusion = getenv("GGML_VK_DISABLE_FUSION") != nullptr;
 
         device->add_rms_fusion = !device->disable_fusion &&
@@ -8319,7 +8416,7 @@ static void ggml_vk_buffer_write_2d(vk_buffer& dst, size_t offset, const void *
         }
 
         ggml_vk_submit(subctx, dst->device->fence);
-        VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences");
+        VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences", dst->device);
         dst->device->device.resetFences({ dst->device->fence });
         ggml_vk_queue_command_pools_cleanup(dst->device);
     }
@@ -8431,7 +8528,7 @@ static void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, si
         ggml_vk_ctx_end(subctx);
         ggml_vk_submit(subctx, src->device->fence);
         VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX),
-                 "vk_buffer_read_2d uma waitForFences");
+                 "vk_buffer_read_2d uma waitForFences", src->device);
         src->device->device.resetFences({ src->device->fence });
         ggml_vk_queue_command_pools_cleanup(src->device);
 
@@ -8452,7 +8549,7 @@ static void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, si
         ggml_vk_ctx_end(subctx);
 
         ggml_vk_submit(subctx, src->device->fence);
-        VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_read_2d waitForFences");
+        VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_read_2d waitForFences", src->device);
         src->device->device.resetFences({ src->device->fence });
         ggml_vk_queue_command_pools_cleanup(src->device);
 
@@ -8487,7 +8584,7 @@ static void ggml_vk_buffer_copy(vk_buffer& dst, size_t dst_offset, vk_buffer& sr
         ggml_vk_buffer_copy_async(subctx, dst, dst_offset, src, src_offset, size);
         ggml_vk_ctx_end(subctx);
         ggml_vk_submit(subctx, src->device->fence);
-        VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences");
+        VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences", src->device);
         src->device->device.resetFences({ src->device->fence });
         ggml_vk_queue_command_pools_cleanup(src->device);
     } else {
@@ -8531,7 +8628,7 @@ static void ggml_vk_buffer_memset(vk_buffer& dst, size_t offset, uint32_t c, siz
     ggml_vk_ctx_end(subctx);
 
     ggml_vk_submit(subctx, dst->device->fence);
-    VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_memset waitForFences");
+    VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_memset waitForFences", dst->device);
     dst->device->device.resetFences({ dst->device->fence });
     ggml_vk_queue_command_pools_cleanup(dst->device);
 }
@@ -11382,11 +11479,6 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
             case GGML_TYPE_BF16: return ctx->device->pipeline_col2im_1d_bf16;
             default:             return nullptr;
         }
-    case GGML_OP_POOL_1D:
-        if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
-            return ctx->device->pipeline_pool1d_f32;
-        }
-        return nullptr;
     case GGML_OP_POOL_2D:
         if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
             return ctx->device->pipeline_pool2d_f32;
@@ -11402,11 +11494,6 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
             return ctx->device->pipeline_rwkv_wkv7_f32;
         }
         return nullptr;
-    case GGML_OP_GATED_LINEAR_ATTN:
-        if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
-            return ctx->device->pipeline_gated_linear_attn_f32;
-        }
-        return nullptr;
     case GGML_OP_GATED_DELTA_NET:
         if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
             const uint32_t S_v = dst->src[2]->ne[0];
@@ -11914,13 +12001,6 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co
         {
             elements = { uint32_t(dst->ne[0]), uint32_t(dst->ne[1]), 1 };
         } break;
-    case GGML_OP_POOL_1D:
-        {
-            const uint32_t N = dst->ne[3] * dst->ne[2];
-            const uint32_t OC = dst->ne[1];
-            const uint32_t OL = dst->ne[0];
-            elements = { N * OC * OL, 1, 1};
-        } break;
     case GGML_OP_POOL_2D:
         {
             const uint32_t N = dst->ne[3];
@@ -12437,41 +12517,6 @@ static void ggml_vk_rwkv_wkv7(ggml_backend_vk_context * ctx, vk_context& subctx,
     );
 }
 
-static void ggml_vk_gated_linear_attn(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
-    const size_t seq_length = dst->src[0]->ne[2];
-    const size_t n_embed    = dst->ne[0];
-    const size_t n_heads    = dst->src[0]->ne[1];
-    const size_t n_seqs     = dst->src[4]->ne[1];
-
-    float scale;
-    memcpy(&scale, dst->op_params, sizeof(float));
-
-    GGML_ASSERT(dst->buffer != nullptr);
-
-    vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, dst->src[0], dst->src[1], dst->src[2], dst, dst->op);
-    GGML_ASSERT(pipeline != nullptr);
-
-    ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1);
-
-    vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst);
-    vk_subbuffer src_buf[5] = {};
-    for (int i = 0; i < 5; i++) {
-        src_buf[i] = ggml_vk_tensor_subbuffer(ctx, dst->src[i]);
-    }
-
-    const vk_op_gated_linear_attn_push_constants pc = {
-        (uint32_t)n_seqs,
-        (uint32_t)seq_length,
-        (uint32_t)n_embed,
-        (uint32_t)n_heads,
-        scale,
-    };
-
-    ggml_vk_dispatch_pipeline(ctx, subctx, pipeline,
-        {src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], dst_buf},
-        pc, { (uint32_t)(n_seqs * n_heads), 1, 1 });
-}
-
 static void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
     const ggml_tensor * src_q     = dst->src[0];
     const ggml_tensor * src_v     = dst->src[2];
@@ -13277,16 +13322,12 @@ static void ggml_vk_soft_max_back(ggml_backend_vk_context * ctx, vk_context& sub
 
 static void ggml_vk_topk_moe(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) {
     topk_moe_mode mode = ctx->fused_topk_moe_mode;
-    const bool has_bias = mode == TOPK_MOE_SIGMOID_NORM_BIAS || mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS;
     ggml_tensor * logits = cgraph->nodes[node_idx + 0]->src[0];
-    ggml_tensor * bias = mode == TOPK_MOE_SIGMOID_NORM_BIAS       ? cgraph->nodes[node_idx + 2]->src[1] :
-                         mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS ? cgraph->nodes[node_idx + 3]->src[1] :
-                                                                   logits;
+    ggml_tensor * bias = (mode == TOPK_MOE_SIGMOID_NORM_BIAS) ? cgraph->nodes[node_idx + 2]->src[1] : logits;
     ggml_tensor * weights = cgraph->nodes[node_idx + ctx->num_additional_fused_ops];
-    ggml_tensor * ids = mode == TOPK_MOE_SIGMOID_NORM_BIAS       ? cgraph->nodes[node_idx + 4] :
-                        mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS ? cgraph->nodes[node_idx + 5] :
-                        mode == TOPK_MOE_LATE_SOFTMAX             ? cgraph->nodes[node_idx + 1] :
-                                                                   cgraph->nodes[node_idx + 3];
+    ggml_tensor * ids = (mode == TOPK_MOE_SIGMOID_NORM_BIAS) ? cgraph->nodes[node_idx + 4] :
+                        (mode == TOPK_MOE_LATE_SOFTMAX) ?      cgraph->nodes[node_idx + 1] :
+                                                               cgraph->nodes[node_idx + 3];
 
     GGML_ASSERT(logits->type == GGML_TYPE_F32);
     GGML_ASSERT(bias->type == GGML_TYPE_F32);
@@ -13326,24 +13367,16 @@ static void ggml_vk_topk_moe(ggml_backend_vk_context * ctx, vk_context& subctx,
         pc.clamp_min = ggml_get_op_params_f32(clamp, 0);
         pc.clamp_max = ggml_get_op_params_f32(clamp, 1);
     }
-    if (mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS) {
-        ggml_tensor * clamp = cgraph->nodes[node_idx + 9];
-        GGML_ASSERT(clamp->op == GGML_OP_CLAMP);
-        pc.clamp_min = ggml_get_op_params_f32(clamp, 0);
-        pc.clamp_max = ggml_get_op_params_f32(clamp, 1);
-    }
 
 #define GATING_FUNC_SOFTMAX 0
 #define GATING_FUNC_SIGMOID 1
 #define GATING_FUNC_SOFTMAX_WEIGHT 2
-#define GATING_FUNC_SQRT_SOFTPLUS 3
 
-    pc.gating_func = mode == TOPK_MOE_SIGMOID_NORM_BIAS       ? GATING_FUNC_SIGMOID :
-                     mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS ? GATING_FUNC_SQRT_SOFTPLUS :
-                     mode == TOPK_MOE_LATE_SOFTMAX             ? GATING_FUNC_SOFTMAX_WEIGHT :
-                                                                 GATING_FUNC_SOFTMAX;
-    pc.has_bias = has_bias;
-    pc.with_norm = mode == TOPK_MOE_EARLY_SOFTMAX_NORM || has_bias;
+    pc.gating_func = mode == TOPK_MOE_SIGMOID_NORM_BIAS ? GATING_FUNC_SIGMOID :
+                     mode == TOPK_MOE_LATE_SOFTMAX ?      GATING_FUNC_SOFTMAX_WEIGHT :
+                                                          GATING_FUNC_SOFTMAX;
+    pc.has_bias = mode == TOPK_MOE_SIGMOID_NORM_BIAS;
+    pc.with_norm = mode == TOPK_MOE_EARLY_SOFTMAX_NORM || mode == TOPK_MOE_SIGMOID_NORM_BIAS;
     if (ctx->fused_topk_moe_scale) {
         GGML_ASSERT(weights->op == GGML_OP_SCALE);
         pc.output_scale = ggml_get_op_params_f32(weights, 0);
@@ -13887,29 +13920,6 @@ static void ggml_vk_snake_dispatch_fused(ggml_backend_vk_context * ctx, vk_conte
     ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { x_buf, a_buf, inv_b_buf, dst_buf }, pc, elements);
 }
 
-static void ggml_vk_pool_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
-    uint32_t op = static_cast<uint32_t>(dst->op_params[0]);
-    const int32_t k0 = dst->op_params[1];
-    const int32_t s0 = dst->op_params[2];
-    const int32_t p0 = dst->op_params[3];
-
-    const uint32_t IL = src0->ne[0];
-
-    const uint32_t N = dst->ne[3] * dst->ne[2];
-
-    const uint32_t OC = dst->ne[1];
-    const uint32_t OL = dst->ne[0];
-
-    const uint32_t parallel_elements = N * OC * OL;
-
-    ggml_vk_op_f32<vk_op_pool1d_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_POOL_1D, {
-        IL, OL, OC,
-        parallel_elements,
-        op,
-        k0, s0, p0,
-    });
-}
-
 static void ggml_vk_pool_2d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
     uint32_t op = static_cast<uint32_t>(dst->op_params[0]);
     const int32_t k1 = dst->op_params[1];
@@ -14266,7 +14276,7 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t
 
     auto begin = std::chrono::high_resolution_clock::now();
     ggml_vk_submit(subctx, ctx->fence);
-    VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_matmul waitForFences");
+    VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_matmul waitForFences", ctx->device);
     ctx->device->device.resetFences({ ctx->fence });
     ggml_vk_queue_command_pools_cleanup(ctx->device);
 
@@ -14468,7 +14478,7 @@ static void ggml_vk_test_dequant(ggml_backend_vk_context * ctx, size_t ne, ggml_
     auto begin = std::chrono::high_resolution_clock::now();
 
     ggml_vk_submit(subctx, ctx->fence);
-    VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences");
+    VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences", ctx->device);
     ctx->device->device.resetFences({ ctx->fence });
     ggml_vk_queue_command_pools_cleanup(ctx->device);
 
@@ -14754,7 +14764,7 @@ static void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m,
     auto begin = std::chrono::high_resolution_clock::now();
 
     ggml_vk_submit(subctx, ctx->fence);
-    VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences");
+    VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences", ctx->device);
     ctx->device->device.resetFences({ ctx->fence });
     ggml_vk_queue_command_pools_cleanup(ctx->device);
 
@@ -15421,10 +15431,6 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr
     case GGML_OP_CONV_TRANSPOSE_1D:
         ggml_vk_conv_transpose_1d(ctx, compute_ctx, src0, src1, node);
 
-        break;
-    case GGML_OP_POOL_1D:
-        ggml_vk_pool_1d(ctx, compute_ctx, src0, node);
-
         break;
     case GGML_OP_POOL_2D:
         ggml_vk_pool_2d(ctx, compute_ctx, src0, node);
@@ -15471,11 +15477,6 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr
 
         break;
 
-    case GGML_OP_GATED_LINEAR_ATTN:
-        ggml_vk_gated_linear_attn(ctx, compute_ctx, node);
-
-        break;
-
     case GGML_OP_GATED_DELTA_NET:
         ggml_vk_gated_delta_net(ctx, compute_ctx, node);
 
@@ -15553,7 +15554,9 @@ static void ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_cgraph *
             memset(mset.dst, mset.val, mset.n);
         }
 
-        if (almost_ready && !ctx->almost_ready_fence_pending) {
+        if (ctx->device->serialize_submissions) {
+            ggml_vk_submit(subctx, ctx->fence);
+        } else if (almost_ready && !ctx->almost_ready_fence_pending) {
             ggml_vk_submit(subctx, ctx->almost_ready_fence);
             ctx->almost_ready_fence_pending = true;
         } else {
@@ -16164,12 +16167,20 @@ static void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {
             memcpy(cpy.dst, cpy.src, cpy.n);
         }
 
-        ggml_vk_submit(compute_ctx, {});
+        if (ctx->device->serialize_submissions) {
+            ggml_vk_submit(compute_ctx, ctx->fence);
+            VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "synchronize waitForFences", ctx->device);
+            ctx->device->device.resetFences({ ctx->fence });
+        } else {
+            ggml_vk_submit(compute_ctx, {});
+        }
         ctx->submit_pending = true;
     }
 
     if (ctx->submit_pending) {
-        if (ctx->device->async_use_transfer_queue && ctx->transfer_semaphore_last_submitted < ctx->transfer_semaphore.value) {
+        if (ctx->device->serialize_submissions) {
+            ctx->submit_pending = false;
+        } else if (ctx->device->async_use_transfer_queue && ctx->transfer_semaphore_last_submitted < ctx->transfer_semaphore.value) {
             vk::TimelineSemaphoreSubmitInfo tl_info{
                 1, &ctx->transfer_semaphore.value,
                 0, nullptr,
@@ -16186,7 +16197,9 @@ static void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {
         } else {
             ctx->device->compute_queue->handle->submit({}, ctx->fence);
         }
-        ggml_vk_wait_for_fence(ctx);
+        if (!ctx->device->serialize_submissions) {
+            ggml_vk_wait_for_fence(ctx);
+        }
         ctx->submit_pending = false;
         if (cmd_buf) {
             cmd_buf->in_use = false;
@@ -16457,20 +16470,6 @@ static bool ggml_vk_can_fuse_topk_moe(ggml_backend_vk_context * ctx, const struc
             return false;
         }
         break;
-    case TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS:
-        softmax = cgraph->nodes[node_idx + 0]; // really softplus
-        weights = cgraph->nodes[node_idx + 11];
-        get_rows = cgraph->nodes[node_idx + 6];
-        argsort = cgraph->nodes[node_idx + 4];
-        if (ggml_get_unary_op(softmax) != GGML_UNARY_OP_SOFTPLUS) {
-            return false;
-        }
-        // bias is expected to be 1D
-        if (ggml_nrows(cgraph->nodes[node_idx + 3]->src[1]) != 1 ||
-            !ggml_is_contiguous(cgraph->nodes[node_idx + 3]->src[1])) {
-            return false;
-        }
-        break;
     case TOPK_MOE_EARLY_SOFTMAX:
         softmax = cgraph->nodes[node_idx + 0];
         weights = cgraph->nodes[node_idx + 4];
@@ -16494,9 +16493,7 @@ static bool ggml_vk_can_fuse_topk_moe(ggml_backend_vk_context * ctx, const struc
     probs = probs->src[0];
     ggml_tensor * selection_probs = argsort->src[0];
 
-    if (probs != selection_probs &&
-        mode != TOPK_MOE_SIGMOID_NORM_BIAS &&
-        mode != TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS) {
+    if (probs != selection_probs && mode != TOPK_MOE_SIGMOID_NORM_BIAS) {
         return false;
     }
 
@@ -16758,6 +16755,10 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
     VK_LOG_DEBUG("ggml_backend_vk_graph_compute(" << cgraph->n_nodes << " nodes)");
     ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
 
+    ctx->device->diag_cgraph = nullptr;
+    ctx->device->diag_prev_start = -1;
+    ctx->device->diag_prev_end = -1;
+
     if (vk_instance.debug_utils_support) {
         vk::DebugUtilsLabelEXT dul = {};
         dul.pLabelName = "ggml_backend_vk_graph_compute";
@@ -16849,6 +16850,36 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
     }
     uint64_t flops_per_submit = std::min(flops_cap, ctx->last_total_flops / 40u);
 
+    auto const submit_after = [&](int start, int end) {
+        if (ctx->device->serialize_submissions) {
+            try {
+                auto res = ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX);
+                if (res != vk::Result::eSuccess) {
+                    GGML_LOG_ERROR("ggml_vulkan: waitForFences error during serialized submission\n");
+                    throw vk::SystemError(vk::make_error_code(res), "ggml_vulkan: waitForFences during serialized submission");
+                }
+            } catch (vk::DeviceLostError &) {
+                ggml_vk_print_device_fault_info(ctx->device);
+                GGML_LOG_ERROR("ggml_vulkan: device lost on %s waiting for submission (nodes %d to %d):\n",
+                        ctx->device->name.c_str(), start, end);
+                ggml_vk_print_node_list(cgraph, start, end);
+                throw;
+            }
+            ctx->device->device.resetFences({ ctx->fence });
+            ctx->submit_pending = false;
+            ctx->device->diag_cgraph = cgraph;
+            ctx->device->diag_prev_start = start;
+            ctx->device->diag_prev_end = end;
+        }
+        first_node_in_batch = true;
+        submitted_nodes = 0;
+        batch_flops = 0;
+        if (submit_count < 3) {
+            flops_per_submit *= 2;
+        }
+        submit_count++;
+    };
+
     for (int i = 0; i < cgraph->n_nodes; i++) {
         if (first_node_in_batch) {
             submit_node_idx = i;
@@ -16856,15 +16887,27 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
 
         {
             auto node_flops = ggml_vk_get_node_flops(cgraph->nodes[i]);
-            batch_flops += node_flops;
             total_flops += node_flops;
+
+            // Flush the current batch before recording a node that would push it over the flop threshold
+            if (flops_per_submit != 0 && submitted_nodes > 0 && batch_flops + node_flops >= flops_per_submit) {
+                vk_context flush_ctx = ggml_vk_get_compute_ctx(ctx);
+                ggml_vk_ctx_end(flush_ctx);
+                flush_ctx->exit_tensor_idx = -1;
+                ctx->compute_ctx.reset();
+                ggml_vk_compute_forward(ctx, cgraph, cgraph->nodes[submit_node_idx], submit_node_idx, false);
+                submit_after(submit_node_idx, i - 1);
+                submit_node_idx = i;
+            }
+
+            batch_flops += node_flops;
         }
 
         // op_srcs_fused_elementwise indicates whether an op's srcs all contribute to
         // the fused result in an elementwise-way. This affects whether the memory for
         // the src is allowed to overlap the memory for the destination.
         // The array is sized to handle the largest fusion (asserted later).
-        bool op_srcs_fused_elementwise[13];
+        bool op_srcs_fused_elementwise[12];
 
         ctx->fused_topk_moe_mode = TOPK_MOE_COUNT;
         ctx->fused_topk_moe_scale = false;
@@ -16975,15 +17018,6 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
                 ctx->fused_topk_moe_mode = TOPK_MOE_SIGMOID_NORM_BIAS;
                 fusion_string = "TOPK_MOE_SIGMOID_NORM_BIAS";
                 std::fill_n(op_srcs_fused_elementwise, ctx->num_additional_fused_ops + 1, false);
-            } else if (ggml_can_fuse_subgraph(cgraph, i, topk_moe_sqrt_softplus_norm_bias, { i + 5, i + 11 }) &&
-                       ggml_check_edges(cgraph, i, topk_moe_sqrt_softplus_norm_bias_edges) &&
-                       ggml_vk_can_fuse_topk_moe(ctx, cgraph, i, TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS)) {
-                ctx->num_additional_fused_ops = topk_moe_sqrt_softplus_norm_bias.size() - 1;
-                // view of argsort writes to memory
-                ctx->fused_ops_write_mask |= 1 << 5;
-                ctx->fused_topk_moe_mode = TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS;
-                fusion_string = "TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS";
-                std::fill_n(op_srcs_fused_elementwise, ctx->num_additional_fused_ops + 1, false);
             } else if (ggml_can_fuse_subgraph(cgraph, i, topk_moe_early_softmax, { i + 3, i + 4 }) &&
                        ggml_check_edges(cgraph, i, topk_moe_early_softmax_edges) &&
                        ggml_vk_can_fuse_topk_moe(ctx, cgraph, i, TOPK_MOE_EARLY_SOFTMAX)) {
@@ -17111,13 +17145,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
         }
 
         if (submit && enqueued) {
-            first_node_in_batch = true;
-            submitted_nodes = 0;
-            batch_flops = 0;
-            if (submit_count < 3) {
-                flops_per_submit *= 2;
-            }
-            submit_count++;
+            submit_after(submit_node_idx, i + (int)ctx->num_additional_fused_ops);
         }
         i += ctx->num_additional_fused_ops;
         ctx->num_additional_fused_ops = 0;
@@ -17133,13 +17161,13 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
         ggml_vk_ctx_end(compute_ctx);
 
         ggml_vk_submit(compute_ctx, ctx->device->fence);
-        VK_CHECK(ctx->device->device.waitForFences({ ctx->device->fence }, true, UINT64_MAX), "GGML_VULKAN_PERF waitForFences");
+        VK_CHECK(ctx->device->device.waitForFences({ ctx->device->fence }, true, UINT64_MAX), "GGML_VULKAN_PERF waitForFences", ctx->device);
         ctx->device->device.resetFences({ ctx->device->fence });
         ctx->compute_ctx.reset();
 
         // Get the results and pass them to the logger
         std::vector<uint64_t> timestamps(cgraph->n_nodes + 1);
-        VK_CHECK(ctx->device->device.getQueryPoolResults(ctx->query_pool, 0, ctx->query_idx, (cgraph->n_nodes + 1)*sizeof(uint64_t), timestamps.data(), sizeof(uint64_t), vk::QueryResultFlagBits::e64 | vk::QueryResultFlagBits::eWait), "get timestamp results");
+        VK_CHECK(ctx->device->device.getQueryPoolResults(ctx->query_pool, 0, ctx->query_idx, (cgraph->n_nodes + 1)*sizeof(uint64_t), timestamps.data(), sizeof(uint64_t), vk::QueryResultFlagBits::e64 | vk::QueryResultFlagBits::eWait), "get timestamp results", ctx->device);
         if (!vk_perf_logger_concurrent) {
             // Log each op separately
             for (int i = 1; i < ctx->query_idx; i++) {
@@ -17250,9 +17278,6 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph *
         if (keep_pattern(topk_moe_sigmoid_norm_bias)) {
             continue;
         }
-        if (keep_pattern(topk_moe_sqrt_softplus_norm_bias)) {
-            continue;
-        }
         if (keep_pattern(topk_moe_early_softmax)) {
             continue;
         }
@@ -17283,7 +17308,6 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph *
             // Don't pull forward nodes from fusion patterns
             if (match_pattern(topk_moe_early_softmax_norm, j) ||
                 match_pattern(topk_moe_sigmoid_norm_bias, j) ||
-                match_pattern(topk_moe_sqrt_softplus_norm_bias, j) ||
                 match_pattern(topk_moe_early_softmax, j) ||
                 match_pattern(topk_moe_late_softmax, j) ||
                 match_pattern(snake_pattern, j)) {
@@ -18176,16 +18200,11 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
         case GGML_OP_CONV_2D_DW:
             return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16)
                 && op->src[1]->type == GGML_TYPE_F32;
-        case GGML_OP_POOL_1D:
-            return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32;
         case GGML_OP_POOL_2D:
             return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32;
         case GGML_OP_RWKV_WKV6:
         case GGML_OP_RWKV_WKV7:
             return true; // all inputs are contiguous, see ggml.c
-        case GGML_OP_GATED_LINEAR_ATTN:
-            // the shader block size is hardcoded to head_size 64
-            return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && op->src[0]->ne[0] == 64;
         case GGML_OP_GATED_DELTA_NET:
             {
                 const uint32_t S_v = op->src[2]->ne[0];
@@ -18366,7 +18385,7 @@ static void ggml_backend_vk_device_event_synchronize(ggml_backend_dev_t dev, ggm
         vk::Semaphore sem = vkev->tl_semaphore.s;
         uint64_t val = vkev->tl_semaphore.value;
         vk::SemaphoreWaitInfo swi{vk::SemaphoreWaitFlags{}, sem, val};
-        VK_CHECK(device->device.waitSemaphores(swi, UINT64_MAX), "event_synchronize");
+        VK_CHECK(device->device.waitSemaphores(swi, UINT64_MAX), "event_synchronize", device);
 
         // Reset and move submitted events
         for (auto& event : vkev->events_submitted) {
@@ -18646,22 +18665,6 @@ static uint32_t ggml_vk_intel_shader_core_count(const vk::PhysicalDevice& vkdev)
     }
 }
 
-static bool ggml_vk_intel_windows_driver_equals_or_newer_than(uint32_t driver_version, uint32_t threshold_major, uint32_t threshold_minor) {
-#if defined(_WIN32)
-    // Intel Windows encodes xxx.yyyy as [31:14].[13:0].
-    const uint32_t major = driver_version >> 14;
-    const uint32_t minor = driver_version & 0x3fff;
-
-    return major > threshold_major || (major == threshold_major && minor >= threshold_minor);
-#else
-    GGML_UNUSED(driver_version);
-    GGML_UNUSED(threshold_major);
-    GGML_UNUSED(threshold_minor);
-    return true;
-#endif
-}
-
-
 // checks
 
 #ifdef GGML_VULKAN_CHECK_RESULTS
@@ -19116,13 +19119,6 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph *
             const int32_t oc     = tensor->op_params[1];
             const int32_t p0     = tensor->op_params[2];
             tensor_clone = ggml_col2im_1d(ggml_ctx, src_clone[0], stride, oc, p0);
-        } else if (tensor->op == GGML_OP_POOL_1D) {
-            enum ggml_op_pool op = static_cast<ggml_op_pool>(tensor->op_params[0]);
-            const int32_t k0 = tensor->op_params[1];
-            const int32_t s0 = tensor->op_params[2];
-            const int32_t p0 = tensor->op_params[3];
-
-            tensor_clone = ggml_pool_1d(ggml_ctx, src_clone[0], op, k0, s0, p0);
         } else if (tensor->op == GGML_OP_POOL_2D) {
             enum ggml_op_pool op = static_cast<ggml_op_pool>(tensor->op_params[0]);
             const int32_t k0 = tensor->op_params[1];
@@ -19175,10 +19171,6 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph *
         } else if (tensor->op == GGML_OP_RWKV_WKV7) {
             tensor_clone = ggml_rwkv_wkv7(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3],
             src_clone[4], src_clone[5], src_clone[6]);
-        } else if (tensor->op == GGML_OP_GATED_LINEAR_ATTN) {
-            const float * op_params = (const float *)tensor->op_params;
-            tensor_clone = ggml_gated_linear_attn(ggml_ctx, src_clone[0], src_clone[1],
-            src_clone[2], src_clone[3], src_clone[4], op_params[0]);
         } else if (tensor->op == GGML_OP_GATED_DELTA_NET) {
             tensor_clone = ggml_gated_delta_net(ggml_ctx, src_clone[0], src_clone[1],
             src_clone[2], src_clone[3], src_clone[4], src_clone[5],

ggml/src/ggml-vulkan/vulkan-shaders/gla.comp

@@ -1,82 +0,0 @@
-#version 450
-
-#extension GL_EXT_control_flow_attributes : require
-
-#define BLOCK_SIZE 64
-layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
-
-layout(push_constant) uniform Parameters {
-    uint B;
-    uint T;
-    uint C;
-    uint H;
-    float scale;
-};
-
-layout(binding = 0) readonly buffer KBuf { A_TYPE k[]; };
-layout(binding = 1) readonly buffer VBuf { A_TYPE v[]; };
-layout(binding = 2) readonly buffer QBuf { A_TYPE q[]; };
-layout(binding = 3) readonly buffer GBuf { A_TYPE g[]; };
-layout(binding = 4) readonly buffer StateBuf { A_TYPE state_in[]; };
-layout(binding = 5) buffer DstBuf { A_TYPE dst[]; };
-
-shared A_TYPE _k[BLOCK_SIZE], _q[BLOCK_SIZE], _g[BLOCK_SIZE];
-
-void main() {
-    const uint head_size = BLOCK_SIZE;
-    const uint batch_id = gl_WorkGroupID.x / H;
-    const uint head_id = gl_WorkGroupID.x % H;
-    const uint tid = gl_LocalInvocationID.x;
-
-    const uint state_size = C * head_size;
-    const uint n_seq_tokens = T / B;
-
-    if (batch_id >= B || head_id >= H) {
-        return;
-    }
-
-    // state[i] holds column tid of this head's state matrix: S[i][tid]
-    A_TYPE state[BLOCK_SIZE];
-    [[unroll]] for (uint i = 0; i < head_size; i++) {
-        state[i] = state_in[batch_id * state_size + head_id * head_size * head_size
-                          + i * head_size + tid];
-    }
-
-    const uint start_t = batch_id * n_seq_tokens * C + head_id * head_size + tid;
-    const uint end_t = (batch_id + 1) * n_seq_tokens * C + head_id * head_size + tid;
-
-    for (uint t = start_t; t < end_t; t += C) {
-        barrier();
-        _k[tid] = k[t];
-        _q[tid] = q[t];
-        _g[tid] = g[t];
-        barrier();
-
-        const A_TYPE v_val = v[t];
-        A_TYPE y = 0.0;
-
-        [[unroll]] for (uint i = 0; i < head_size; i += 4) {
-            vec4 k_vec = vec4(_k[i], _k[i+1], _k[i+2], _k[i+3]);
-            vec4 q_vec = vec4(_q[i], _q[i+1], _q[i+2], _q[i+3]);
-            vec4 g_vec = vec4(_g[i], _g[i+1], _g[i+2], _g[i+3]);
-            vec4 s_vec = vec4(state[i], state[i+1], state[i+2], state[i+3]);
-
-            vec4 kv = k_vec * v_val;
-
-            s_vec = s_vec * g_vec + kv;
-            y += dot(q_vec, s_vec);
-
-            state[i]   = s_vec.x;
-            state[i+1] = s_vec.y;
-            state[i+2] = s_vec.z;
-            state[i+3] = s_vec.w;
-        }
-
-        dst[t] = y * scale;
-    }
-
-    [[unroll]] for (uint i = 0; i < head_size; i++) {
-        dst[T * C + batch_id * state_size + head_id * head_size * head_size
-            + i * head_size + tid] = state[i];
-    }
-}

ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp

@@ -1,65 +0,0 @@
-#version 450
-
-#include "types.glsl"
-
-#extension GL_EXT_shader_16bit_storage : require
-
-layout(push_constant) uniform parameter {
-    uint IL;
-    uint OL;
-    uint OC;
-    uint pelements;
-    uint op;
-    int k0;
-    int s0;
-    int p0;
-} p;
-
-#define BLOCK_SIZE 512
-#define FLT_MAX 3.402823466e+38F
-#define OP_POOL_MAX 0u
-#define OP_POOL_AVG 1u
-
-layout (local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
-
-layout(binding = 0) readonly buffer X {A_TYPE data_a[];};
-layout(binding = 1) writeonly buffer D {D_TYPE data_d[];};
-
-void main() {
-    const uint idx = gl_GlobalInvocationID.x;
-    if (idx >= p.pelements) {
-        return;
-    }
-
-    const uint nc = idx / p.OL;
-    const uint cur_ol = idx % p.OL;
-
-    const int start = int(cur_ol) * p.s0 - p.p0;
-    const int bl = max(start, 0);
-    const int el = min(max(start + p.k0, 0), int(p.IL));
-
-    const int window_size = el - bl;
-    const float scale = window_size > 0 ? 1.0 / float(window_size) : 0.0;
-    float res;
-
-    if (p.op == OP_POOL_AVG) {
-        res = 0.0;
-    } else if (p.op == OP_POOL_MAX) {
-        res = -FLT_MAX;
-    } else {
-        return;
-    }
-
-    #pragma unroll
-    for (uint i = bl; i < el; i++) {
-        const float cur = D_TYPE(data_a[nc * p.IL + i]);
-
-        if (p.op == OP_POOL_AVG) {
-            res += cur * scale;
-        } else if (p.op == OP_POOL_MAX) {
-            res = max(res, cur);
-        }
-    }
-
-    data_d[nc * p.OL + cur_ol] = res;
-}

ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp

@@ -10,7 +10,6 @@
 #define GATING_FUNC_SOFTMAX 0
 #define GATING_FUNC_SIGMOID 1
 #define GATING_FUNC_SOFTMAX_WEIGHT 2
-#define GATING_FUNC_SQRT_SOFTPLUS 3
 
 layout (push_constant) uniform parameter
 {
@@ -121,13 +120,6 @@ void main() {
             const uint expert = i + lane;
             probs[i / WARP_SIZE] = (n_experts % WARP_SIZE == 0 || expert < n_experts) ? 1.f / (1.f + exp(-probs[i / WARP_SIZE])) : -INFINITY;
         }
-    } else if (gating_func == GATING_FUNC_SQRT_SOFTPLUS) {
-        [[unroll]]
-        for (uint i = 0; i < n_experts; i += WARP_SIZE) {
-            const uint expert = i + lane;
-            const float val = probs[i / WARP_SIZE];
-            probs[i / WARP_SIZE] = (n_experts % WARP_SIZE == 0 || expert < n_experts) ? sqrt(val > 20.0f ? val : log(1.0f + exp(val))) : -INFINITY;
-        }
     }
 
     float selection_probs[experts_per_thread];

ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp

@@ -1052,13 +1052,10 @@ void process_shaders() {
     string_to_spv("snake_f16",  "snake.comp", {{"DATA_A_F16", "1"},  {"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
     string_to_spv("snake_bf16", "snake.comp", {{"DATA_A_BF16", "1"}, {"DATA_D_BF16", "1"}, {"A_TYPE", "uint16_t"},  {"D_TYPE", "uint16_t"}});
 
-    string_to_spv("pool1d_f32", "pool1d.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}}));
     string_to_spv("pool2d_f32", "pool2d.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}}));
 
     string_to_spv("rwkv_wkv6_f32", "wkv6.comp", merge_maps(base_dict, {{"A_TYPE", "float"}}));
 
-    string_to_spv("gated_linear_attn_f32", "gla.comp", merge_maps(base_dict, {{"A_TYPE", "float"}}));
-
     string_to_spv("rwkv_wkv7_f32", "wkv7.comp", merge_maps(base_dict, {{"A_TYPE", "float"}}));
 
     string_to_spv("gated_delta_net_f32", "gated_delta_net.comp", merge_maps(base_dict, {{"FLOAT_TYPE", "float"}, {"USE_SUBGROUP_ADD", "1"}, {"USE_SUBGROUP_CLUSTERED", "1"}}));

ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp

@@ -2774,10 +2774,6 @@ class ggml_webgpu_shader_lib {
                 defines.push_back("TYPE_F32");
                 variant += "_f32";
                 break;
-            case GGML_TYPE_F16:
-                defines.push_back("TYPE_F16");
-                variant += "_f16";
-                break;
             case GGML_TYPE_I32:
                 defines.push_back("TYPE_I32");
                 variant += "_i32";

ggml/src/ggml-webgpu/ggml-webgpu.cpp

@@ -4290,8 +4290,7 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const
             supports_op = (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_I32);
             break;
         case GGML_OP_REPEAT:
-            supports_op = (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_I32 ||
-                           src0->type == GGML_TYPE_I16);
+            supports_op = (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_I32 || src0->type == GGML_TYPE_I16);
             break;
         case GGML_OP_CPY:
         case GGML_OP_CONT:

ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl

@@ -27,9 +27,6 @@ struct Params {
 #ifdef TYPE_I32
 #define DataType i32
 #endif
-#ifdef TYPE_F16
-#define DataType f16
-#endif
 #ifdef TYPE_I16
 // same size (16-bit) is sufficient for repeat
 #define DataType f16

gguf-py/gguf/constants.py

@@ -11,7 +11,6 @@ GGUF_MAGIC             = 0x46554747  # "GGUF"
 GGUF_VERSION           = 3
 GGUF_DEFAULT_ALIGNMENT = 32
 GGML_QUANT_VERSION     = 2  # GGML_QNT_VERSION from ggml.h
-GGML_MAX_DIMS          = 4  # GGML_MAX_DIMS from ggml.h
 
 #
 # metadata keys
@@ -354,7 +353,6 @@ class Keys:
         class Attention:
             HEAD_COUNT      = "clip.vision.attention.head_count"
             HEAD_COUNT_KV   = "clip.vision.attention.head_count_kv" # used by mimovl (GQA)
-            HEAD_DIM        = "clip.vision.attention.head_dim" # set when qkv width != n_embd
             LAYERNORM_EPS   = "clip.vision.attention.layer_norm_epsilon"
 
         class Projector:
@@ -2330,13 +2328,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.SSM_NORM,
         MODEL_TENSOR.SSM_IN,
         MODEL_TENSOR.SSM_BETA_ALPHA,
-        MODEL_TENSOR.SSM_OUT,
-        MODEL_TENSOR.NEXTN_EH_PROJ,
-        MODEL_TENSOR.NEXTN_EMBED_TOKENS,
-        MODEL_TENSOR.NEXTN_ENORM,
-        MODEL_TENSOR.NEXTN_HNORM,
-        MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
-        MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
+        MODEL_TENSOR.SSM_OUT
     ],
     MODEL_ARCH.QWEN3VL: [
         MODEL_TENSOR.TOKEN_EMBD,
@@ -3222,13 +3214,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.FFN_DOWN_SHEXP,
         MODEL_TENSOR.FFN_UP_SHEXP,
         MODEL_TENSOR.FFN_EXP_PROBS_B,
-        # NextN/MTP tensors
-        MODEL_TENSOR.NEXTN_EH_PROJ,
-        MODEL_TENSOR.NEXTN_EMBED_TOKENS,
-        MODEL_TENSOR.NEXTN_ENORM,
-        MODEL_TENSOR.NEXTN_HNORM,
-        MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
-        MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
     ],
     MODEL_ARCH.DEEPSEEK2OCR: [
         MODEL_TENSOR.TOKEN_EMBD,
@@ -3345,12 +3330,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.FFN_GATE_SHEXP,
         MODEL_TENSOR.FFN_DOWN_SHEXP,
         MODEL_TENSOR.FFN_UP_SHEXP,
-        MODEL_TENSOR.NEXTN_EH_PROJ,
-        MODEL_TENSOR.NEXTN_EMBED_TOKENS,
-        MODEL_TENSOR.NEXTN_ENORM,
-        MODEL_TENSOR.NEXTN_HNORM,
-        MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
-        MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
     ],
     MODEL_ARCH.ERNIE4_5_MOE: [
         MODEL_TENSOR.TOKEN_EMBD,
@@ -4397,35 +4376,10 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.ATTN_OUT,
         MODEL_TENSOR.ATTN_Q_NORM,
         MODEL_TENSOR.ATTN_K_NORM,
-        MODEL_TENSOR.ATTN_SINKS,
-        MODEL_TENSOR.ATTN_Q_A,
-        MODEL_TENSOR.ATTN_Q_B,
-        MODEL_TENSOR.ATTN_Q_A_NORM,
-        MODEL_TENSOR.ATTN_KV,
-        MODEL_TENSOR.ATTN_KV_NORM,
-        MODEL_TENSOR.ATTN_OUT_A,
-        MODEL_TENSOR.ATTN_OUT_B,
-        MODEL_TENSOR.HC_ATTN_FN,
-        MODEL_TENSOR.HC_ATTN_BASE,
-        MODEL_TENSOR.HC_ATTN_SCALE,
-        MODEL_TENSOR.HC_FFN_FN,
-        MODEL_TENSOR.HC_FFN_BASE,
-        MODEL_TENSOR.HC_FFN_SCALE,
-        MODEL_TENSOR.HC_HEAD_FN,
-        MODEL_TENSOR.HC_HEAD_BASE,
-        MODEL_TENSOR.HC_HEAD_SCALE,
         MODEL_TENSOR.FFN_NORM,
         MODEL_TENSOR.FFN_GATE,
         MODEL_TENSOR.FFN_DOWN,
         MODEL_TENSOR.FFN_UP,
-        MODEL_TENSOR.FFN_GATE_INP,
-        MODEL_TENSOR.FFN_EXP_PROBS_B,
-        MODEL_TENSOR.FFN_GATE_EXP,
-        MODEL_TENSOR.FFN_DOWN_EXP,
-        MODEL_TENSOR.FFN_UP_EXP,
-        MODEL_TENSOR.FFN_GATE_SHEXP,
-        MODEL_TENSOR.FFN_DOWN_SHEXP,
-        MODEL_TENSOR.FFN_UP_SHEXP,
         MODEL_TENSOR.FC,
         MODEL_TENSOR.ENC_OUTPUT_NORM,
         # optional DSpark heads

gguf-py/gguf/gguf_reader.py

@@ -22,7 +22,6 @@ if __name__ == "__main__":
     sys.path.insert(0, str(Path(__file__).parent.parent))
 
 from gguf.constants import (
-    GGML_MAX_DIMS,
     GGML_QUANT_SIZES,
     GGUF_DEFAULT_ALIGNMENT,
     GGUF_MAGIC,
@@ -267,8 +266,6 @@ class GGUFReader:
         # Get Tensor Dimensions Count
         n_dims = self._get(offs, np.uint32)
         offs += int(n_dims.nbytes)
-        if n_dims[0] > GGML_MAX_DIMS:
-            raise ValueError(f'Tensor dimensions count {n_dims[0]} exceeds GGML_MAX_DIMS ({GGML_MAX_DIMS})')
 
         # Get Tensor Dimension Array
         dims = self._get(offs, np.uint64, n_dims[0])
@@ -329,10 +326,7 @@ class GGUFReader:
                 raise ValueError(f'Found duplicated tensor with name {tensor_name}')
             tensor_names.add(tensor_name)
             ggml_type = GGMLQuantizationType(raw_dtype[0])
-            # use Python ints: np.prod on uint64 wraps silently on overflow
-            n_elems = 1
-            for dim in dims.tolist():
-                n_elems *= int(dim)
+            n_elems = int(np.prod(dims))
             np_dims = tuple(reversed(dims.tolist()))
             block_size, type_size = GGML_QUANT_SIZES[ggml_type]
             n_bytes = n_elems * type_size // block_size

gguf-py/gguf/gguf_writer.py

@@ -1226,9 +1226,6 @@ class GGUFWriter:
     def add_vision_head_count_kv(self, value: int) -> None:
         self.add_uint32(Keys.ClipVision.Attention.HEAD_COUNT_KV, value)
 
-    def add_vision_head_dim(self, value: int) -> None:
-        self.add_uint32(Keys.ClipVision.Attention.HEAD_DIM, value)
-
     def add_vision_attention_layernorm_eps(self, value: float) -> None:
         self.add_float32(Keys.ClipVision.Attention.LAYERNORM_EPS, value)
 

gguf-py/tests/test_gguf_reader_validation.py

@@ -1,37 +0,0 @@
-import struct
-import numpy as np
-import pytest
-
-from gguf.gguf_reader import GGUFReader
-
-
-def _write_gguf(path, n_dims_field, dims):
-    buf = b'GGUF' + struct.pack('<IQQ', 3, 1, 0)  # version 3, 1 tensor, 0 kv
-    name = b'bad_tensor'
-    buf += struct.pack('<Q', len(name)) + name
-    buf += struct.pack('<I', n_dims_field)
-    for d in dims:
-        buf += struct.pack('<Q', d)
-    buf += struct.pack('<I', 0)  # dtype F32
-    buf += struct.pack('<Q', 0)  # tensor offset
-    buf += b'\x00' * 64
-    path.write_bytes(buf)
-
-
-def test_n_dims_upper_bound(tmp_path):
-    # crafted file claims 1_000_000 dims; must be rejected, not read past EOF
-    p = tmp_path / 'evil_ndims.gguf'
-    _write_gguf(p, 1_000_000, [1] * 8)
-    with pytest.raises(ValueError, match='exceeds GGML_MAX_DIMS'):
-        GGUFReader(p)
-
-
-def test_dims_product_no_uint64_wraparound(tmp_path):
-    # dims whose true product overflows uint64; np.prod would wrap to 4 and
-    # silently pass an undersized read. The reader must not accept it.
-    dims = [4194305, 4194305, 211106198978564]
-    assert int(np.prod(np.array(dims, dtype=np.uint64))) == 4  # the wrap bug
-    p = tmp_path / 'evil_overflow.gguf'
-    _write_gguf(p, len(dims), dims)
-    with pytest.raises(ValueError):
-        GGUFReader(p)

include/llama.h

@@ -1424,11 +1424,10 @@ extern "C" {
 
     /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first.
     LLAMA_API struct llama_sampler * llama_sampler_init_penalties(
-                             int32_t   n_vocab,
                              int32_t   penalty_last_n,   // last n tokens to penalize (0 = disable penalty, -1 = context size)
-                               float   penalty_repeat,   // must be > 0.0, 1.0 = disabled
-                               float   penalty_freq,     // must be finite, 0.0 = disabled
-                               float   penalty_present); // must be finite, 0.0 = disabled
+                               float   penalty_repeat,   // 1.0 = disabled
+                               float   penalty_freq,     // 0.0 = disabled
+                               float   penalty_present); // 0.0 = disabled
 
     ///  @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982
     LLAMA_API struct llama_sampler * llama_sampler_init_dry(

models/templates/deepseek-ai-DeepSeek-V4-Flash-0731.jinja

@@ -1,140 +0,0 @@
-{%- if not add_generation_prompt is defined -%}
-  {%- set add_generation_prompt = false -%}
-{%- endif -%}
-{%- if not thinking is defined -%}
-  {%- if enable_thinking is defined -%}
-    {%- set thinking = enable_thinking -%}
-  {%- else -%}
-    {%- set thinking = false -%}
-  {%- endif -%}
-{%- endif -%}
-{%- if not drop_thinking is defined -%}
-  {%- set drop_thinking = true -%}
-{%- endif -%}
-{%- set dsml_token = '｜DSML｜' -%}
-{%- set thinking_start_token = '<think>' -%}
-{%- set thinking_end_token = '</think>' -%}
-{%- set reasoning_effort_high = 'Reasoning Effort: Absolute maximum with no shortcuts permitted.\nYou MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\nExplicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n' -%}
-{%- set reasoning_effort_max = 'Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\nYou MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\nDo not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n' -%}
-{%- set response_format_template = '## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n' -%}
-{%- set has_tools = false -%}
-{%- set tools_header = '## Tools\n\nYou have access to a set of tools to help answer the user\'s question. You can invoke tools by writing a "<' + dsml_token + 'tool_calls>" block like the following:\n\n<' + dsml_token + 'tool_calls>\n<' + dsml_token + 'invoke name="$TOOL_NAME">\n<' + dsml_token + 'parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</' + dsml_token + 'parameter>\n...\n</' + dsml_token + 'invoke>\n<' + dsml_token + 'invoke name="$TOOL_NAME2">\n...\n</' + dsml_token + 'invoke>\n</' + dsml_token + 'tool_calls>\n\nString parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.\n\nIf thinking_mode is enabled (triggered by ' + thinking_start_token + '), you MUST output your complete reasoning inside ' + thinking_start_token + '...' + thinking_end_token + ' BEFORE any tool calls or final response.\n\nOtherwise, output directly after ' + thinking_end_token + ' with tool calls or final response.\n\n### Available Tool Schemas\n\n' -%}
-{%- set tools_footer = '\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n' -%}
-{%- set ns = namespace(system_prompt='', is_first_sp=true, has_tool_calls=false) -%}
-{%- for message in messages -%}
-  {%- if message['role'] == 'system' -%}
-    {%- if ns.is_first_sp -%}
-      {%- set ns.system_prompt = ns.system_prompt + (message['content'] or '') -%}
-      {%- set ns.is_first_sp = false -%}
-    {%- else -%}
-      {%- set ns.system_prompt = ns.system_prompt + '\n\n' + (message['content'] or '') -%}
-    {%- endif -%}
-  {%- endif -%}
-{%- endfor -%}
-{%- if tools is defined and tools -%}
-  {%- set has_tools = true -%}
-  {%- set ts = namespace(schemas='') -%}
-  {%- for tool in tools -%}
-    {%- if tool['type'] == 'function' -%}
-      {%- set ts.schemas = ts.schemas + (tool['function'] | tojson) + '\n' -%}
-    {%- endif -%}
-  {%- endfor -%}
-  {%- if ns.system_prompt -%}
-    {%- set ns.system_prompt = ns.system_prompt + '\n\n' + tools_header + ts.schemas + tools_footer -%}
-  {%- else -%}
-    {%- set ns.system_prompt = tools_header + ts.schemas + tools_footer -%}
-  {%- endif -%}
-{%- endif -%}
-{%- if response_format is defined -%}
-  {%- if ns.system_prompt -%}
-    {%- set ns.system_prompt = ns.system_prompt + '\n\n' -%}
-  {%- endif -%}
-  {%- set ns.system_prompt = ns.system_prompt + response_format_template + (response_format | tojson) -%}
-{%- endif -%}
-{{- bos_token -}}
-{%- if messages and thinking and reasoning_effort is defined and reasoning_effort == 'high' -%}
-  {{- reasoning_effort_high -}}
-{%- elif messages and thinking and reasoning_effort is defined and reasoning_effort == 'max' -%}
-  {{- reasoning_effort_max -}}
-{%- endif -%}
-{{- ns.system_prompt -}}
-{%- set last_user_idx = namespace(value=-1) -%}
-{%- for message in messages -%}
-  {%- if message['role'] == 'user' or message['role'] == 'developer' or message['role'] == 'tool' -%}
-    {%- set last_user_idx.value = loop.index0 -%}
-  {%- endif -%}
-{%- endfor -%}
-{%- set state = namespace(in_user=false) -%}
-{%- for message in messages -%}
-  {%- if message['role'] == 'tool' -%}
-    {%- set ns.has_tool_calls = true -%}
-  {%- endif -%}
-{%- endfor -%}
-{%- for message in messages -%}
-  {%- if message['role'] == 'user' or message['role'] == 'developer' -%}
-    {%- if state.in_user -%}
-      {{- '\n\n' -}}
-    {%- else -%}
-      {{- '<｜User｜>' -}}
-      {%- set state.in_user = true -%}
-    {%- endif -%}
-    {{- message['content'] or '' -}}
-  {%- elif message['role'] == 'tool' -%}
-    {%- if state.in_user -%}
-      {{- '\n\n' -}}
-    {%- else -%}
-      {{- '<｜User｜>' -}}
-      {%- set state.in_user = true -%}
-    {%- endif -%}
-    {{- '<tool_result>' + (message['content'] or '') + '</tool_result>' -}}
-  {%- elif message['role'] == 'assistant' -%}
-    {%- set state.in_user = false -%}
-    {{- '<｜Assistant｜>' -}}
-    {%- set is_after_last_user = loop.index0 > last_user_idx.value -%}
-    {%- set keep_reasoning = thinking and ((not drop_thinking) or has_tools or is_after_last_user or ns.has_tool_calls) -%}
-    {%- if keep_reasoning -%}
-      {{- thinking_start_token -}}
-      {%- if message['reasoning_content'] is defined and message['reasoning_content'] -%}
-        {{- message['reasoning_content'] -}}
-      {%- endif -%}
-      {{- thinking_end_token -}}
-    {%- else -%}
-      {{- thinking_end_token -}}
-    {%- endif -%}
-    {%- if message['content'] is defined and message['content'] -%}
-      {{- message['content'] -}}
-    {%- endif -%}
-    {%- if message['tool_calls'] -%}
-      {{- '\n\n<' + dsml_token + 'tool_calls>\n' -}}
-      {%- for tool in message['tool_calls'] -%}
-        {%- set func = tool['function'] -%}
-        {{- '<' + dsml_token + 'invoke name="' + func['name'] + '">\n' -}}
-        {%- set args = func['arguments'] -%}
-        {%- if args is string -%}
-          {%- set args = args | from_json -%}
-        {%- endif -%}
-        {%- for key, val in args.items() -%}
-          {%- if val is string -%}
-            {{- '<' + dsml_token + 'parameter name="' + key + '" string="true">' + val + '</' + dsml_token + 'parameter>\n' -}}
-          {%- else -%}
-            {{- '<' + dsml_token + 'parameter name="' + key + '" string="false">' + (val | tojson) + '</' + dsml_token + 'parameter>\n' -}}
-          {%- endif -%}
-        {%- endfor -%}
-        {%- if not args -%}
-          {{- '\n' -}}
-        {%- endif -%}
-        {{- '</' + dsml_token + 'invoke>\n' -}}
-      {%- endfor -%}
-      {{- '</' + dsml_token + 'tool_calls>' -}}
-    {%- endif -%}
-    {{- '<｜end▁of▁sentence｜>' -}}
-  {%- endif -%}
-{%- endfor -%}
-{%- if add_generation_prompt -%}
-  {{- '<｜Assistant｜>' -}}
-  {%- if thinking -%}
-    {{- thinking_start_token -}}
-  {%- else -%}
-    {{- thinking_end_token -}}
-  {%- endif -%}
-{%- endif -%}

models/templates/deepseek-ai-DeepSeek-V4.jinja

@@ -9,14 +9,11 @@
   {%- endif -%}
 {%- endif -%}
 {%- if not drop_thinking is defined -%}
-  {%- set drop_thinking = true -%}
+  {%- set drop_thinking = false -%}
 {%- endif -%}
 {%- set dsml_token = '｜DSML｜' -%}
 {%- set thinking_start_token = '<think>' -%}
 {%- set thinking_end_token = '</think>' -%}
-{%- set reasoning_effort_max = 'Reasoning Effort: Absolute maximum with no shortcuts permitted.\nYou MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\nExplicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n' -%}
-{%- set response_format_template = '## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n' -%}
-{%- set has_tools = false -%}
 {%- set tools_header = '## Tools\n\nYou have access to a set of tools to help answer the user\'s question. You can invoke tools by writing a "<' + dsml_token + 'tool_calls>" block like the following:\n\n<' + dsml_token + 'tool_calls>\n<' + dsml_token + 'invoke name="$TOOL_NAME">\n<' + dsml_token + 'parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</' + dsml_token + 'parameter>\n...\n</' + dsml_token + 'invoke>\n<' + dsml_token + 'invoke name="$TOOL_NAME2">\n...\n</' + dsml_token + 'invoke>\n</' + dsml_token + 'tool_calls>\n\nString parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.\n\nIf thinking_mode is enabled (triggered by ' + thinking_start_token + '), you MUST output your complete reasoning inside ' + thinking_start_token + '...' + thinking_end_token + ' BEFORE any tool calls or final response.\n\nOtherwise, output directly after ' + thinking_end_token + ' with tool calls or final response.\n\n### Available Tool Schemas\n\n' -%}
 {%- set tools_footer = '\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n' -%}
 {%- set ns = namespace(system_prompt='', is_first_sp=true, has_tool_calls=false) -%}
@@ -31,7 +28,6 @@
   {%- endif -%}
 {%- endfor -%}
 {%- if tools is defined and tools -%}
-  {%- set has_tools = true -%}
   {%- set ts = namespace(schemas='') -%}
   {%- for tool in tools -%}
     {%- if tool['type'] == 'function' -%}
@@ -44,16 +40,7 @@
     {%- set ns.system_prompt = tools_header + ts.schemas + tools_footer -%}
   {%- endif -%}
 {%- endif -%}
-{%- if response_format is defined -%}
-  {%- if ns.system_prompt -%}
-    {%- set ns.system_prompt = ns.system_prompt + '\n\n' -%}
-  {%- endif -%}
-  {%- set ns.system_prompt = ns.system_prompt + response_format_template + (response_format | tojson) -%}
-{%- endif -%}
 {{- bos_token -}}
-{%- if messages and thinking and reasoning_effort is defined and reasoning_effort == 'max' -%}
-  {{- reasoning_effort_max -}}
-{%- endif -%}
 {{- ns.system_prompt -}}
 {%- set last_user_idx = namespace(value=-1) -%}
 {%- for message in messages -%}
@@ -88,8 +75,8 @@
     {%- set state.in_user = false -%}
     {{- '<｜Assistant｜>' -}}
     {%- set is_after_last_user = loop.index0 > last_user_idx.value -%}
-    {%- set keep_reasoning = thinking and ((not drop_thinking) or has_tools or is_after_last_user or ns.has_tool_calls) -%}
-    {%- if keep_reasoning -%}
+    {%- set retain_reasoning = (not drop_thinking) or (is_after_last_user or ns.has_tool_calls) -%}
+    {%- if retain_reasoning and thinking -%}
       {{- thinking_start_token -}}
       {%- if message['reasoning_content'] is defined and message['reasoning_content'] -%}
         {{- message['reasoning_content'] -}}
@@ -117,9 +104,6 @@
             {{- '<' + dsml_token + 'parameter name="' + key + '" string="false">' + (val | tojson) + '</' + dsml_token + 'parameter>\n' -}}
           {%- endif -%}
         {%- endfor -%}
-        {%- if not args -%}
-          {{- '\n' -}}
-        {%- endif -%}
         {{- '</' + dsml_token + 'invoke>\n' -}}
       {%- endfor -%}
       {{- '</' + dsml_token + 'tool_calls>' -}}
@@ -134,4 +118,4 @@
   {%- else -%}
     {{- thinking_end_token -}}
   {%- endif -%}
-{%- endif -%}
+{%- endif -%}

scripts/sync-ggml.last

@@ -1 +1 @@
-90951f99af1fbebef3fbdd58ff5b8715b0bb9c43
+06ca97616793248fadb410ea8d69c7511b2005e4

scripts/sync_vendor.py

@@ -5,7 +5,7 @@ import os
 import sys
 import subprocess
 
-HTTPLIB_VERSION = "refs/tags/v0.52.0"
+HTTPLIB_VERSION = "refs/tags/v0.51.0"
 
 vendor = {
     "https://github.com/nlohmann/json/releases/latest/download/json.hpp":     "vendor/nlohmann/json.hpp",

skills/code-review/SKILL.md

@@ -47,7 +47,6 @@ Mandatory on every review; any finding here is **blocking**. Rule of thumb: GGUF
 - **Sizes/counts from tensor dims:** validate before allocating. Products like `ne[i]*nb[i]`/nbytes can overflow on crafted dims into an undersized alloc then heap overflow. Overflow checks must run BEFORE the arithmetic they guard - padding/alignment macros wrap to 0 near `SIZE_MAX`, so a guard after the pad passes.
 - **GGUF strings/arrays:** cap declared lengths and element counts before using them to size a loop or buffer; validate element type and length before casting an array to a pointer or reading fixed indices (`[i+1]`, `[0..2]`).
 - **File-supplied counts indexing fixed arrays:** bound any count (e.g. layer/block count into a `LLAMA_MAX_*` array) before indexing; watch checks that only fire when an optional key is present.
-- **Declared vs actual array length:** check the declared length of a GGUF array against the count actually read, not just against a buffer size.
 - **Bounds comparisons:** flag narrowing casts (`size_t`->`int32_t`) and signed/unsigned mixing that can bypass a length check and copy past a buffer.
 - **Parsed/derived indices:** range-check `stoi`/`atoi` results and catch parse throws; never use a default or derived token id (EOS/BOS/...) as an index without a bounds check.
 - **Reused/reserved buffers:** recheck bounds after a buffer is shrunk or reused; watch `reserve()` then index-by-assumed-size, and header fields read before their length is checked.
@@ -132,8 +131,6 @@ Enforce the `AGENTS.md` / `CONTRIBUTING.md` coding and naming guidelines on ever
 - Reuse existing infrastructure over introducing new components; no new third-party dependencies, extra headers, or files unless clearly justified.
 - Keep it simple: a simpler change doing 90% is often preferable to a complex one doing 100%. Flag unnecessary templates/fancy STL; basic `for` loops are fine here.
 - Every added line should be something the contributor can explain and defend to a reviewer without AI help - flag anything that looks copied-in without understanding.
-- `Co-authored-by:` must be reserved for human co-authors; AI contributions (claude, cursor, codex, etc.) must use `Assisted-by:`; if this point is violated, it's a blocking finding.
-- Any mentions of Minja must be treated as blocking; see `AGENTS.md` for why.
 
 ## Reporting
 

src/CMakeLists.txt

@@ -25,7 +25,6 @@ add_library(llama
             llama-kv-cache.cpp
             llama-kv-cache-iswa.cpp
             llama-kv-cache-dsa.cpp
-            llama-kv-cache-msa.cpp
             llama-kv-cache-dsv4.cpp
             llama-memory.cpp
             llama-memory-hybrid.cpp

src/llama-arch.cpp

@@ -968,7 +968,6 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
         case LLM_ARCH_KIMI_LINEAR:
         case LLM_ARCH_QWEN35:
         case LLM_ARCH_QWEN35MOE:
-        case LLM_ARCH_DEEPSEEK4:
             return true;
         default:
             return false;
@@ -991,7 +990,6 @@ bool llm_arch_supports_rs_rollback(const llm_arch & arch) {
     switch (arch) {
         case LLM_ARCH_QWEN35:
         case LLM_ARCH_QWEN35MOE:
-        case LLM_ARCH_DEEPSEEK4:
             return true;
         default:
             return false;

src/llama-context.cpp

@@ -120,9 +120,8 @@ llama_context::llama_context(
     cparams.no_perf                 = params.no_perf;
     cparams.warmup                  = false;
 
-    // +1: id n_layer() taps the output of the last layer ("input" of the head)
-    cparams.embeddings_layer_inp.resize(hparams.n_layer() + 1, false);
-    embd_layer_inp.resize(hparams.n_layer() + 1);
+    cparams.embeddings_layer_inp.resize(hparams.n_layer(), false);
+    embd_layer_inp.resize(hparams.n_layer());
 
     cparams.ctx_type     = params.ctx_type;
     cparams.pooling_type = params.pooling_type;
@@ -1165,7 +1164,7 @@ void llama_context::set_embeddings_nextn(bool value, bool masked) {
 void llama_context::set_embeddings_layer_inp(uint32_t lid, bool enable) {
     LLAMA_LOG_DEBUG("%s: lid = %d, enable = %d\n", __func__, lid, enable);
 
-    GGML_ASSERT(lid <= model.hparams.n_layer());
+    GGML_ASSERT(lid < model.hparams.n_layer());
 
     cparams.embeddings_layer_inp[lid] = enable;
 
@@ -1717,8 +1716,7 @@ int llama_context::decode(const llama_batch & batch_inp) {
     const auto & hparams = model.hparams;
 
     const int64_t n_vocab = vocab.n_tokens();
-    const bool    mtp_embd = cparams.ctx_type == LLAMA_CONTEXT_TYPE_MTP && batch_inp.embd;
-    const int64_t n_embd  = mtp_embd ? hparams.n_embd_out() : hparams.n_embd_inp();
+    const int64_t n_embd  = hparams.n_embd_inp();
 
     // when computing embeddings, all tokens are output
     const bool output_all   = cparams.embeddings;
@@ -2277,9 +2275,8 @@ void llama_context::extract_layer_inputs(const llm_graph_result * res, size_t to
 }
 
 void llama_context::output_reorder() {
-    const uint64_t n_vocab     = model.vocab.n_tokens();
-    const uint64_t n_embd      = model.hparams.n_embd;
-    const uint64_t n_embd_out  = model.hparams.n_embd_out();
+    const uint64_t n_vocab = model.vocab.n_tokens();
+    const uint64_t n_embd  = model.hparams.n_embd;
 
     for (size_t s = 0; s < output_swaps.size(); ++s) {
         const uint64_t i0 = output_swaps[s].i0;
@@ -2292,14 +2289,14 @@ void llama_context::output_reorder() {
         }
 
         if (embd.size > 0) {
-            for (uint64_t k = 0; k < n_embd_out; k++) {
-                std::swap(embd.data[i0*n_embd_out + k], embd.data[i1*n_embd_out + k]);
+            for (uint64_t k = 0; k < n_embd; k++) {
+                std::swap(embd.data[i0*n_embd + k], embd.data[i1*n_embd + k]);
             }
         }
 
         if (embd_nextn.size > 0) {
-            for (uint64_t k = 0; k < n_embd_out; k++) {
-                std::swap(embd_nextn.data[i0*n_embd_out + k], embd_nextn.data[i1*n_embd_out + k]);
+            for (uint64_t k = 0; k < n_embd; k++) {
+                std::swap(embd_nextn.data[i0*n_embd + k], embd_nextn.data[i1*n_embd + k]);
             }
         }
 
@@ -2354,7 +2351,6 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
         model.arch == LLM_ARCH_QWEN35 ||
         model.arch == LLM_ARCH_QWEN35MOE ||
         model.arch == LLM_ARCH_DEEPSEEK4 ||
-        (model.arch == LLM_ARCH_DFLASH && model.hparams.dsv4_hc_mult > 0) ||
         model.arch == LLM_ARCH_NANBEIGE ||
         model.arch == LLM_ARCH_MINIMAX_M3) {
         return std::max<uint32_t>(n_tokens * 40, 32u * model.n_tensors());

src/llama-graph.cpp

@@ -8,7 +8,6 @@
 #include "llama-kv-cache.h"
 #include "llama-kv-cache-iswa.h"
 #include "llama-kv-cache-dsa.h"
-#include "llama-kv-cache-msa.h"
 #include "llama-kv-cache-dsv4.h"
 #include "llama-memory-hybrid.h"
 #include "llama-memory-hybrid-iswa.h"
@@ -519,40 +518,6 @@ bool llm_graph_input_attn_k::can_reuse(const llm_graph_params & params) {
     return res;
 }
 
-llm_graph_input_attn_kv_msa::llm_graph_input_attn_kv_msa(
-        const llama_hparams & hparams,
-        const llama_cparams & cparams,
-        const llama_kv_cache_msa_context * mctx) :
-    llm_graph_input_attn_kv(hparams, cparams, mctx->get_base()),
-    mctx_msa(mctx) {
-}
-
-void llm_graph_input_attn_kv_msa::set_input(const llama_ubatch * ubatch) {
-    llm_graph_input_attn_kv::set_input(ubatch);
-
-    if (self_k_idxs_idx) {
-        mctx_msa->get_idx()->set_input_k_idxs(self_k_idxs_idx, ubatch);
-    }
-}
-
-bool llm_graph_input_attn_kv_msa::can_reuse(const llm_graph_params & params) {
-    mctx_msa = static_cast<const llama_kv_cache_msa_context *>(params.mctx);
-
-    // the parent class operates on the base cache context
-    this->mctx = mctx_msa->get_base();
-
-    bool res = true;
-
-    res &= self_k_idxs->ne[0] == params.ubatch.n_tokens;
-    if (self_k_idxs_idx) {
-        res &= self_k_idxs_idx->ne[0] == params.ubatch.n_tokens;
-    }
-
-    res &= can_reuse_kq_mask(self_kq_mask, this->mctx, params.ubatch, params.cparams);
-
-    return res;
-}
-
 void llm_graph_input_attn_k_dsa::set_input(const llama_ubatch * ubatch) {
     mctx->get_mla()->set_input_k_idxs(self_k_idxs_mla, ubatch);
 
@@ -654,63 +619,6 @@ bool llm_graph_input_attn_kv_iswa::can_reuse(const llm_graph_params & params) {
     return res;
 }
 
-void llm_graph_input_attn_k_iswa::set_input(const llama_ubatch * ubatch) {
-    // base tensors may not be allocated if there are no non-SWA attention layers
-    if (self_k_idxs && self_k_idxs->buffer) {
-        mctx->get_base()->set_input_k_idxs(self_k_idxs, ubatch);
-    }
-
-    // the kq mask guards on its own buffer: shared cells leave idxs unbacked while the mask stays live
-    if (self_kq_mask && self_kq_mask->buffer) {
-        mctx->get_base()->set_input_kq_mask(self_kq_mask, ubatch, cparams.causal_attn);
-    }
-
-    // swa tensors may not be allocated if there are no SWA attention layers
-    if (self_k_idxs_swa && self_k_idxs_swa->buffer) {
-        mctx->get_swa()->set_input_k_idxs(self_k_idxs_swa, ubatch);
-    }
-
-    if (self_kq_mask_swa && self_kq_mask_swa->buffer) {
-        mctx->get_swa()->set_input_kq_mask(self_kq_mask_swa, ubatch, cparams.causal_attn);
-    }
-
-    if (self_k_rot && self_k_rot->buffer) {
-        mctx->get_base()->set_input_k_rot(self_k_rot);
-    }
-
-    if (self_k_rot_swa && self_k_rot_swa->buffer) {
-        mctx->get_swa()->set_input_k_rot(self_k_rot_swa);
-    }
-}
-
-bool llm_graph_input_attn_k_iswa::can_reuse(const llm_graph_params & params) {
-    const auto * mctx = static_cast<const llama_kv_cache_iswa_context *>(params.mctx);
-
-    this->mctx = mctx;
-
-    bool res = true;
-
-    // base tensors may not be allocated if there are no non-SWA attention layers
-    if (self_k_idxs && self_k_idxs->buffer) {
-        res &= self_k_idxs->ne[0] == params.ubatch.n_tokens;
-    }
-
-    if (self_kq_mask && self_kq_mask->buffer) {
-        res &= can_reuse_kq_mask(self_kq_mask, mctx->get_base(), params.ubatch, params.cparams);
-    }
-
-    // swa tensors may not be allocated if there are no SWA attention layers
-    if (self_k_idxs_swa && self_k_idxs_swa->buffer) {
-        res &= self_k_idxs_swa->ne[0] == params.ubatch.n_tokens;
-    }
-
-    if (self_kq_mask_swa && self_kq_mask_swa->buffer) {
-        res &= can_reuse_kq_mask(self_kq_mask_swa, mctx->get_swa(), params.ubatch, params.cparams);
-    }
-
-    return res;
-}
-
 static void dsv4_set_i64(ggml_tensor * dst, const std::vector<int64_t> & src) {
     if (!dst || !dst->buffer) {
         return;
@@ -846,10 +754,6 @@ static void dsv4_set_comp_inputs(
     dsv4_set_i32(inp.state_pos, plan.state_pos);
     dsv4_set_i32(inp.state_persist_src_idxs, plan.state_persist_src_idxs);
     dsv4_set_i32(inp.state_persist_dst_idxs, plan.state_persist_dst_idxs);
-    dsv4_set_i32(inp.state_restore_src_idxs, plan.state_restore_src_idxs);
-    dsv4_set_i32(inp.state_restore_dst_idxs, plan.state_restore_dst_idxs);
-    dsv4_set_i32(inp.state_snapshot_src_idxs, plan.state_snapshot_src_idxs);
-    dsv4_set_i32(inp.state_snapshot_dst_idxs, plan.state_snapshot_dst_idxs);
     dsv4_set_i32(inp.state_read_idxs, plan.state_read_idxs);
     dsv4_set_i64(inp.state_write_idxs, plan.state_write_idxs);
     dsv4_set_i32(inp.state_write_pos, plan.state_write_pos);
@@ -894,10 +798,6 @@ static bool dsv4_can_reuse_comp_input(
     res &= dsv4_can_reuse_tensor_1d(inp.state_pos, plan.state_pos.size());
     res &= dsv4_can_reuse_tensor_1d(inp.state_persist_src_idxs, plan.state_persist_src_idxs.size());
     res &= dsv4_can_reuse_tensor_1d(inp.state_persist_dst_idxs, plan.state_persist_dst_idxs.size());
-    res &= dsv4_can_reuse_tensor_1d(inp.state_restore_src_idxs, plan.state_restore_src_idxs.size());
-    res &= dsv4_can_reuse_tensor_1d(inp.state_restore_dst_idxs, plan.state_restore_dst_idxs.size());
-    res &= dsv4_can_reuse_tensor_1d(inp.state_snapshot_src_idxs, plan.state_snapshot_src_idxs.size());
-    res &= dsv4_can_reuse_tensor_1d(inp.state_snapshot_dst_idxs, plan.state_snapshot_dst_idxs.size());
     res &= dsv4_can_reuse_tensor_1d(inp.state_read_idxs, plan.state_read_idxs.size());
     res &= dsv4_can_reuse_tensor_1d(inp.state_write_idxs, plan.state_write_idxs.size());
     res &= dsv4_can_reuse_tensor_1d(inp.state_write_pos, plan.state_write_pos.size());
@@ -932,10 +832,6 @@ static void dsv4_build_comp_inputs(
     inp.state_pos = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_pos.size(), std::string("dsv4_") + name + "_state_pos");
     inp.state_persist_src_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_persist_src_idxs.size(), std::string("dsv4_") + name + "_state_persist_src_idxs");
     inp.state_persist_dst_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_persist_dst_idxs.size(), std::string("dsv4_") + name + "_state_persist_dst_idxs");
-    inp.state_restore_src_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_restore_src_idxs.size(), std::string("dsv4_") + name + "_state_restore_src_idxs");
-    inp.state_restore_dst_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_restore_dst_idxs.size(), std::string("dsv4_") + name + "_state_restore_dst_idxs");
-    inp.state_snapshot_src_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_snapshot_src_idxs.size(), std::string("dsv4_") + name + "_state_snapshot_src_idxs");
-    inp.state_snapshot_dst_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_snapshot_dst_idxs.size(), std::string("dsv4_") + name + "_state_snapshot_dst_idxs");
     inp.state_read_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_read_idxs.size(), std::string("dsv4_") + name + "_state_read_idxs");
     inp.state_write_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I64, plan.state_write_idxs.size(), std::string("dsv4_") + name + "_state_write_idxs");
     inp.state_write_pos = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_write_pos.size(), std::string("dsv4_") + name + "_state_write_pos");
@@ -1299,7 +1195,7 @@ void llm_graph_result::reset() {
     t_embd_pooled = nullptr;
     t_h_nextn     = nullptr;
 
-    t_layer_inp.resize(LLAMA_MAX_LAYERS + 1);
+    t_layer_inp.resize(LLAMA_MAX_LAYERS);
     std::fill(t_layer_inp.begin(), t_layer_inp.end(), nullptr);
 
     t_sampled.clear();
@@ -1754,7 +1650,7 @@ ggml_tensor * llm_graph_context::build_ffn(
                         tmp = ggml_clamp(ctx0, tmp, -limit, limit);
                         cb(tmp, "ffn_up_clamped", il);
 
-                        if (arch == LLM_ARCH_DEEPSEEK4 || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0)) {
+                        if (arch == LLM_ARCH_DEEPSEEK4) {
                             cur = ggml_clamp(ctx0, cur, -INFINITY, limit);
                             cb(cur, "ffn_gate_clamped", il);
                             cur = ggml_swiglu_split(ctx0, cur, tmp);
@@ -2149,7 +2045,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
                         up = ggml_clamp(ctx0, up, -limit, limit);
                         cb(up, "ffn_moe_up_clamped", il);
 
-                        if (arch == LLM_ARCH_DEEPSEEK4 || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0)) {
+                        if (arch == LLM_ARCH_DEEPSEEK4) {
                             cur = ggml_clamp(ctx0, cur, -INFINITY, limit);
                             cb(cur, "ffn_moe_gate_clamped", il);
                             cur = ggml_swiglu_split(ctx0, cur, up);
@@ -3066,75 +2962,6 @@ ggml_tensor * llm_graph_context::build_attn(
     return cur;
 }
 
-ggml_tensor * llm_graph_context::build_attn(
-        llm_graph_input_attn_k_iswa * inp,
-        ggml_tensor * wo,
-        ggml_tensor * wo_b,
-        ggml_tensor * wo_s,
-        ggml_tensor * q_cur,
-        ggml_tensor * k_cur,
-        ggml_tensor * v_cur,
-        ggml_tensor * kq_b,
-        ggml_tensor * sinks,
-        ggml_tensor * v_mla,
-            float     kq_scale,
-            int       il) const {
-    const bool is_swa = hparams.is_swa(il);
-
-    GGML_UNUSED(v_cur);
-
-    auto * k_rot = is_swa ? inp->self_k_rot_swa : inp->self_k_rot;
-
-    if (k_rot) {
-        q_cur = llama_mul_mat_hadamard(ctx0, q_cur, k_rot);
-        if (k_cur) {
-            k_cur = llama_mul_mat_hadamard(ctx0, k_cur, k_rot);
-        }
-    }
-
-    // these nodes are added to the graph together so that they are not reordered
-    // by doing so, the number of splits in the graph is reduced
-    ggml_build_forward_expand(gf, q_cur);
-
-    if (k_cur) {
-        ggml_build_forward_expand(gf, k_cur);
-    }
-
-    const auto * mctx_iswa = inp->mctx;
-    const auto * mctx_cur = is_swa ? mctx_iswa->get_swa() : mctx_iswa->get_base();
-
-    // optionally store to KV cache
-    if (k_cur) {
-        const auto & k_idxs = is_swa ? inp->get_k_idxs_swa() : inp->get_k_idxs();
-
-        ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_cur, k_idxs, il));
-    }
-
-    const auto & kq_mask = is_swa ? inp->get_kq_mask_swa() : inp->get_kq_mask();
-
-    // MLA-style attention: the cached K is used as V
-    ggml_tensor * q = q_cur;
-    ggml_tensor * k = mctx_cur->get_k(ctx0, il);
-    ggml_tensor * v = k;
-
-    ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);
-    cb(cur, "kqv_out", il);
-
-    if (k_rot) {
-        cur = llama_mul_mat_hadamard(ctx0, cur, k_rot);
-    }
-
-    if (wo) {
-        cur = build_lora_mm(wo, cur, wo_s);
-    }
-
-    if (wo_b) {
-        cur = ggml_add(ctx0, cur, wo_b);
-    }
-
-    return cur;
-}
-
 llm_graph_input_attn_cross * llm_graph_context::build_attn_inp_cross() const {
     auto inp = std::make_unique<llm_graph_input_attn_cross>(cross);
 
@@ -3222,34 +3049,6 @@ llm_graph_input_attn_k_dsa * llm_graph_context::build_attn_inp_k_dsa() const {
     return (llm_graph_input_attn_k_dsa *) res->add_input(std::move(inp));
 }
 
-llm_graph_input_attn_kv_msa * llm_graph_context::build_attn_inp_kv_msa(bool msa_enabled) const {
-    const auto * mctx_cur = static_cast<const llama_kv_cache_msa_context *>(mctx);
-
-    auto inp = std::make_unique<llm_graph_input_attn_kv_msa>(hparams, cparams, mctx_cur);
-
-    const auto * mctx_base = mctx_cur->get_base();
-    const auto * mctx_idx  = mctx_cur->get_idx();
-
-    {
-        GGML_ASSERT(hparams.swa_type == LLAMA_SWA_TYPE_NONE && "Use llama_kv_cache_iswa for SWA");
-
-        inp->self_k_idxs = mctx_base->build_input_k_idxs(ctx0, ubatch);
-        inp->self_v_idxs = mctx_base->build_input_v_idxs(ctx0, ubatch);
-
-        inp->self_kq_mask = build_attn_inp_kq_mask(ctx0, mctx_base, ubatch, cparams);
-        inp->self_kq_mask_cnv = inp->self_kq_mask;
-    }
-
-    inp->self_k_rot = mctx_base->build_input_k_rot(ctx0);
-    inp->self_v_rot = mctx_base->build_input_v_rot(ctx0);
-
-    if (msa_enabled) {
-        inp->self_k_idxs_idx = mctx_idx->build_input_k_idxs(ctx0, ubatch);
-    }
-
-    return (llm_graph_input_attn_kv_msa *) res->add_input(std::move(inp));
-}
-
 // TODO: maybe separate the inner implementation into a separate function
 //       like with the non-sliding window equivalent
 //       once sliding-window hybrid caches are a thing.
@@ -3285,34 +3084,6 @@ llm_graph_input_attn_kv_iswa * llm_graph_context::build_attn_inp_kv_iswa() const
     return (llm_graph_input_attn_kv_iswa *) res->add_input(std::move(inp));
 }
 
-llm_graph_input_attn_k_iswa * llm_graph_context::build_attn_inp_k_iswa() const {
-    const auto * mctx_cur = static_cast<const llama_kv_cache_iswa_context *>(mctx);
-
-    auto inp = std::make_unique<llm_graph_input_attn_k_iswa>(hparams, cparams, mctx_cur);
-
-    {
-        inp->self_k_idxs = mctx_cur->get_base()->build_input_k_idxs(ctx0, ubatch);
-
-        inp->self_kq_mask = build_attn_inp_kq_mask(ctx0, mctx_cur->get_base(), ubatch, cparams);
-        inp->self_kq_mask_cnv = inp->self_kq_mask;
-    }
-
-    {
-        GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE && "Use llama_kv_cache for non-SWA");
-
-        inp->self_k_idxs_swa = mctx_cur->get_swa()->build_input_k_idxs(ctx0, ubatch);
-
-        inp->self_kq_mask_swa = build_attn_inp_kq_mask(ctx0, mctx_cur->get_swa(), ubatch, cparams);
-        inp->self_kq_mask_swa_cnv = inp->self_kq_mask_swa;
-    }
-
-    inp->self_k_rot = mctx_cur->get_base()->build_input_k_rot(ctx0);
-
-    inp->self_k_rot_swa = mctx_cur->get_swa()->build_input_k_rot(ctx0);
-
-    return (llm_graph_input_attn_k_iswa *) res->add_input(std::move(inp));
-}
-
 llm_graph_input_dsv4 * llm_graph_context::build_inp_dsv4() const {
     const auto * mctx_cur = static_cast<const llama_kv_cache_dsv4_context *>(mctx);
     const auto * raw_ctx  = mctx_cur->get_raw();

src/llama-graph.h

@@ -23,7 +23,6 @@ struct llama_memory_context_i;
 
 class llama_kv_cache_context;
 class llama_kv_cache_dsa_context;
-class llama_kv_cache_msa_context;
 class llama_kv_cache_dsv4_raw_context;
 class llama_kv_cache_dsv4_context;
 class llama_kv_cache_iswa_context;
@@ -426,26 +425,6 @@ public:
     const llama_kv_cache_dsa_context * mctx;
 };
 
-// standard K/V attention input against the base cache, plus destination indices for the indexer key cache
-class llm_graph_input_attn_kv_msa : public llm_graph_input_attn_kv {
-public:
-    llm_graph_input_attn_kv_msa(
-            const llama_hparams & hparams,
-            const llama_cparams & cparams,
-            const llama_kv_cache_msa_context * mctx);
-    ~llm_graph_input_attn_kv_msa() = default;
-
-    void set_input(const llama_ubatch * ubatch) override;
-
-    bool can_reuse(const llm_graph_params & params) override;
-
-    ggml_tensor * get_k_idxs_idx() const { return self_k_idxs_idx; }
-
-    ggml_tensor * self_k_idxs_idx = nullptr; // I64 [n_batch]
-
-    const llama_kv_cache_msa_context * mctx_msa;
-};
-
 class llm_graph_input_attn_kv_iswa : public llm_graph_input_i {
 public:
     llm_graph_input_attn_kv_iswa(
@@ -492,45 +471,6 @@ public:
     const llama_kv_cache_iswa_context * mctx;
 };
 
-class llm_graph_input_attn_k_iswa : public llm_graph_input_i {
-public:
-    llm_graph_input_attn_k_iswa(
-            const llama_hparams & hparams,
-            const llama_cparams & cparams,
-            const llama_kv_cache_iswa_context * mctx) :
-        hparams(hparams),
-        cparams(cparams),
-        mctx(mctx) {
-    }
-    ~llm_graph_input_attn_k_iswa() = default;
-
-    void set_input(const llama_ubatch * ubatch) override;
-
-    bool can_reuse(const llm_graph_params & params) override;
-
-    ggml_tensor * get_k_idxs()     const { return self_k_idxs; }
-    ggml_tensor * get_k_idxs_swa() const { return self_k_idxs_swa; }
-
-    ggml_tensor * get_kq_mask()     const { return self_kq_mask_cnv; }
-    ggml_tensor * get_kq_mask_swa() const { return self_kq_mask_swa_cnv; }
-
-    ggml_tensor * self_k_idxs     = nullptr; // I64 [n_batch]
-    ggml_tensor * self_k_idxs_swa = nullptr; // I64 [n_batch]
-
-    ggml_tensor * self_kq_mask         = nullptr; // F32/F16 [n_kv, n_batch/n_stream, 1, n_stream]
-    ggml_tensor * self_kq_mask_cnv     = nullptr; //         [n_kv, n_batch/n_stream, 1, n_stream]
-    ggml_tensor * self_kq_mask_swa     = nullptr; // F32/F16 [n_kv, n_batch/n_stream, 1, n_stream]
-    ggml_tensor * self_kq_mask_swa_cnv = nullptr; //         [n_kv, n_batch/n_stream, 1, n_stream]
-
-    ggml_tensor * self_k_rot = nullptr;
-    ggml_tensor * self_k_rot_swa = nullptr;
-
-    const llama_hparams hparams;
-    const llama_cparams cparams;
-
-    const llama_kv_cache_iswa_context * mctx;
-};
-
 // DSV4 raw graph inputs are SWA-only, but their mask may be stream-shaped
 // so raw K can be concatenated with DSV4 compressed K in one attention op.
 class llm_graph_input_dsv4_raw {
@@ -565,10 +505,6 @@ public:
         ggml_tensor * state_pos        = nullptr; // I32 [n_state]
         ggml_tensor * state_persist_src_idxs = nullptr; // I32 [n_state_persist]
         ggml_tensor * state_persist_dst_idxs = nullptr; // I32 [n_state_persist]
-        ggml_tensor * state_restore_src_idxs = nullptr; // I32 [n_state_restore]
-        ggml_tensor * state_restore_dst_idxs = nullptr; // I32 [n_state_restore]
-        ggml_tensor * state_snapshot_src_idxs = nullptr; // I32 [n_state_snapshot]
-        ggml_tensor * state_snapshot_dst_idxs = nullptr; // I32 [n_state_snapshot]
         ggml_tensor * state_read_idxs  = nullptr; // I32 [ratio*n_state_write]
         ggml_tensor * state_write_idxs = nullptr; // I64 [n_state_write]
         ggml_tensor * state_write_pos  = nullptr; // I32 [n_state_write]
@@ -1132,7 +1068,7 @@ struct llm_graph_context {
     ggml_tensor * build_attn_mha(
             ggml_tensor * q,       // [n_embd_head_q, n_head_q, n_tokens]
             ggml_tensor * k,       // [n_embd_head_k, n_head_k, n_tokens]
-            ggml_tensor * v,       // [n_embd_head_v, n_head_v, n_tokens] (v_trans = false)
+            ggml_tensor * v,       // [n_embd_head_v, n_head_v, n_tokens] (v_trans == false)
             ggml_tensor * kq_b,
             ggml_tensor * kq_mask,
             ggml_tensor * sinks,   // [n_head_q]
@@ -1190,8 +1126,6 @@ struct llm_graph_context {
 
     llm_graph_input_attn_k_dsa * build_attn_inp_k_dsa() const;
 
-    llm_graph_input_attn_kv_msa * build_attn_inp_kv_msa(bool msa_enabled) const;
-
     ggml_tensor * build_attn(
             llm_graph_input_attn_k_dsa * inp,
             ggml_tensor * wo,
@@ -1226,24 +1160,6 @@ struct llm_graph_context {
                   float   kq_scale,
                     int   il) const;
 
-    llm_graph_input_attn_k_iswa * build_attn_inp_k_iswa() const;
-
-    // note: if k_cur is not provided, it will not be stored in the memory
-    // note: the K cache is used as V (MLA-style attention)
-    ggml_tensor * build_attn(
-            llm_graph_input_attn_k_iswa * inp,
-            ggml_tensor * wo,
-            ggml_tensor * wo_b,
-            ggml_tensor * wo_s,
-            ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens]
-            ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens] optional
-            ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens] optional
-            ggml_tensor * kq_b,
-            ggml_tensor * sinks, // [n_head_q]
-            ggml_tensor * v_mla, // [n_embd_head_v_mla, n_embd_head_v, n_head_v]
-                  float   kq_scale,
-                    int   il) const;
-
     llm_graph_input_attn_cross * build_attn_inp_cross() const;
 
     ggml_tensor * build_attn(

src/llama-hparams.cpp

@@ -180,6 +180,16 @@ uint32_t llama_hparams::n_embd_v_gqa_max() const {
     return val;
 }
 
+uint32_t llama_hparams::n_embd_k_idx(uint32_t il) const {
+    if (!indexer_kv || indexer_head_size == 0) {
+        return 0; // arch without a MSA indexer
+    }
+    if (il < n_layer_dense_lead) {
+        return 0; // leading dense layers carry no indexer
+    }
+    return indexer_head_size; // 128
+}
+
 uint32_t llama_hparams::n_embd_r() const {
     if (wkv_head_size != 0) {
         // for RWKV models

src/llama-hparams.h

@@ -230,6 +230,8 @@ struct llama_hparams {
     // MSA
     uint32_t indexer_block_size  = 0;
     uint32_t indexer_local_blocks = 0;
+    // MSA stores its indexer keys in the main KV cache (k_idx tensors);
+    bool indexer_kv = false;
 
     // Indexer is "full" (1) or "shared" (0)
     // Shared indexers reuse top-k from previous full layer
@@ -354,6 +356,9 @@ struct llama_hparams {
     uint32_t n_embd_k_gqa_max() const;
     uint32_t n_embd_v_gqa_max() const;
 
+    // dimension of the single-head MSA indexer key stream
+    uint32_t n_embd_k_idx(uint32_t il = 0) const;
+
     // dimension of the rolling state embeddings
     // corresponds to Mamba's conv_states size or RWKV's token_shift states size
     uint32_t n_embd_r() const;

src/llama-kv-cache-dsa.cpp

@@ -23,8 +23,7 @@ llama_kv_cache_dsa::llama_kv_cache_dsa(
                  uint32_t   n_pad,
                  uint32_t   n_swa,
            llama_swa_type   swa_type,
-    const layer_filter_cb & filter_mla,
-    const layer_filter_cb & filter_lid,
+    const layer_filter_cb & filter,
     const  layer_reuse_cb & reuse) :
     hparams_lid(model.hparams), n_stream(unified ? 1 : n_seq_max) {
 
@@ -33,7 +32,7 @@ llama_kv_cache_dsa::llama_kv_cache_dsa(
     kv_mla = std::make_unique<llama_kv_cache>(
             model, model.hparams, type_k, type_v,
             v_trans, offload, unified, kv_size, n_seq_max, n_pad,
-            n_swa, swa_type, nullptr, filter_mla, reuse, nullptr);
+            n_swa, swa_type, nullptr, filter, reuse, nullptr);
 
     // we use llama_kv_cache for caching indexer keys
     // by hand-tweaking some hparams we fool it to create
@@ -50,7 +49,7 @@ llama_kv_cache_dsa::llama_kv_cache_dsa(
     kv_lid = std::make_unique<llama_kv_cache>(
             model, hparams_lid, type_k, type_v,
             v_trans, offload, unified, kv_size, n_seq_max, n_pad,
-            n_swa, swa_type, nullptr, filter_lid, reuse, nullptr);
+            n_swa, swa_type, nullptr, filter, reuse, nullptr);
 }
 
 void llama_kv_cache_dsa::clear(bool data) {

src/llama-kv-cache-dsa.h

@@ -26,8 +26,7 @@ public:
                      uint32_t   n_pad,
                      uint32_t   n_swa,
                llama_swa_type   swa_type,
-        const layer_filter_cb & filter_mla,
-        const layer_filter_cb & filter_lid,
+        const layer_filter_cb & filter,
         const  layer_reuse_cb & reuse);
 
     ~llama_kv_cache_dsa() = default;

src/llama-kv-cache-dsv4.cpp

@@ -252,8 +252,7 @@ static void dsv4_state_write_tensor_streams(
         uint32_t           tensor_rows,
         uint32_t           n_rows,
         uint32_t           s0,
-        uint32_t           ns,
-        const std::vector<uint32_t> * stream_ids = nullptr) {
+        uint32_t           ns) {
     const int32_t  type_i   = (int32_t) tensor->type;
     const uint64_t ne0      = tensor->ne[0];
     const uint64_t rows     = n_rows;
@@ -274,16 +273,8 @@ static void dsv4_state_write_tensor_streams(
         return;
     }
 
-    if (stream_ids && stream_ids->size() != ns) {
-        throw std::runtime_error("DSV4 state tensor stream map size mismatch");
-    }
-
     for (uint32_t s = 0; s < ns; ++s) {
-        const uint32_t stream = stream_ids ? (*stream_ids)[s] : s0 + s;
-        if ((int64_t) stream >= tensor->ne[2]) {
-            throw std::runtime_error("DSV4 state tensor stream out of range");
-        }
-        const size_t offset = (size_t) stream*stream_stride;
+        const size_t offset = (size_t) (s0 + s)*stream_stride;
         io.write_tensor(tensor, offset, size);
     }
 }
@@ -430,9 +421,7 @@ static llama_kv_cache_dsv4_context::comp_plan dsv4_build_comp_plan(
         bool overlap,
         uint32_t state_size,
         uint32_t kv_size,
-        uint32_t n_stream,
-        uint32_t n_rs_seq,
-        const std::vector<uint32_t> & rs_idx) {
+        uint32_t n_stream) {
     llama_kv_cache_dsv4_context::comp_plan plan;
     plan.n_visible.resize(ubatch.n_tokens);
     plan.n_stream = dsv4_comp_graph_n_stream(ubatch, n_stream);
@@ -462,7 +451,6 @@ static llama_kv_cache_dsv4_context::comp_plan dsv4_build_comp_plan(
     std::vector<int32_t> overlap_cur_reads;
 
     std::map<std::pair<llama_seq_id, llama_pos>, int64_t> curr_token_idx_map;
-    std::map<llama_seq_id, uint32_t> state_write_counts;
 
     for (uint32_t i = 0; i < ubatch.n_tokens; ++i) {
         for (int32_t s = 0; s < ubatch.n_seq_id[i]; ++s) {
@@ -525,7 +513,6 @@ static llama_kv_cache_dsv4_context::comp_plan dsv4_build_comp_plan(
 
             plan.state_write_idxs.push_back(cache_off + pos/ratio);
             plan.state_write_pos.push_back((int32_t) source_start);
-            ++state_write_counts[seq_id];
 
             if (overlap) {
                 const llama_pos prev_start = source_start - ratio;
@@ -544,57 +531,33 @@ static llama_kv_cache_dsv4_context::comp_plan dsv4_build_comp_plan(
         }
     }
 
-    if (ratio == DSV4_CSA_RATIO && !plan.state_pos.empty()) {
+    if (ratio == DSV4_CSA_RATIO && plan.state_write_idxs.empty() && !plan.state_pos.empty()) {
+        // Non-boundary CSA steps still need a write op so their graph matches
+        // boundary steps. Use a padded scratch row that is masked from attention.
         assert(kv_size > 0);
 
-        // Pad each stream to the reserve plan's block count.
-        const auto append_dummy_block = [&](llama_seq_id seq_id, uint32_t i) {
-            const int64_t cache_off = dsv4_stream_offset(n_stream, seq_id, kv_size);
-            const int32_t source_idx = state_source_idx(seq_id, ubatch.pos[i]);
+        uint32_t i = 0;
+        while (i < ubatch.n_tokens && ubatch.pos[i] < 0) {
+            ++i;
+        }
+        assert(i < ubatch.n_tokens);
 
-            plan.state_write_idxs.push_back(cache_off + kv_size - 1);
-            plan.state_write_pos .push_back(0);
+        const llama_pos    pos    = ubatch.pos[i];
+        const llama_seq_id seq_id = ubatch.seq_id[i][0];
+        const int64_t cache_off = dsv4_stream_offset(n_stream, seq_id, kv_size);
+        const int32_t source_idx = state_source_idx(seq_id, pos);
 
-            if (overlap) {
-                for (uint32_t j = 0; j < ratio; ++j) {
-                    overlap_prev_reads.push_back(source_idx);
-                    overlap_cur_reads .push_back(source_idx);
-                }
-            } else {
-                for (uint32_t j = 0; j < ratio; ++j) {
-                    plan.state_read_idxs.push_back(source_idx);
-                }
-            }
-        };
+        plan.state_write_idxs.push_back(cache_off + kv_size - 1);
+        plan.state_write_pos .push_back(0);
 
-        if (dsv4_ubatch_has_coupled(ubatch)) {
-            if (plan.state_write_idxs.empty()) {
-                uint32_t i = 0;
-                while (i < ubatch.n_tokens && ubatch.pos[i] < 0) {
-                    ++i;
-                }
-                assert(i < ubatch.n_tokens);
-                append_dummy_block(ubatch.seq_id[i][0], i);
+        if (overlap) {
+            for (uint32_t j = 0; j < ratio; ++j) {
+                overlap_prev_reads.push_back(source_idx);
+                overlap_cur_reads .push_back(source_idx);
             }
         } else {
-            const uint32_t n_blocks = (std::max<uint32_t>(1, ubatch.n_seq_tokens) + ratio - 1)/ratio;
-
-            for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) {
-                const llama_seq_id seq_id = ubatch.seq_id_unq[s];
-                const uint32_t n_writes = state_write_counts[seq_id];
-                if (n_writes >= n_blocks) {
-                    continue;
-                }
-                if (n_writes + 1 != n_blocks) {
-                    throw std::runtime_error("DSV4 CSA sequence positions are not contiguous");
-                }
-
-                uint32_t i = 0;
-                while (i < ubatch.n_tokens && (ubatch.pos[i] < 0 || !dsv4_token_has_seq(ubatch, i, seq_id))) {
-                    ++i;
-                }
-                assert(i < ubatch.n_tokens);
-                append_dummy_block(seq_id, i);
+            for (uint32_t j = 0; j < ratio; ++j) {
+                plan.state_read_idxs.push_back(source_idx);
             }
         }
     }
@@ -620,63 +583,6 @@ static llama_kv_cache_dsv4_context::comp_plan dsv4_build_comp_plan(
         plan.state_persist_dst_idxs.push_back(row.dst);
     }
 
-
-    if (n_rs_seq > 0) {
-        for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) {
-            const llama_seq_id seq_id = ubatch.seq_id_unq[s];
-            if (seq_id < 0 || (uint32_t) seq_id >= n_stream) {
-                continue;
-            }
-
-            const int64_t stream_off = dsv4_stream_offset(n_stream, seq_id, state_size);
-            const uint32_t rollback = (uint32_t) seq_id < rs_idx.size() ? rs_idx[seq_id] : 0;
-            // Keep the restore graph fixed-width when no rollback is pending.
-            const int64_t src_plane = rollback > 0 && rollback <= n_rs_seq ? (int64_t) rollback*state_rows : 0;
-            for (uint32_t r = 0; r < state_size; ++r) {
-                plan.state_restore_src_idxs.push_back((int32_t) (src_plane + stream_off + r));
-                plan.state_restore_dst_idxs.push_back((int32_t) (stream_off + r));
-            }
-
-            std::vector<uint32_t> token_idxs;
-            token_idxs.reserve(ubatch.n_tokens);
-            for (uint32_t i = 0; i < ubatch.n_tokens; ++i) {
-                if (dsv4_token_has_seq(ubatch, i, seq_id)) {
-                    token_idxs.push_back(i);
-                }
-            }
-            if (token_idxs.empty()) {
-                continue;
-            }
-
-            const uint32_t n_seq_tokens = (uint32_t) token_idxs.size();
-            const int64_t scratch_off = (int64_t) state_rows*(1 + n_rs_seq);
-            for (uint32_t d = 1; d <= n_rs_seq; ++d) {
-                const int64_t dst_plane = (int64_t) d*state_rows;
-
-                for (uint32_t r = 0; r < state_size; ++r) {
-                    int32_t src;
-                    if (d <= n_seq_tokens) {
-                        const uint32_t prefix = n_seq_tokens - d;
-                        src = (int32_t) (stream_off + r);
-
-                        for (uint32_t j = 0; j < prefix; ++j) {
-                            const uint32_t i_tok = token_idxs[j];
-                            if (ubatch.pos[i_tok] >= 0 && (uint32_t) (ubatch.pos[i_tok]%state_size) == r) {
-                                src = (int32_t) (scratch_off + i_tok);
-                            }
-                        }
-                    } else {
-                        const int64_t src_plane = (int64_t) (d - n_seq_tokens)*state_rows;
-                        src = (int32_t) (src_plane + stream_off + r);
-                    }
-
-                    plan.state_snapshot_src_idxs.push_back(src);
-                    plan.state_snapshot_dst_idxs.push_back((int32_t) (dst_plane + stream_off + r));
-                }
-            }
-        }
-    }
-
     static const bool debug = []() {
         const char * env = getenv("LLAMA_DSV4_COMPRESS_DEBUG");
         return env && atoi(env) > 0;
@@ -698,14 +604,12 @@ static std::vector<llama_kv_cache_dsv4_context::comp_plan> dsv4_build_comp_plans
         bool overlap,
         uint32_t state_size,
         uint32_t kv_size,
-        uint32_t n_stream,
-        uint32_t n_rs_seq,
-        const std::vector<uint32_t> & rs_idx) {
+        uint32_t n_stream) {
     std::vector<llama_kv_cache_dsv4_context::comp_plan> plans;
     plans.reserve(ubatches.size());
 
     for (const llama_ubatch & ubatch : ubatches) {
-        plans.push_back(dsv4_build_comp_plan(ubatch, ratio, overlap, state_size, kv_size, n_stream, n_rs_seq, rs_idx));
+        plans.push_back(dsv4_build_comp_plan(ubatch, ratio, overlap, state_size, kv_size, n_stream));
     }
 
     return plans;
@@ -792,8 +696,7 @@ static llama_kv_cache_dsv4_context::comp_plan dsv4_build_reserve_comp_plan(
         bool overlap,
         uint32_t state_size,
         uint32_t kv_size,
-        uint32_t n_stream,
-        uint32_t n_rs_seq) {
+        uint32_t n_stream) {
     llama_kv_cache_dsv4_context::comp_plan plan;
     plan.n_visible.resize(ubatch.n_tokens);
     plan.n_stream = dsv4_comp_graph_n_stream(ubatch, n_stream);
@@ -811,16 +714,10 @@ static llama_kv_cache_dsv4_context::comp_plan dsv4_build_reserve_comp_plan(
 
     const uint64_t state_rows = (uint64_t) state_size*n_stream;
     const size_t n_persist = (size_t) std::min<uint64_t>(ubatch.n_tokens, state_rows);
-    const size_t n_restore = n_rs_seq > 0 ? (size_t) state_size*std::max<uint32_t>(1, ubatch.n_seqs_unq) : 0;
-    const size_t n_snapshot = (size_t) n_rs_seq*state_size*std::max<uint32_t>(1, ubatch.n_seqs_unq);
 
     plan.state_pos .resize(ubatch.n_tokens);
     plan.state_persist_src_idxs.resize(n_persist);
     plan.state_persist_dst_idxs.resize(n_persist);
-    plan.state_restore_src_idxs.resize(n_restore);
-    plan.state_restore_dst_idxs.resize(n_restore);
-    plan.state_snapshot_src_idxs.resize(n_snapshot);
-    plan.state_snapshot_dst_idxs.resize(n_snapshot);
     plan.state_read_idxs .resize((overlap ? 2u : 1u)*ratio*n_blocks);
     plan.state_write_idxs.resize(n_blocks);
     plan.state_write_pos .resize(n_blocks);
@@ -846,14 +743,12 @@ llama_dsv4_comp_state::llama_dsv4_comp_state(
             uint32_t        ratio,
             uint32_t        state_size,
             uint32_t        n_embd_state,
-            uint32_t        n_rs_seq,
         const char    * name,
         const llama_memory_i::layer_filter_cb & filter) :
     ratio(ratio),
     state_size(state_size),
     n_embd_state(n_embd_state),
-    n_stream(unified ? 1 : n_seq_max),
-    n_rs_seq(n_rs_seq) {
+    n_stream(unified ? 1 : n_seq_max) {
     const llama_hparams & hparams = model.hparams;
 
     struct ggml_backend_buft_comparator {
@@ -909,9 +804,8 @@ llama_dsv4_comp_state::llama_dsv4_comp_state(
             throw std::runtime_error("failed to create ggml context for DSV4 compressor state");
         }
 
-        const uint32_t n_planes = n_stream*(1 + n_rs_seq);
-        ggml_tensor * kv    = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd_state, state_size, n_planes);
-        ggml_tensor * score = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd_state, state_size, n_planes);
+        ggml_tensor * kv    = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd_state, state_size, n_stream);
+        ggml_tensor * score = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd_state, state_size, n_stream);
 
         ggml_format_name(kv,    "dsv4_%s_state_kv_l%d",    name, il);
         ggml_format_name(score, "dsv4_%s_state_score_l%d", name, il);
@@ -943,8 +837,8 @@ llama_dsv4_comp_state::llama_dsv4_comp_state(
         ctxs_bufs.emplace_back(std::move(ctx), buf);
     }
 
-    LLAMA_LOG_INFO("%s: %s ratio = %u, state = %u x %u, streams = %u, rs_seq = %u, layers = %zu, size = %7.2f MiB\n",
-            __func__, name, ratio, state_size, n_embd_state, n_stream, n_rs_seq, layers.size(), total_size()/1024.0/1024.0);
+    LLAMA_LOG_INFO("%s: %s ratio = %u, state = %u x %u, streams = %u, layers = %zu, size = %7.2f MiB\n",
+            __func__, name, ratio, state_size, n_embd_state, n_stream, layers.size(), total_size()/1024.0/1024.0);
 }
 
 void llama_dsv4_comp_state::clear(llama_seq_id seq_id, bool data) {
@@ -954,13 +848,9 @@ void llama_dsv4_comp_state::clear(llama_seq_id seq_id, bool data) {
 
     if (seq_id >= 0) {
         GGML_ASSERT((uint32_t) seq_id < n_stream);
-
         for (const auto & layer : layers) {
-            for (uint32_t d = 0; d <= n_rs_seq; ++d) {
-                const uint32_t stream = d*n_stream + (uint32_t) seq_id;
-                dsv4_clear_tensor_stream(layer.kv,    stream);
-                dsv4_clear_tensor_stream(layer.score, stream);
-            }
+            dsv4_clear_tensor_stream(layer.kv,    (uint32_t) seq_id);
+            dsv4_clear_tensor_stream(layer.score, (uint32_t) seq_id);
         }
         return;
     }
@@ -978,8 +868,6 @@ void llama_dsv4_comp_state::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_
         return;
     }
 
-    clear(seq_id_dst, true);
-
     sc_info.ssrc.push_back((uint32_t) seq_id_src);
     sc_info.sdst.push_back((uint32_t) seq_id_dst);
 }
@@ -1008,14 +896,6 @@ uint32_t llama_dsv4_comp_state::get_n_stream() const {
     return n_stream;
 }
 
-uint32_t llama_dsv4_comp_state::get_n_rs_seq() const {
-    return n_rs_seq;
-}
-
-uint32_t llama_dsv4_comp_state::get_n_rows() const {
-    return state_size*n_stream;
-}
-
 std::map<ggml_backend_buffer_type_t, size_t> llama_dsv4_comp_state::memory_breakdown() const {
     std::map<ggml_backend_buffer_type_t, size_t> ret;
     for (const auto & [_, buf] : ctxs_bufs) {
@@ -1025,26 +905,13 @@ std::map<ggml_backend_buffer_type_t, size_t> llama_dsv4_comp_state::memory_break
     return ret;
 }
 
-void llama_dsv4_comp_state::state_write(
-        llama_io_write_i & io,
-        llama_seq_id seq_id,
-        llama_state_seq_flags flags,
-        const std::vector<uint32_t> & rs_idx) const {
+void llama_dsv4_comp_state::state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const {
     GGML_UNUSED(flags);
 
     uint32_t s0;
     uint32_t ns;
     dsv4_state_src_stream_range(n_stream, seq_id, s0, ns);
 
-    std::vector<uint32_t> stream_ids(ns);
-    for (uint32_t s = 0; s < ns; ++s) {
-        const uint32_t seq = seq_id >= 0 ? (uint32_t) seq_id : s0 + s;
-        if (seq >= rs_idx.size() || rs_idx[seq] > n_rs_seq) {
-            throw std::runtime_error("DSV4 recurrent state rollback index out of range");
-        }
-        stream_ids[s] = rs_idx[seq]*n_stream + s0 + s;
-    }
-
     const uint32_t version      = DSV4_COMP_STATE_VER;
     const uint32_t n_layer      = layers.size();
 
@@ -1058,8 +925,8 @@ void llama_dsv4_comp_state::state_write(
     for (const auto & layer : layers) {
         io.write(&layer.il, sizeof(layer.il));
 
-        dsv4_state_write_tensor_streams(io, layer.kv,    state_size, state_size, s0, ns, &stream_ids);
-        dsv4_state_write_tensor_streams(io, layer.score, state_size, state_size, s0, ns, &stream_ids);
+        dsv4_state_write_tensor_streams(io, layer.kv,    state_size, state_size, s0, ns);
+        dsv4_state_write_tensor_streams(io, layer.score, state_size, state_size, s0, ns);
     }
 }
 
@@ -1105,40 +972,28 @@ void llama_dsv4_comp_state::state_read(llama_io_read_i & io, llama_seq_id seq_id
     }
 }
 
-ggml_tensor * llama_dsv4_comp_state::get_kv_all(ggml_context * ctx, int32_t il) const {
+ggml_tensor * llama_dsv4_comp_state::get_kv(ggml_context * ctx, int32_t il) const {
     const int32_t ids = map_layer_ids.at(il);
+
     ggml_tensor * state = layers[ids].kv;
 
-    return ggml_view_2d(ctx, state, state->ne[0], get_n_rows()*(1 + n_rs_seq), state->nb[1], 0);
-}
-
-ggml_tensor * llama_dsv4_comp_state::get_score_all(ggml_context * ctx, int32_t il) const {
-    const int32_t ids = map_layer_ids.at(il);
-    ggml_tensor * state = layers[ids].score;
-
-    return ggml_view_2d(ctx, state, state->ne[0], get_n_rows()*(1 + n_rs_seq), state->nb[1], 0);
-}
-
-ggml_tensor * llama_dsv4_comp_state::get_kv(ggml_context * ctx, int32_t il) const {
-    ggml_tensor * state = get_kv_all(ctx, il);
-    const size_t row_size = ggml_row_size(state->type, state->ne[0]);
-
-    return ggml_view_2d(ctx, state, state->ne[0], get_n_rows(), state->nb[1], 0*row_size);
+    return ggml_reshape_2d(ctx, state, state->ne[0], state->ne[1]*state->ne[2]);
 }
 
 ggml_tensor * llama_dsv4_comp_state::get_score(ggml_context * ctx, int32_t il) const {
-    ggml_tensor * state = get_score_all(ctx, il);
-    const size_t row_size = ggml_row_size(state->type, state->ne[0]);
+    const int32_t ids = map_layer_ids.at(il);
 
-    return ggml_view_2d(ctx, state, state->ne[0], get_n_rows(), state->nb[1], 0*row_size);
+    ggml_tensor * state = layers[ids].score;
+
+    return ggml_reshape_2d(ctx, state, state->ne[0], state->ne[1]*state->ne[2]);
 }
 
 ggml_tensor * llama_dsv4_comp_state::cpy_kv(ggml_context * ctx, ggml_tensor * cur, ggml_tensor * idxs, int32_t il) const {
-    return ggml_set_rows(ctx, get_kv_all(ctx, il), cur, idxs);
+    return ggml_set_rows(ctx, get_kv(ctx, il), cur, idxs);
 }
 
 ggml_tensor * llama_dsv4_comp_state::cpy_score(ggml_context * ctx, ggml_tensor * cur, ggml_tensor * idxs, int32_t il) const {
-    return ggml_set_rows(ctx, get_score_all(ctx, il), cur, idxs);
+    return ggml_set_rows(ctx, get_score(ctx, il), cur, idxs);
 }
 
 size_t llama_dsv4_comp_state::total_size() const {
@@ -1167,16 +1022,13 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
                  uint32_t   n_seq_max,
                  uint32_t   n_ubatch,
                  uint32_t   n_pad,
-                 uint32_t   n_rs_seq,
     const layer_filter_cb & filter,
     const  layer_reuse_cb & reuse) :
     hparams_raw(model.hparams),
     hparams_csa(model.hparams),
     hparams_hca(model.hparams),
     hparams_lid(model.hparams),
-    n_seq_max(n_seq_max),
-    n_rs_seq(n_rs_seq),
-    rs_idx(n_seq_max, 0) {
+    n_seq_max(n_seq_max) {
 
     const layer_filter_cb filter_raw = [&](int32_t il) {
         if (filter && !filter(il)) {
@@ -1191,11 +1043,6 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
     // Keep DSV4 KV/state streams per sequence even when public KV mode is unified.
     const bool unified_raw = false;
 
-    hparams_raw.n_layer_nextn = 0;
-    hparams_csa.n_layer_nextn = 0;
-    hparams_hca.n_layer_nextn = 0;
-    hparams_lid.n_layer_nextn = 0;
-
     LLAMA_LOG_INFO("%s: creating DSV4 raw KV cache\n", __func__);
 
     dsv4_make_k_only(hparams_raw);
@@ -1262,19 +1109,19 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
 
     csa_state = std::make_unique<llama_dsv4_comp_state>(
             model, offload, unified_compressed, n_seq_max, DSV4_CSA_RATIO, 2*DSV4_CSA_RATIO,
-            2*model.hparams.n_embd_head_k(), n_rs_seq, "csa", filter_csa);
+            2*model.hparams.n_embd_head_k(), "csa", filter_csa);
 
     LLAMA_LOG_INFO("%s: creating DSV4 HCA compressor state\n", __func__);
 
     hca_state = std::make_unique<llama_dsv4_comp_state>(
             model, offload, unified_compressed, n_seq_max, DSV4_HCA_RATIO, DSV4_HCA_RATIO,
-            model.hparams.n_embd_head_k(), n_rs_seq, "hca", filter_hca);
+            model.hparams.n_embd_head_k(), "hca", filter_hca);
 
     LLAMA_LOG_INFO("%s: creating DSV4 lightning-indexer compressor state\n", __func__);
 
     lid_state = std::make_unique<llama_dsv4_comp_state>(
             model, offload, unified_compressed, n_seq_max, DSV4_CSA_RATIO, 2*DSV4_CSA_RATIO,
-            2*model.hparams.indexer_head_size, n_rs_seq, "lid", filter_csa);
+            2*model.hparams.indexer_head_size, "lid", filter_csa);
 
     // DSV4 attention reads compressed-K / compressor-state rows that the current
     // graph does not necessarily overwrite; uninitialized buffer contents would
@@ -1408,35 +1255,17 @@ bool llama_kv_cache_dsv4::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1
     }
 
     if (p0 > 0) {
-        if (seq_id < 0 || (uint32_t) seq_id >= n_seq_max) {
+        if (seq_id < 0 || (uint32_t) seq_id >= n_seq_max ||
+                p0 <= kv_raw->seq_pos_max(seq_id)) {
             return false;
         }
 
-        const llama_pos pos_max = kv_raw->seq_pos_max(seq_id);
-        if (p0 > pos_max) {
-            bool res = true;
+        bool res = true;
 
-            res = res & kv_raw->seq_rm(seq_id, p0, -1);
-            res = res & kv_csa->seq_rm(seq_id, p0/DSV4_CSA_RATIO, -1);
-            res = res & kv_hca->seq_rm(seq_id, p0/DSV4_HCA_RATIO, -1);
-            res = res & kv_lid->seq_rm(seq_id, p0/DSV4_CSA_RATIO, -1);
-
-            return res;
-        }
-
-        if (n_rs_seq == 0) {
-            return false;
-        }
-
-        const llama_pos rollback = pos_max - (p0 - 1);
-        if (rollback < 1 || rollback > (llama_pos) n_rs_seq) {
-            return false;
-        }
-
-        const bool res = kv_raw->seq_rm(seq_id, p0, p1);
-        if (res) {
-            rs_idx[seq_id] = (uint32_t) rollback;
-        }
+        res = res & kv_raw->seq_rm(seq_id, p0, -1);
+        res = res & kv_csa->seq_rm(seq_id, p0/DSV4_CSA_RATIO, -1);
+        res = res & kv_hca->seq_rm(seq_id, p0/DSV4_HCA_RATIO, -1);
+        res = res & kv_lid->seq_rm(seq_id, p0/DSV4_CSA_RATIO, -1);
 
         return res;
     }
@@ -1461,10 +1290,6 @@ void llama_kv_cache_dsv4::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_ds
     csa_state->seq_cp(seq_id_src, seq_id_dst);
     hca_state->seq_cp(seq_id_src, seq_id_dst);
     lid_state->seq_cp(seq_id_src, seq_id_dst);
-
-    if (seq_id_src != seq_id_dst) {
-        rs_idx[seq_id_dst] = 0;
-    }
 }
 
 void llama_kv_cache_dsv4::seq_keep(llama_seq_id seq_id) {
@@ -1561,9 +1386,9 @@ void llama_kv_cache_dsv4::state_write(llama_io_write_i & io, llama_seq_id seq_id
         dsv4_state_write_k_cache(io, kv_lid.get(), seq_id, flags, n_rows_lid);
     }
 
-    csa_state->state_write(io, seq_id, flags, rs_idx);
-    hca_state->state_write(io, seq_id, flags, rs_idx);
-    lid_state->state_write(io, seq_id, flags, rs_idx);
+    csa_state->state_write(io, seq_id, flags);
+    hca_state->state_write(io, seq_id, flags);
+    lid_state->state_write(io, seq_id, flags);
 }
 
 void llama_kv_cache_dsv4::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) {
@@ -1607,12 +1432,6 @@ void llama_kv_cache_dsv4::state_read(llama_io_read_i & io, llama_seq_id seq_id,
     hca_state->state_read(io, seq_id, flags);
     lid_state->state_read(io, seq_id, flags);
 
-    if (seq_id >= 0) {
-        GGML_ASSERT((uint32_t) seq_id < n_seq_max);
-        rs_idx[seq_id] = 0;
-    } else {
-        std::fill(rs_idx.begin(), rs_idx.end(), 0);
-    }
 }
 
 llama_kv_cache_iswa * llama_kv_cache_dsv4::get_raw() const {
@@ -1643,31 +1462,6 @@ llama_dsv4_comp_state * llama_kv_cache_dsv4::get_lid_state() const {
     return lid_state.get();
 }
 
-uint32_t llama_kv_cache_dsv4::get_n_rs_seq() const {
-    return n_rs_seq;
-}
-
-const std::vector<uint32_t> & llama_kv_cache_dsv4::get_rs_idx() const {
-    return rs_idx;
-}
-
-void llama_kv_cache_dsv4::reset_rs_idx_for_ubatches(const std::vector<llama_ubatch> & ubatches) {
-    if (n_rs_seq == 0) {
-        return;
-    }
-
-    for (const llama_ubatch & ubatch : ubatches) {
-        for (uint32_t i = 0; i < ubatch.n_tokens; ++i) {
-            for (int32_t s = 0; s < ubatch.n_seq_id[i]; ++s) {
-                const llama_seq_id seq_id = ubatch.seq_id[i][s];
-                if (seq_id >= 0 && (uint32_t) seq_id < n_seq_max) {
-                    rs_idx[seq_id] = 0;
-                }
-            }
-        }
-    }
-}
-
 void llama_kv_cache_dsv4::clear_compressed(llama_seq_id seq_id, bool data) {
     if (seq_id < 0) {
         kv_csa->clear(data);
@@ -1694,12 +1488,6 @@ void llama_kv_cache_dsv4::clear_compressed(llama_seq_id seq_id, bool data) {
     csa_state->clear(seq_id, data);
     hca_state->clear(seq_id, data);
     lid_state->clear(seq_id, data);
-
-    if (seq_id >= 0) {
-        rs_idx[seq_id] = 0;
-    } else {
-        std::fill(rs_idx.begin(), rs_idx.end(), 0);
-    }
 }
 
 //
@@ -1991,14 +1779,10 @@ llama_kv_cache_dsv4_context::llama_kv_cache_dsv4_context(
         std::vector<llama_ubatch> ubatches_raw) :
     ubatches(std::move(ubatches)),
     plans_csa(dsv4_build_comp_plans(this->ubatches, DSV4_CSA_RATIO, true,
-                kv->get_csa_state()->get_state_size(), kv->get_csa()->get_size(), kv->get_csa_state()->get_n_stream(),
-                kv->get_n_rs_seq(), kv->get_rs_idx())),
+                kv->get_csa_state()->get_state_size(), kv->get_csa()->get_size(), kv->get_csa_state()->get_n_stream())),
     plans_hca(dsv4_build_comp_plans(this->ubatches, DSV4_HCA_RATIO, false,
-                kv->get_hca_state()->get_state_size(), kv->get_hca()->get_size(), kv->get_hca_state()->get_n_stream(),
-                kv->get_n_rs_seq(), kv->get_rs_idx())),
-    plans_lid(dsv4_build_comp_plans(this->ubatches, DSV4_CSA_RATIO, true,
-                kv->get_lid_state()->get_state_size(), kv->get_lid()->get_size(), kv->get_lid_state()->get_n_stream(),
-                kv->get_n_rs_seq(), kv->get_rs_idx())),
+                kv->get_hca_state()->get_state_size(), kv->get_hca()->get_size(), kv->get_hca_state()->get_n_stream())),
+    plans_lid(plans_csa),
     ctx_raw(std::make_unique<llama_kv_cache_dsv4_raw_context>(
                 kv->get_raw(),
                 std::move(sinfos_raw_base_write),
@@ -2025,7 +1809,6 @@ llama_kv_cache_dsv4_context::llama_kv_cache_dsv4_context(
     hca_state(kv->get_hca_state()),
     lid_state(kv->get_lid_state()),
     status(ctx_raw->get_status()) {
-    kv->reset_rs_idx_for_ubatches(this->ubatches);
 }
 
 llama_kv_cache_dsv4_context::~llama_kv_cache_dsv4_context() = default;
@@ -2161,7 +1944,7 @@ const llama_kv_cache_dsv4_context::comp_plan & llama_kv_cache_dsv4_context::get_
 
     reserve_plan_csa = dsv4_build_reserve_comp_plan(
             ubatch, DSV4_CSA_RATIO, true,
-            csa_state->get_state_size(), get_csa()->get_n_kv(), csa_state->get_n_stream(), csa_state->get_n_rs_seq());
+            csa_state->get_state_size(), get_csa()->get_n_kv(), csa_state->get_n_stream());
 
     return reserve_plan_csa;
 }
@@ -2175,7 +1958,7 @@ const llama_kv_cache_dsv4_context::comp_plan & llama_kv_cache_dsv4_context::get_
 
     reserve_plan_hca = dsv4_build_reserve_comp_plan(
             ubatch, DSV4_HCA_RATIO, false,
-            hca_state->get_state_size(), get_hca()->get_n_kv(), hca_state->get_n_stream(), hca_state->get_n_rs_seq());
+            hca_state->get_state_size(), get_hca()->get_n_kv(), hca_state->get_n_stream());
 
     return reserve_plan_hca;
 }
@@ -2189,7 +1972,7 @@ const llama_kv_cache_dsv4_context::comp_plan & llama_kv_cache_dsv4_context::get_
 
     reserve_plan_lid = dsv4_build_reserve_comp_plan(
             ubatch, DSV4_CSA_RATIO, true,
-            lid_state->get_state_size(), get_lid()->get_n_kv(), lid_state->get_n_stream(), lid_state->get_n_rs_seq());
+            lid_state->get_state_size(), get_lid()->get_n_kv(), lid_state->get_n_stream());
 
     return reserve_plan_lid;
 }

src/llama-kv-cache-dsv4.h

@@ -22,7 +22,6 @@ public:
             uint32_t        ratio,
             uint32_t        state_size,
             uint32_t        n_embd_state,
-            uint32_t        n_rs_seq,
             const char    * name,
         const llama_memory_i::layer_filter_cb & filter);
 
@@ -30,21 +29,17 @@ public:
     void seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst);
     void apply_copies(const stream_copy_info & sc_info) const;
 
-    uint32_t get_ratio()      const;
+    uint32_t get_ratio()    const;
     uint32_t get_state_size() const;
-    uint32_t get_n_stream()   const;
-    uint32_t get_n_rs_seq()   const;
-    uint32_t get_n_rows()     const;
+    uint32_t get_n_stream() const;
 
     std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const;
 
-    void state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags, const std::vector<uint32_t> & rs_idx) const;
+    void state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const;
     void state_read (llama_io_read_i  & io, llama_seq_id seq_id, llama_state_seq_flags flags);
 
-    ggml_tensor * get_kv       (ggml_context * ctx, int32_t il) const;
-    ggml_tensor * get_score    (ggml_context * ctx, int32_t il) const;
-    ggml_tensor * get_kv_all   (ggml_context * ctx, int32_t il) const;
-    ggml_tensor * get_score_all(ggml_context * ctx, int32_t il) const;
+    ggml_tensor * get_kv   (ggml_context * ctx, int32_t il) const;
+    ggml_tensor * get_score(ggml_context * ctx, int32_t il) const;
 
     ggml_tensor * cpy_kv   (ggml_context * ctx, ggml_tensor * cur, ggml_tensor * idxs, int32_t il) const;
     ggml_tensor * cpy_score(ggml_context * ctx, ggml_tensor * cur, ggml_tensor * idxs, int32_t il) const;
@@ -64,7 +59,6 @@ private:
     const uint32_t state_size;
     const uint32_t n_embd_state;
     const uint32_t n_stream;
-    const uint32_t n_rs_seq;
 
     std::vector<std::pair<ggml_context_ptr, ggml_backend_buffer_ptr>> ctxs_bufs;
 
@@ -99,7 +93,6 @@ public:
                      uint32_t   n_seq_max,
                      uint32_t   n_ubatch,
                      uint32_t   n_pad,
-                     uint32_t   n_rs_seq,
         const layer_filter_cb & filter,
         const  layer_reuse_cb & reuse);
 
@@ -148,10 +141,6 @@ public:
     llama_dsv4_comp_state * get_hca_state() const;
     llama_dsv4_comp_state * get_lid_state() const;
 
-    uint32_t get_n_rs_seq() const;
-    const std::vector<uint32_t> & get_rs_idx() const;
-    void reset_rs_idx_for_ubatches(const std::vector<llama_ubatch> & ubatches);
-
 private:
     llama_hparams hparams_raw;
     llama_hparams hparams_csa;
@@ -159,9 +148,6 @@ private:
     llama_hparams hparams_lid;
 
     const uint32_t n_seq_max;
-    const uint32_t n_rs_seq;
-
-    std::vector<uint32_t> rs_idx;
 
     std::unique_ptr<llama_kv_cache_iswa> kv_raw;
     std::unique_ptr<llama_kv_cache>      kv_csa;
@@ -282,17 +268,6 @@ public:
         std::vector<int32_t> state_persist_src_idxs;
         std::vector<int32_t> state_persist_dst_idxs;
 
-        // Device-side rollback restore copies snapshot planes back to the
-        // current compressor-state plane before the graph reads it.
-        std::vector<int32_t> state_restore_src_idxs;
-        std::vector<int32_t> state_restore_dst_idxs;
-
-        // Device-side rollback snapshots copy rows from the graph-local
-        // [persistent_state | current_ubatch_scratch] tensor into rollback
-        // planes after the graph has computed current-token compressor state.
-        std::vector<int32_t> state_snapshot_src_idxs;
-        std::vector<int32_t> state_snapshot_dst_idxs;
-
         // Flattened source row ids used for state-backed commits. Source rows
         // index the graph-local [persistent_state | current_ubatch_scratch]
         // tensor. For overlapped compression the first half is previous rows

src/llama-kv-cache-msa.cpp

@@ -1,395 +0,0 @@
-#include "llama-kv-cache-msa.h"
-
-#include "llama-impl.h"
-#include "llama-batch.h"
-#include "llama-model.h"
-
-#include <algorithm>
-#include <cassert>
-#include <cmath>
-
-// llama_kv_cache_msa
-
-llama_kv_cache_msa::llama_kv_cache_msa(
-        const llama_model & model,
-                ggml_type   type_k,
-                ggml_type   type_v,
-                     bool   v_trans,
-                     bool   offload,
-                     bool   unified,
-                 uint32_t   kv_size,
-                 uint32_t   n_seq_max,
-                 uint32_t   n_pad,
-                 uint32_t   n_swa,
-           llama_swa_type   swa_type,
-    const layer_filter_cb & filter,
-    const layer_filter_cb & filter_idx,
-    const  layer_reuse_cb & reuse) :
-    hparams_idx(model.hparams),
-    n_stream(unified ? 1 : n_seq_max), n_seq_max(n_seq_max), n_pad(n_pad),
-    n_swa(n_swa), swa_type(swa_type) {
-
-    LLAMA_LOG_INFO("%s: creating main KV cache, size = %u cells\n", __func__, kv_size);
-
-    kv_base = std::make_unique<llama_kv_cache>(
-            model, model.hparams, type_k, type_v,
-            v_trans, offload, unified, kv_size, n_seq_max, n_pad,
-            n_swa, swa_type, nullptr, filter, reuse, nullptr);
-
-    // the MSA indexer uses a single key head per layer
-    std::fill(hparams_idx.n_head_kv_arr.begin(), hparams_idx.n_head_kv_arr.end(), 1);
-    hparams_idx.n_embd_head_k_full = model.hparams.indexer_head_size;
-    // the rope parameters are kept identical to the main cache
-
-    LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size);
-
-    kv_idx = std::make_unique<llama_kv_cache>(
-            model, hparams_idx, type_k, type_v,
-            v_trans, offload, unified, kv_size, n_seq_max, n_pad,
-            n_swa, swa_type, nullptr, filter_idx, reuse, nullptr);
-}
-
-void llama_kv_cache_msa::clear(bool data) {
-    kv_base->clear(data);
-    kv_idx ->clear(data);
-}
-
-bool llama_kv_cache_msa::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) {
-    bool res = true;
-
-    res = res & kv_base->seq_rm(seq_id, p0, p1);
-    res = res & kv_idx ->seq_rm(seq_id, p0, p1);
-
-    return res;
-}
-
-void llama_kv_cache_msa::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) {
-    kv_base->seq_cp(seq_id_src, seq_id_dst, p0, p1);
-    kv_idx ->seq_cp(seq_id_src, seq_id_dst, p0, p1);
-}
-
-void llama_kv_cache_msa::seq_keep(llama_seq_id seq_id) {
-    kv_base->seq_keep(seq_id);
-    kv_idx ->seq_keep(seq_id);
-}
-
-void llama_kv_cache_msa::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) {
-    kv_base->seq_add(seq_id, p0, p1, shift);
-    kv_idx ->seq_add(seq_id, p0, p1, shift);
-}
-
-void llama_kv_cache_msa::seq_div(llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) {
-    kv_base->seq_div(seq_id, p0, p1, d);
-    kv_idx ->seq_div(seq_id, p0, p1, d);
-}
-
-llama_pos llama_kv_cache_msa::seq_pos_min(llama_seq_id seq_id) const {
-    return kv_base->seq_pos_min(seq_id);
-}
-
-llama_pos llama_kv_cache_msa::seq_pos_max(llama_seq_id seq_id) const {
-    return kv_base->seq_pos_max(seq_id);
-}
-
-std::map<ggml_backend_buffer_type_t, size_t> llama_kv_cache_msa::memory_breakdown() const {
-    std::map<ggml_backend_buffer_type_t, size_t> mb = kv_base->memory_breakdown();
-    for (const auto & buft_size : kv_idx->memory_breakdown()) {
-        mb[buft_size.first] += buft_size.second;
-    }
-    return mb;
-}
-
-llama_memory_context_ptr llama_kv_cache_msa::init_batch(
-            llama_batch_allocr & balloc,
-            uint32_t n_ubatch,
-            bool embd_all) {
-    GGML_UNUSED(embd_all);
-
-    do {
-        balloc.split_reset();
-
-        std::vector<llama_ubatch> ubatches;
-        while (true) {
-            auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true, 0);
-
-            if (ubatch.n_tokens == 0) {
-                break;
-            }
-
-            ubatches.push_back(std::move(ubatch));
-        }
-
-        if (balloc.get_n_used() < balloc.get_n_tokens()) {
-            // failed to find a suitable split
-            break;
-        }
-
-        auto sinfos_base = kv_base->prepare(ubatches);
-        if (sinfos_base.empty()) {
-            break;
-        }
-
-        auto sinfos_idx = kv_idx->prepare(ubatches);
-        if (sinfos_idx.empty()) {
-            break;
-        }
-
-        assert(sinfos_base.size() == sinfos_idx.size());
-
-        return std::make_unique<llama_kv_cache_msa_context>(
-                this, std::move(sinfos_base), std::move(sinfos_idx), std::move(ubatches));
-    } while (false);
-
-    return std::make_unique<llama_kv_cache_msa_context>(LLAMA_MEMORY_STATUS_FAILED_PREPARE);
-}
-
-llama_memory_context_ptr llama_kv_cache_msa::init_full() {
-    return std::make_unique<llama_kv_cache_msa_context>(this);
-}
-
-llama_memory_context_ptr llama_kv_cache_msa::init_update(llama_context * lctx, bool optimize) {
-    return std::make_unique<llama_kv_cache_msa_context>(this, lctx, optimize);
-}
-
-bool llama_kv_cache_msa::get_can_shift() const {
-    return kv_base->get_can_shift() &&
-           kv_idx ->get_can_shift() &&
-           kv_base->get_size() == kv_idx->get_size();
-}
-
-void llama_kv_cache_msa::state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const {
-    kv_base->state_write(io, seq_id, flags);
-    kv_idx ->state_write(io, seq_id, flags);
-}
-
-void llama_kv_cache_msa::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) {
-    kv_base->state_read(io, seq_id, flags);
-    kv_idx ->state_read(io, seq_id, flags);
-}
-
-llama_kv_cache * llama_kv_cache_msa::get_base() const {
-    return kv_base.get();
-}
-
-llama_kv_cache * llama_kv_cache_msa::get_idx() const {
-    return kv_idx.get();
-}
-
-// llama_kv_cache_msa_context
-
-llama_kv_cache_msa_context::llama_kv_cache_msa_context(llama_memory_status status) :
-    kv(nullptr), status(status) {}
-
-llama_kv_cache_msa_context::llama_kv_cache_msa_context(
-        llama_kv_cache_msa * kv) :
-    kv(kv),
-    ctx_base(kv->get_base()->init_full()),
-    ctx_idx (kv->get_idx ()->init_full()),
-    status(llama_memory_status_combine(ctx_base->get_status(), ctx_idx->get_status())) {
-}
-
-llama_kv_cache_msa_context::llama_kv_cache_msa_context(
-        llama_kv_cache_msa * kv,
-        llama_context * lctx,
-        bool optimize) :
-    kv(kv),
-    ctx_base(kv->get_base()->init_update(lctx, optimize)),
-    ctx_idx (kv->get_idx ()->init_update(lctx, optimize)),
-    status(llama_memory_status_combine(ctx_base->get_status(), ctx_idx->get_status())) {
-}
-
-llama_kv_cache_msa_context::llama_kv_cache_msa_context(
-        llama_kv_cache_msa * kv,
-        slot_info_vec_t sinfos_base,
-        slot_info_vec_t sinfos_idx,
-        std::vector<llama_ubatch> ubatches) :
-    kv(kv),
-    ubatches(std::move(ubatches)),
-    // here we copy the ubatches. not sure if this is ideal
-    ctx_base(new llama_kv_cache_context(kv->get_base(), std::move(sinfos_base), this->ubatches)),
-    ctx_idx (new llama_kv_cache_context(kv->get_idx (), std::move(sinfos_idx),  this->ubatches)),
-    status(llama_memory_status_combine(ctx_base->get_status(), ctx_idx->get_status())) {
-}
-
-llama_kv_cache_msa_context::~llama_kv_cache_msa_context() = default;
-
-bool llama_kv_cache_msa_context::next() {
-    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);
-
-    ctx_base->next();
-    ctx_idx ->next();
-
-    if (++i_next >= ubatches.size()) {
-        return false;
-    }
-
-    return true;
-}
-
-bool llama_kv_cache_msa_context::apply() {
-    assert(!llama_memory_status_is_fail(status));
-
-    bool res = true;
-
-    res = res & ctx_base->apply();
-    res = res & ctx_idx ->apply();
-
-    return res;
-}
-
-llama_memory_status llama_kv_cache_msa_context::get_status() const {
-    return status;
-}
-
-const llama_ubatch & llama_kv_cache_msa_context::get_ubatch() const {
-    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);
-
-    return ubatches[i_next];
-}
-
-const llama_kv_cache_context * llama_kv_cache_msa_context::get_base() const {
-    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);
-
-    return static_cast<const llama_kv_cache_context *>(ctx_base.get());
-}
-
-const llama_kv_cache_context * llama_kv_cache_msa_context::get_idx() const {
-    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);
-
-    return static_cast<const llama_kv_cache_context *>(ctx_idx.get());
-}
-
-uint32_t llama_kv_cache_msa_context::get_n_pos() const {
-    // pad the value so that the graph remains constant across batches and can be reused
-    const uint32_t n_pad_cur = std::max(kv->get_n_pad(), 256u);
-
-    llama_pos pos_max = -1;
-
-    for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) kv->get_n_seq_max(); ++seq_id) {
-        pos_max = std::max(pos_max, kv->seq_pos_max(seq_id));
-    }
-
-    return std::max(n_pad_cur, GGML_PAD((uint32_t) (pos_max + 1), n_pad_cur));
-}
-
-void llama_kv_cache_msa_context::set_input_cell_pos(ggml_tensor * dst, const llama_ubatch * ubatch, int32_t div) const {
-    GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer));
-    GGML_ASSERT(dst->type == GGML_TYPE_I32);
-    GGML_ASSERT(div > 0);
-
-    const int64_t n_tokens    = ubatch->n_tokens;
-    const int64_t n_kv        = dst->ne[0];
-    const int64_t n_stream_ub = dst->ne[1];
-
-    GGML_ASSERT(n_tokens % n_stream_ub == 0);
-    const int64_t n_tps = n_tokens/n_stream_ub;
-
-    int32_t * data = (int32_t *) dst->data;
-
-    for (int64_t s = 0; s < n_stream_ub; ++s) {
-        const llama_seq_id seq_id = ubatch->seq_id[s*n_tps][0];
-
-        const auto & cells = kv->get_base()->get_cells(seq_id);
-
-        for (int64_t j = 0; j < n_kv; ++j) {
-            // the value for empty or other-sequence cells is irrelevant as consumers mask them
-            data[s*n_kv + j] =
-                cells.is_empty(j) || !cells.seq_has(j, seq_id)
-                    ? 0
-                    : (int32_t) (cells.pos_get(j)/div);
-        }
-    }
-}
-
-void llama_kv_cache_msa_context::set_input_pos_slot(ggml_tensor * dst, const llama_ubatch * ubatch) const {
-    GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer));
-    GGML_ASSERT(dst->type == GGML_TYPE_I32 || dst->type == GGML_TYPE_F32);
-
-    const int64_t n_tokens    = ubatch->n_tokens;
-    const int64_t n_pos       = dst->ne[0];
-    const int64_t n_stream_ub = dst->ne[1];
-
-    GGML_ASSERT(n_tokens % n_stream_ub == 0);
-    const int64_t n_tps = n_tokens/n_stream_ub;
-
-    for (int64_t s = 0; s < n_stream_ub; ++s) {
-        const llama_seq_id seq_id = ubatch->seq_id[s*n_tps][0];
-
-        const auto & cells = kv->get_base()->get_cells(seq_id);
-
-        std::vector<int32_t> map(n_pos, 0);
-
-        for (uint32_t j = 0; j < cells.size(); ++j) {
-            if (cells.is_empty(j) || !cells.seq_has(j, seq_id)) {
-                continue;
-            }
-
-            const llama_pos p0 = cells.pos_get(j);
-
-            if (p0 < 0 || p0 >= n_pos) {
-                continue;
-            }
-
-            map[p0] = (int32_t) j;
-        }
-
-        if (dst->type == GGML_TYPE_I32) {
-            int32_t * data = (int32_t *) dst->data + s*n_pos;
-            std::copy(map.begin(), map.end(), data);
-        } else {
-            float * data = (float *) dst->data + s*n_pos;
-            for (int64_t p = 0; p < n_pos; ++p) {
-                data[p] = (float) map[p];
-            }
-        }
-    }
-}
-
-void llama_kv_cache_msa_context::set_input_pos_mask(ggml_tensor * dst, const llama_ubatch * ubatch) const {
-    GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer));
-    GGML_ASSERT(dst->type == GGML_TYPE_F32);
-
-    const int64_t n_tokens = ubatch->n_tokens;
-    const int64_t n_pos    = dst->ne[0];
-
-    GGML_ASSERT(dst->ne[1] == n_tokens);
-
-    const uint32_t       n_swa    = kv->get_n_swa();
-    const llama_swa_type swa_type = kv->get_swa_type();
-
-    float * data = (float *) dst->data;
-
-    std::fill(data, data + n_pos*n_tokens, -INFINITY);
-
-    for (int64_t i = 0; i < n_tokens; ++i) {
-        const llama_seq_id seq_id = ubatch->seq_id[i][0];
-
-        const auto & cells = kv->get_base()->get_cells(seq_id);
-
-        const llama_pos p1 = ubatch->pos[i];
-
-        for (uint32_t j = 0; j < cells.size(); ++j) {
-            if (cells.is_empty(j) || !cells.seq_has(j, seq_id)) {
-                continue;
-            }
-
-            const llama_pos p0 = cells.pos_get(j);
-
-            if (p0 < 0 || p0 >= n_pos) {
-                continue;
-            }
-
-            // causal mask
-            if (p0 > p1) {
-                continue;
-            }
-
-            // apply SWA if any
-            if (llama_hparams::is_masked_swa(n_swa, swa_type, p0, p1)) {
-                continue;
-            }
-
-            data[i*n_pos + p0] = 0.0f;
-        }
-    }
-}

src/llama-kv-cache-msa.h

@@ -1,153 +0,0 @@
-#pragma once
-
-#include "llama-kv-cache.h"
-
-#include <vector>
-
-// llama_kv_cache_msa
-
-// uses two instances of llama_kv_cache, one for K/V tensors, and one for the MSA indexer tensors
-// both receive identical sequence operations and identical ubatches, so their cell layouts stay in synced.
-// the context also exposes per-ubatch pos - cell translation maps populated from llama_kv_cells via
-// llama_kv_cache::get_cells(), which the model graph uses to run MSA block selection in position space
-
-class llama_kv_cache_msa : public llama_memory_i {
-public:
-    llama_kv_cache_msa(
-            const llama_model & model,
-                    ggml_type   type_k,
-                    ggml_type   type_v,
-                         bool   v_trans,
-                         bool   offload,
-                         bool   unified,
-                     uint32_t   kv_size,
-                     uint32_t   n_seq_max,
-                     uint32_t   n_pad,
-                     uint32_t   n_swa,
-               llama_swa_type   swa_type,
-        const layer_filter_cb & filter,
-        const layer_filter_cb & filter_idx,
-        const  layer_reuse_cb & reuse);
-
-    ~llama_kv_cache_msa() = default;
-
-    // llama_memory_i
-
-    llama_memory_context_ptr init_batch(
-            llama_batch_allocr & balloc,
-            uint32_t n_ubatch,
-            bool embd_all) override;
-
-    llama_memory_context_ptr init_full() override;
-
-    llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) override;
-
-    bool get_can_shift() const override;
-
-    void clear(bool data) override;
-
-    bool seq_rm  (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1) override;
-    void seq_cp  (llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) override;
-    void seq_keep(llama_seq_id seq_id)                                                          override;
-    void seq_add (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1, llama_pos shift) override;
-    void seq_div (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1, int d) override;
-
-    llama_pos seq_pos_min(llama_seq_id seq_id) const override;
-    llama_pos seq_pos_max(llama_seq_id seq_id) const override;
-
-    std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const override;
-
-    // state write/load
-
-    void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;
-    void state_read (llama_io_read_i  & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
-
-    // llama_kv_cache_msa specific API
-
-    llama_kv_cache * get_base() const;
-    llama_kv_cache * get_idx () const;
-
-    uint32_t       get_n_pad()    const { return n_pad; }
-    uint32_t       get_n_seq_max() const { return n_seq_max; }
-    uint32_t       get_n_swa()    const { return n_swa; }
-    llama_swa_type get_swa_type() const { return swa_type; }
-
-private:
-    // keep the indexer KV cache hparams instance here as llama_kv_cache stores only a reference
-    llama_hparams hparams_idx;
-
-    const uint32_t n_stream  = 1;
-    const uint32_t n_seq_max = 1;
-    const uint32_t n_pad     = 1;
-
-    const uint32_t       n_swa    = 0;
-    const llama_swa_type swa_type = LLAMA_SWA_TYPE_NONE;
-
-    std::unique_ptr<llama_kv_cache> kv_base;
-    std::unique_ptr<llama_kv_cache> kv_idx;
-};
-
-class llama_kv_cache_msa_context : public llama_memory_context_i {
-public:
-    using slot_info_vec_t = llama_kv_cache::slot_info_vec_t;
-
-    // used for errors
-    llama_kv_cache_msa_context(llama_memory_status status);
-
-    // used to create a full-cache context
-    llama_kv_cache_msa_context(
-            llama_kv_cache_msa * kv);
-
-    // used to create an update context
-    llama_kv_cache_msa_context(
-            llama_kv_cache_msa * kv,
-            llama_context * lctx,
-            bool optimize);
-
-    // used to create a batch processing context from a batch
-    llama_kv_cache_msa_context(
-            llama_kv_cache_msa * kv,
-            slot_info_vec_t sinfos_base,
-            slot_info_vec_t sinfos_idx,
-            std::vector<llama_ubatch> ubatches);
-
-    virtual ~llama_kv_cache_msa_context();
-
-    // llama_memory_context_i
-
-    bool next()  override;
-    bool apply() override;
-
-    llama_memory_status  get_status() const override;
-    const llama_ubatch & get_ubatch() const override;
-
-    // llama_kv_cache_msa_context specific API
-
-    const llama_kv_cache_context * get_base() const;
-    const llama_kv_cache_context * get_idx () const;
-
-    // max position currently present in the cache plus one, padded MSA blocks are defined over token positions
-    // so the block-selection tensors are sized by this value rather than by the number of cells
-    uint32_t get_n_pos() const;
-
-    // position <-> cell translation maps, populated from the base cache cells
-    // the model graph relates cache contents to token positions only through these per ubatch inputs
-    // value for empty or other-sequence cells is 0 so consumers must mask them
-    void set_input_cell_pos(ggml_tensor * dst, const llama_ubatch * ubatch, int32_t div) const;
-    // positions without a cell map to cell 0, consumers must mask them assumes one sequence per stream
-    void set_input_pos_slot(ggml_tensor * dst, const llama_ubatch * ubatch) const;
-    void set_input_pos_mask(ggml_tensor * dst, const llama_ubatch * ubatch) const;
-
-private:
-    llama_kv_cache_msa * kv;
-
-    // the index of the next ubatch to process
-    size_t i_next = 0;
-
-    std::vector<llama_ubatch> ubatches;
-
-    const llama_memory_context_ptr ctx_base;
-    const llama_memory_context_ptr ctx_idx;
-
-    const llama_memory_status status;
-};

src/llama-kv-cache.cpp

@@ -112,7 +112,7 @@ llama_kv_cache::llama_kv_cache(
         auto it = ctx_map.find(buft);
         if (it == ctx_map.end()) {
             ggml_init_params params = {
-                /*.mem_size   =*/ size_t(2u*(1 + n_stream)*n_layer*ggml_tensor_overhead()),
+                /*.mem_size   =*/ size_t(3u*(1 + n_stream)*n_layer*ggml_tensor_overhead()), //Reserve tensor metadata for up to 3 tensors per layer (K, V, and optional K_idx), plus one view per tensor per stream.
                 /*.mem_buffer =*/ NULL,
                 /*.no_alloc   =*/ true,
             };
@@ -242,9 +242,25 @@ llama_kv_cache::llama_kv_cache(
             v_stream.push_back(has_v ? ggml_view_2d(ctx, v, n_embd_v_gqa, kv_size, v->nb[1], s*v->nb[2]) : nullptr);
         }
 
+        const uint32_t n_embd_k_idx = hparams.n_embd_k_idx(il);
+        ggml_tensor * k_idx = n_embd_k_idx > 0
+            ? ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd_k_idx, kv_size, n_stream)
+            : nullptr;
+        if (k_idx) {
+            ggml_format_name(k_idx, "cache_k_idx_l%d", il);
+            msa_strict_slots = (n_stream == n_seq_max);
+        }
+
+        std::vector<ggml_tensor *> k_idx_stream;
+        for (uint32_t s = 0; s < n_stream; ++s) {
+            k_idx_stream.push_back(k_idx
+                ? ggml_view_2d(ctx, k_idx, n_embd_k_idx, kv_size, k_idx->nb[1], s*k_idx->nb[2])
+                : nullptr);
+        }
+
         map_layer_ids[il] = layers.size();
 
-        layers.push_back({ il, k, v, k_stream, v_stream, });
+        layers.push_back({ il, k, v, k_idx, k_stream, v_stream, k_idx_stream });
     }
 
     if (reuse) {
@@ -293,13 +309,24 @@ llama_kv_cache::llama_kv_cache(
     }
 
     {
-        const size_t memory_size_k = size_k_bytes();
-        const size_t memory_size_v = size_v_bytes();
+        const size_t memory_size_k     = size_k_bytes();
+        const size_t memory_size_v     = size_v_bytes();
+        const size_t memory_size_k_idx = size_k_idx_bytes();
+        const size_t memory_size_total = memory_size_k + memory_size_v + memory_size_k_idx;
 
-        LLAMA_LOG_INFO("%s: size = %7.2f MiB (%6u cells, %3d layers, %2u/%u seqs), K (%s): %7.2f MiB, V (%s): %7.2f MiB\n", __func__,
-                (float)(memory_size_k + memory_size_v) / (1024.0f * 1024.0f), kv_size, (int) layers.size(), n_seq_max, n_stream,
-                ggml_type_name(type_k), (float)memory_size_k / (1024.0f * 1024.0f),
-                ggml_type_name(type_v), (float)memory_size_v / (1024.0f * 1024.0f));
+        constexpr float mib = 1024.0f * 1024.0f;
+
+        const std::string k_log = format(", K (%s): %7.2f MiB", ggml_type_name(type_k), (float) memory_size_k / mib);
+        const std::string v_log = format(", V (%s): %7.2f MiB", ggml_type_name(type_v), (float) memory_size_v / mib);
+
+        std::string k_idx_log;
+        if (memory_size_k_idx > 0) {
+            k_idx_log = format(", K_idx (%s): %7.2f MiB", ggml_type_name(GGML_TYPE_F32), (float) memory_size_k_idx / mib);
+        }
+
+        LLAMA_LOG_INFO("%s: size = %7.2f MiB (%6u cells, %3d layers, %2u/%u seqs)%s%s%s\n", __func__,
+                (float) memory_size_total / mib, kv_size, (int) layers.size(), n_seq_max, n_stream,
+                k_log.c_str(), v_log.c_str(), k_idx_log.c_str());
     }
 
     // TODO: refactor [TAG_KV_CACHE_SHARE_CELLS]
@@ -392,6 +419,39 @@ bool llama_kv_cache::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) {
         p1 = std::numeric_limits<llama_pos>::max();
     }
 
+    // empty range - nothing to remove
+    if (p0 >= p1) {
+        return true;
+    }
+
+    // MSA anchors block selection to absolute cache slots (slot == position). Tail trim and full removal preserve this invariant, but removing a prefix
+    // or middle range would free slots while later cells survive, desynchronizing the indexer cache. Reject such removals before modifying the cache.
+    if (msa_strict_slots) {
+        for (llama_seq_id sid = 0; sid < (llama_seq_id) seq_to_stream.size(); ++sid) {
+            if (seq_id >= 0 && sid != seq_id) {
+                continue;
+            }
+
+            const auto & cells = v_cells[seq_to_stream[sid]];
+
+            const llama_pos pmin = cells.seq_pos_min(sid);
+            const llama_pos pmax = cells.seq_pos_max(sid);
+
+            if (pmin < 0) {
+                continue;   // empty sequence
+            }
+
+            const bool overlaps    = p0 <= pmax && p1 > pmin;   // the range removes something
+            const bool leaves_tail = p1 <= pmax;                // cells beyond the range survive
+
+            if (overlaps && leaves_tail) {
+                LLAMA_LOG_WARN("%s: MSA: partial (non-suffix) removal [%d, %d) for seq %d is not supported "
+                        "(block selection is anchored to cache slots) - rejected\n", __func__, p0, p1, sid);
+                return false;
+            }
+        }
+    }
+
     if (seq_id >= 0) {
         auto & cells = v_cells[seq_to_stream[seq_id]];
         auto & head  = v_heads[seq_to_stream[seq_id]];
@@ -846,6 +906,10 @@ bool llama_kv_cache::update(llama_context * lctx, bool do_shift, const stream_co
                 if (layer.v_stream[ssrc]) {
                     ggml_backend_tensor_copy(layer.v_stream[ssrc], layer.v_stream[sdst]);
                 }
+                if (layer.k_idx_stream[ssrc]) {
+                    GGML_ASSERT(layer.k_idx_stream[sdst]);
+                    ggml_backend_tensor_copy(layer.k_idx_stream[ssrc], layer.k_idx_stream[sdst]);
+                }
             }
         }
     }
@@ -994,6 +1058,44 @@ llama_kv_cache::slot_info llama_kv_cache::find_slot(const llama_ubatch & ubatch,
 
         const auto & cells = v_cells[seq_to_stream[seq_id]];
 
+        if (n_tokens > cells.size()) {
+            LLAMA_LOG_ERROR("%s: n_tokens = %d > size = %u\n", __func__, n_tokens, cells.size());
+            return { };
+        }
+
+        // MSA block selection assumes slot == logical position (append-only streams).
+        if (msa_strict_slots) {
+            for (uint32_t ii = 0; ii < n_tokens; ++ii) {
+                const llama_pos pos = ubatch.pos[s*n_tokens + ii];
+
+                if (pos < 0 || (uint64_t) pos >= cells.size()) {
+                    LLAMA_LOG_WARN("%s: MSA: position %d is outside the cache range [0, %u)\n",
+                            __func__, pos, cells.size());
+                    return { };
+                }
+
+                const uint32_t idx = (uint32_t) pos;
+
+                if (!cells.is_empty(idx)) {
+                    LLAMA_LOG_WARN("%s: MSA: required slot %u is already occupied (stream %u)\n",
+                            __func__, idx, seq_to_stream[seq_id]);
+                    return { };
+                }
+
+                // strictly increasing positions, rules out duplicates and, for contiguous requests, is tightened to exact adjacency
+                if (!res.idxs[s].empty() && (cont ? idx != res.idxs[s].back() + 1
+                                                  : idx <= res.idxs[s].back())) {
+                    LLAMA_LOG_WARN("%s: MSA: token positions are not %s within the ubatch\n",
+                            __func__, cont ? "contiguous" : "strictly increasing");
+                    return { };
+                }
+
+                res.idxs[s].push_back(idx);
+            }
+
+            continue;
+        }
+
         uint32_t head_cur = v_heads[seq_to_stream[seq_id]];
 
         // if we have enough unused cells before the current head ->
@@ -1002,11 +1104,6 @@ llama_kv_cache::slot_info llama_kv_cache::find_slot(const llama_ubatch & ubatch,
             head_cur = 0;
         }
 
-        if (n_tokens > cells.size()) {
-            LLAMA_LOG_ERROR("%s: n_tokens = %d > size = %u\n", __func__, n_tokens, cells.size());
-            return { };
-        }
-
         uint32_t n_tested = 0;
 
         // for continuous slots, we test that all tokens in the ubatch fit, starting from the current head
@@ -1113,6 +1210,15 @@ void llama_kv_cache::apply_ubatch(const slot_info & sinfo, const llama_ubatch &
 
             const auto idx = sinfo.idxs[s][ii];
 
+            if (msa_strict_slots && (llama_pos) idx != ubatch.pos[i]) {
+                LLAMA_LOG_ERROR("%s: MSA slot/position invariant violated: "
+                        "writing pos %d into cell %u (stream %u). The indexer cache "
+                        "would desync and block selection would silently corrupt. "
+                        "This is a bug, please report it with reproduction steps.\n",
+                        __func__, ubatch.pos[i], idx, sinfo.strm[s]);
+                GGML_ABORT("MSA: slot != pos");
+            }
+
             if (!cells.is_empty(idx)) {
                 assert(cells.seq_count(idx) == 1);
 
@@ -1156,7 +1262,8 @@ void llama_kv_cache::apply_ubatch(const slot_info & sinfo, const llama_ubatch &
             LLAMA_LOG_DEBUG("%s: purging positions [%d, %d] of sequence %d from KV cache\n",
                     __func__, cells.seq_pos_min(s), seq_pos_max_rm[s], s);
 
-            seq_rm(s, cells.seq_pos_min(s), seq_pos_max_rm[s] + 1);
+            // under MSA strict slots this path should be unreachable, since strict MSA placement never selects occupied cells
+            GGML_ASSERT(seq_rm(s, cells.seq_pos_min(s), seq_pos_max_rm[s] + 1));
         }
     }
 
@@ -1176,6 +1283,12 @@ bool llama_kv_cache::get_can_shift() const {
     if (hparams.n_pos_per_embd() > 1) {
         return false;
     }
+    // shifting would leave k_idx stale
+    for (const auto & layer : layers) {
+        if (layer.k_idx) {
+            return false;
+        }
+    }
     return true;
 }
 
@@ -1224,12 +1337,6 @@ ggml_tensor * llama_kv_cache::get_k_storage(int32_t il) const {
     return layers[ikv].k;
 }
 
-const llama_kv_cells & llama_kv_cache::get_cells(llama_seq_id seq_id) const {
-    GGML_ASSERT(seq_id >= 0 && (size_t) seq_id < seq_to_stream.size());
-
-    return v_cells[seq_to_stream[seq_id]];
-}
-
 uint32_t llama_kv_cache::get_n_kv(const slot_info & sinfo) const {
     uint32_t result = 0;
 
@@ -1298,6 +1405,23 @@ ggml_tensor * llama_kv_cache::get_v(ggml_context * ctx, int32_t il, uint32_t n_k
             ggml_row_size(v->type, kv_size*n_embd_v_gqa)*sinfo.s0);
 }
 
+ggml_tensor * llama_kv_cache::get_k_idx(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const {
+    const int32_t ikv = map_layer_ids.at(il);
+    auto * k_idx = layers[ikv].k_idx;
+    GGML_ASSERT(k_idx);
+
+    const uint64_t kv_size = get_size();
+    const int64_t  n_idx   = k_idx->ne[0];                 // 128
+    const uint32_t ns      = sinfo.s1 - sinfo.s0 + 1;
+
+    return ggml_view_4d(ctx, k_idx,
+            n_idx, 1, n_kv, ns,
+            ggml_row_size(k_idx->type, n_idx),             // nb1 (single head)
+            ggml_row_size(k_idx->type, n_idx),             // nb2 (per cell)
+            ggml_row_size(k_idx->type, n_idx*kv_size),     // nb3 (per stream)
+            ggml_row_size(k_idx->type, n_idx*kv_size)*sinfo.s0);
+}
+
 ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const {
     GGML_UNUSED(sinfo);
 
@@ -1399,6 +1523,28 @@ ggml_tensor * llama_kv_cache::build_input_k_idxs(ggml_context * ctx, const llama
     return k_idxs;
 }
 
+ggml_tensor * llama_kv_cache::cpy_k_idx(ggml_context * ctx, ggml_tensor * k_idx_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const {
+    GGML_UNUSED(sinfo);
+    const int32_t ikv = map_layer_ids.at(il);
+    ggml_tensor * k_idx = layers[ikv].k_idx;
+    GGML_ASSERT(k_idx && "cpy_k_idx on a layer with no indexer cache");
+
+    const int64_t n_embd_head = k_idx_cur->ne[0];          // 128
+    const int64_t n_head      = k_idx_cur->ne[1];          // 1
+    const int64_t n_tokens    = k_idx_cur->ne[2];
+    const int64_t n_embd_gqa  = n_embd_head*n_head;        // 128
+
+    GGML_ASSERT(ggml_row_size(k_idx_cur->type, n_embd_head) == k_idx_cur->nb[1]);
+    k_idx_cur = ggml_view_2d(ctx, k_idx_cur, n_embd_gqa, n_tokens, k_idx_cur->nb[2], 0);
+
+    const int64_t n_stream = k_idx->ne[2];
+    if (n_stream > 1) {
+        const int64_t kv_size = get_size();
+        k_idx = ggml_reshape_2d(ctx, k_idx, n_embd_gqa, kv_size*n_stream);
+    }
+    return ggml_set_rows(ctx, k_idx, k_idx_cur, k_idxs);   // same k_idxs as the K store
+}
+
 ggml_tensor * llama_kv_cache::build_input_v_idxs(ggml_context * ctx, const llama_ubatch & ubatch) const {
     const uint32_t n_tokens = ubatch.n_tokens;
 
@@ -1833,6 +1979,18 @@ size_t llama_kv_cache::size_v_bytes() const {
     return size_v_bytes;
 }
 
+size_t llama_kv_cache::size_k_idx_bytes() const {
+    size_t size_k_idx_bytes = 0;
+
+    for (const auto & layer : layers) {
+        if (layer.k_idx) {
+            size_k_idx_bytes += ggml_nbytes(layer.k_idx);
+        }
+    }
+
+    return size_k_idx_bytes;
+}
+
 ggml_tensor * llama_kv_cache::build_rope_shift(
         const llama_cparams & cparams,
                ggml_context * ctx,
@@ -2145,6 +2303,36 @@ void llama_kv_cache::state_write_data(llama_io_write_i & io, const cell_ranges_t
         }
     }
 
+    if (size_k_idx_bytes() > 0) {
+        const uint32_t has_k_idx_u32 = 1;
+        io.write(&has_k_idx_u32, sizeof(has_k_idx_u32));
+
+        for (const auto & layer : layers) {
+            const uint32_t layer_has_k_idx = layer.k_idx ? 1 : 0;
+            io.write(&layer_has_k_idx, sizeof(layer_has_k_idx));
+
+            if (!layer_has_k_idx) {
+                continue;
+            }
+
+            GGML_ASSERT(layer.k_idx_stream[cr.strm]);
+
+            const int32_t k_idx_type_i = (int32_t) layer.k_idx->type;
+            io.write(&k_idx_type_i, sizeof(k_idx_type_i));
+
+            const uint64_t k_idx_size_row = ggml_row_size(layer.k_idx->type, layer.k_idx->ne[0]);
+            io.write(&k_idx_size_row, sizeof(k_idx_size_row));
+
+            for (const auto & range : cr.data) {
+                const size_t range_size = range.second - range.first;
+                const size_t buf_size   = range_size * k_idx_size_row;
+                const size_t offset     = range.first * k_idx_size_row;
+
+                io.write_tensor(layer.k_idx_stream[cr.strm], offset, buf_size);
+            }
+        }
+    }
+
     if (!v_trans) {
         for (const auto & layer : layers) {
             const uint32_t il = layer.il;
@@ -2393,6 +2581,68 @@ bool llama_kv_cache::state_read_data(llama_io_read_i & io, uint32_t strm, uint32
         }
     }
 
+    if (size_k_idx_bytes() > 0) {
+        uint32_t has_k_idx_u32 = 0;
+        io.read(&has_k_idx_u32, sizeof(has_k_idx_u32));
+
+        if (has_k_idx_u32 != 1) {
+            LLAMA_LOG_ERROR("%s: missing k_idx data in KV cache state\n", __func__);
+            return false;
+        }
+
+        for (const auto & layer : layers) {
+            uint32_t layer_has_k_idx = 0;
+            io.read(&layer_has_k_idx, sizeof(layer_has_k_idx));
+
+            const uint32_t expected_layer_has_k_idx = layer.k_idx ? 1 : 0;
+
+            if (layer_has_k_idx != expected_layer_has_k_idx) {
+                LLAMA_LOG_ERROR(
+                    "%s: mismatched k_idx state for layer: got %u, expected %u\n",
+                    __func__, layer_has_k_idx, expected_layer_has_k_idx);
+                return false;
+            }
+
+            if (!layer_has_k_idx) {
+                continue;
+            }
+
+            GGML_ASSERT(layer.k_idx_stream[strm]);
+
+            int32_t k_idx_type_i = -1;
+            io.read(&k_idx_type_i, sizeof(k_idx_type_i));
+
+            if (k_idx_type_i != (int32_t) layer.k_idx->type) {
+                LLAMA_LOG_ERROR(
+                    "%s: mismatched k_idx type: got %d, expected %d\n",
+                    __func__, k_idx_type_i, (int32_t) layer.k_idx->type);
+                return false;
+            }
+
+            uint64_t k_idx_size_row = 0;
+            io.read(&k_idx_size_row, sizeof(k_idx_size_row));
+
+            const uint64_t expected_k_idx_size_row = ggml_row_size(layer.k_idx->type, layer.k_idx->ne[0]);
+
+            if (k_idx_size_row != expected_k_idx_size_row) {
+                LLAMA_LOG_ERROR(
+                    "%s: mismatched k_idx row size: got %zu, expected %zu\n",
+                    __func__, (size_t) k_idx_size_row, (size_t) expected_k_idx_size_row);
+                return false;
+            }
+
+            if (cell_count) {
+                if (sinfo.is_contiguous()) {
+                    io.read_tensor(layer.k_idx_stream[strm], sinfo.head() * k_idx_size_row, cell_count * k_idx_size_row);
+                } else {
+                    for (uint32_t i = 0; i < cell_count; ++i) {
+                        io.read_tensor(layer.k_idx_stream[strm], sinfo.idxs[0][i] * k_idx_size_row, k_idx_size_row);
+                    }
+                }
+            }
+        }
+    }
+
     if (!this->v_trans) {
         for (const auto & layer : layers) {
             const uint32_t il = layer.il;
@@ -2594,6 +2844,10 @@ ggml_tensor * llama_kv_cache_context::get_v(ggml_context * ctx, int32_t il) cons
     return kv->get_v(ctx, il, n_kv, sinfos[i_cur]);
 }
 
+ggml_tensor * llama_kv_cache_context::get_k_idx(ggml_context * ctx, int32_t il) const {
+    return kv->get_k_idx(ctx, il, n_kv, sinfos[i_cur]);
+}
+
 ggml_tensor * llama_kv_cache_context::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il) const {
     return kv->cpy_k(ctx, k_cur, k_idxs, il, sinfos[i_cur]);
 }
@@ -2602,6 +2856,10 @@ ggml_tensor * llama_kv_cache_context::cpy_v(ggml_context * ctx, ggml_tensor * v_
     return kv->cpy_v(ctx, v_cur, v_idxs, il, sinfos[i_cur]);
 }
 
+ggml_tensor * llama_kv_cache_context::cpy_k_idx(ggml_context * ctx, ggml_tensor * k_idx_cur, ggml_tensor * k_idxs, int32_t il) const {
+    return kv->cpy_k_idx(ctx, k_idx_cur, k_idxs, il, sinfos[i_cur]);
+}
+
 ggml_tensor * llama_kv_cache_context::build_input_k_idxs(ggml_context * ctx, const llama_ubatch & ubatch) const {
     return kv->build_input_k_idxs(ctx, ubatch);
 }

src/llama-kv-cache.h

@@ -164,8 +164,6 @@ public:
     std::vector<uint32_t> get_layer_ids() const;
     ggml_tensor * get_k_storage(int32_t il) const;
 
-    const llama_kv_cells & get_cells(llama_seq_id seq_id) const;
-
     //
     // graph_build API
     //
@@ -175,10 +173,12 @@ public:
     // get views of the current state of the cache
     ggml_tensor * get_k(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const;
     ggml_tensor * get_v(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const;
+    ggml_tensor * get_k_idx(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const;
 
     // store k_cur and v_cur in the cache based on the provided head location
     ggml_tensor * cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const;
     ggml_tensor * cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggml_tensor * v_idxs, int32_t il, const slot_info & sinfo) const;
+    ggml_tensor * cpy_k_idx(ggml_context * ctx, ggml_tensor * k_idx_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const;
 
     //
     // preparation API
@@ -230,9 +230,11 @@ private:
 
         ggml_tensor * k;
         ggml_tensor * v;
+        ggml_tensor * k_idx;   // MSA single-head indexer keys, F32
 
         std::vector<ggml_tensor *> k_stream;
         std::vector<ggml_tensor *> v_stream;
+        std::vector<ggml_tensor *> k_idx_stream;
     };
 
     bool v_trans = true;  // the value tensor is transposed
@@ -261,6 +263,9 @@ private:
     // env: LLAMA_KV_CACHE_DEBUG
     int debug = 0;
 
+    // set when a k_idx (indexer) cache exists and the stream layout supports MSA (single seq, or one stream per seq)
+    bool msa_strict_slots = false;
+
     // this is the SWA type of the cache - not to be confused with the model SWA type
     const llama_swa_type swa_type = LLAMA_SWA_TYPE_NONE;
 
@@ -293,6 +298,7 @@ private:
 
     size_t size_k_bytes() const;
     size_t size_v_bytes() const;
+    size_t size_k_idx_bytes() const;
 
     ggml_tensor * build_rope_shift(
             const llama_cparams & cparams,
@@ -372,6 +378,7 @@ public:
     // get views of the current state of the cache
     ggml_tensor * get_k(ggml_context * ctx, int32_t il) const;
     ggml_tensor * get_v(ggml_context * ctx, int32_t il) const;
+    ggml_tensor * get_k_idx(ggml_context * ctx, int32_t il) const;
 
     // store k_cur and v_cur in the cache based on the provided head location
     // note: the heads in k_cur and v_cur should be laid out contiguously in memory
@@ -381,6 +388,7 @@ public:
     //   - v_idxs [n_tokens] or [n_tokens*n_embd_v_gqa] depending if V cache is transposed
     ggml_tensor * cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il) const;
     ggml_tensor * cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggml_tensor * v_idxs, int32_t il) const;
+    ggml_tensor * cpy_k_idx(ggml_context * ctx, ggml_tensor * k_idx_cur, ggml_tensor * k_idxs, int32_t il) const;
 
     // create destination indices for each head of the current batch for where it would be written in the KV cache
     // the indices address the global KV cache (not per stream) - this is not relevant for the user of this API, but

src/llama-model-loader.cpp

@@ -857,11 +857,7 @@ struct ggml_tensor * llama_model_loader::require_tensor_meta(const std::string &
     return tensor;
 }
 
-const struct ggml_tensor * llama_model_loader::check_tensor_dims(
-        const std::string & name,
-        const std::vector<int64_t> & ne,
-        bool required,
-        bool allow_reshape) const {
+const struct ggml_tensor * llama_model_loader::check_tensor_dims(const std::string & name, const std::vector<int64_t> & ne, bool required) const {
     const struct ggml_tensor * cur = get_tensor_meta(name.c_str());
 
     if (cur == NULL) {
@@ -871,33 +867,21 @@ const struct ggml_tensor * llama_model_loader::check_tensor_dims(
         throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name.c_str()));
     }
 
-    bool is_ok = true;
-
-    if (allow_reshape) {
-        // check total number of elements only
-        const int64_t ncur = ggml_nelements(cur);
-        int64_t nexp = 1;
-        for (size_t i = 0; i < ne.size(); ++i) {
-            nexp *= ne[i];
-        }
-        if (ncur != nexp) {
-            is_ok = false;
-        }
-    } else {
+    {
+        bool is_ok = true;
         for (size_t i = 0; i < GGML_MAX_DIMS; ++i) {
             if ((i < ne.size() && ne[i] != cur->ne[i]) || (i >= ne.size() && cur->ne[i] != 1)) {
                 is_ok = false;
                 break;
             }
         }
-    }
-
-    if (!is_ok) {
-        throw std::runtime_error(
-                format("%s: tensor '%s' has wrong shape; expected %s, got %s",
-                    __func__, name.c_str(),
-                    llama_format_tensor_shape(ne).c_str(),
-                    llama_format_tensor_shape(cur).c_str()));
+        if (!is_ok) {
+            throw std::runtime_error(
+                    format("%s: tensor '%s' has wrong shape; expected %s, got %s",
+                        __func__, name.c_str(),
+                        llama_format_tensor_shape(ne).c_str(),
+                        llama_format_tensor_shape(cur).c_str()));
+        }
     }
 
     return cur;
@@ -1262,25 +1246,11 @@ struct ggml_tensor * llama_model_loader::create_tensor(
         return ret;
     }
 
-    LLAMA_LOG_DEBUG("%s: loading tensor %s\n", __func__, tn.str().c_str());
-    const struct ggml_tensor * cur = check_tensor_dims(tn.str(), ne, !(flags & TENSOR_NOT_REQUIRED), flags & TENSOR_ALLOW_RESHAPE);
-    if (cur == NULL) {
-        return NULL;
-    }
-
-    ggml_tensor t_meta = *cur;
-    if (flags & TENSOR_ALLOW_RESHAPE) {
-        for (size_t dim = 0; dim < GGML_MAX_DIMS; dim++) {
-            t_meta.ne[dim] = dim < ne.size() ? ne.begin()[dim] : 1;
-            t_meta.nb[dim] = dim == 0 ? ggml_type_size(t_meta.type) : t_meta.ne[dim-1]*t_meta.nb[dim-1];
-        }
-    }
-
-    ggml_backend_buffer_type_t buft = buft_for_tensor(&t_meta);
+    ggml_tensor * t_meta = get_tensor_meta(tn.str().c_str());
+    ggml_backend_buffer_type_t buft = buft_for_tensor(t_meta);
     if (buft == nullptr) {
-        return nullptr;
+        return nullptr; // return type is ggml_tensor *
     }
-
     ggml_context * ctx = ctx_for_buft(buft);
 
     // if duplicated, check if the original tensor was allocated in the same buffer type context and avoid creating a new one
@@ -1291,13 +1261,20 @@ struct ggml_tensor * llama_model_loader::create_tensor(
         }
     }
 
+    LLAMA_LOG_DEBUG("%s: loading tensor %s\n", __func__, tn.str().c_str());
+    const struct ggml_tensor * cur = check_tensor_dims(tn.str(), ne, !(flags & TENSOR_NOT_REQUIRED));
+
+    if (cur == NULL) {
+        return NULL;
+    }
+
     const bool duplicated = flags & TENSOR_DUPLICATED;
 
-    struct ggml_tensor * tensor = ggml_dup_tensor(ctx, &t_meta);
-    ggml_set_name(tensor, ggml_get_name(&t_meta));
+    struct ggml_tensor * tensor = ggml_dup_tensor(ctx, cur);
+    ggml_set_name(tensor, ggml_get_name(cur));
 
     if (duplicated) {
-        size_data += ggml_nbytes(&t_meta);
+        size_data += ggml_nbytes(cur);
     } else {
         n_created++;
     }
@@ -1305,6 +1282,34 @@ struct ggml_tensor * llama_model_loader::create_tensor(
     return tensor;
 }
 
+struct ggml_tensor * llama_model_loader::create_tensor_as_view(struct ggml_context * ctx, struct ggml_tensor * base, const std::string & name, const std::initializer_list<int64_t> & ne, size_t offset, bool required) {
+    const struct ggml_tensor * cur = check_tensor_dims(name, ne, required);
+
+    if (cur == NULL) {
+        return NULL;
+    }
+
+    if (cur->type != base->type) {
+        throw std::runtime_error(format("%s: tensor '%s' has wrong type; expected %s, got %s", __func__, name.c_str(), ggml_type_name(base->type), ggml_type_name(cur->type)));
+    }
+
+    std::array<int64_t, GGML_MAX_DIMS> dims;
+    for (size_t i = 0; i < GGML_MAX_DIMS; ++i) {
+        dims[i] = i < ne.size() ? ne.begin()[i] : 1;
+    }
+
+    struct ggml_tensor * tensor = ggml_view_4d(ctx, base,
+                                    dims[0], dims[1], dims[2], dims[3],
+                                    cur->nb[1], cur->nb[2], cur->nb[3],
+                                    offset);
+
+    ggml_set_name(tensor, name.c_str());
+
+    n_created++;
+
+    return tensor;
+}
+
 void llama_model_loader::done_getting_tensors(bool partial) const {
     if (n_created > n_tensors) {
         throw std::runtime_error(format("%s: too many tensors created; expected %d, got %d", __func__, n_tensors, n_created));

src/llama-model-loader.h

@@ -67,7 +67,6 @@ struct llama_model_loader {
     static const int TENSOR_DUPLICATED      = 1 << 1;
     static const int TENSOR_SKIP            = 1 << 2;
     static const int TENSOR_SKIP_IF_VIRTUAL = 1 << 3;
-    static const int TENSOR_ALLOW_RESHAPE   = 1 << 4;
 
     int n_kv      = 0;
     int n_tensors = 0;
@@ -178,16 +177,14 @@ struct llama_model_loader {
 
     struct ggml_tensor * require_tensor_meta(const std::string & name) const;
 
-    const struct ggml_tensor * check_tensor_dims(
-            const std::string & name,
-            const std::vector<int64_t> & ne,
-            bool required,
-            bool allow_reshape) const;
+    const struct ggml_tensor * check_tensor_dims(const std::string & name, const std::vector<int64_t> & ne, bool required) const;
 
     struct ggml_tensor * create_tensor(
         const llama_hparams & hparams, const buft_list_t * buft_list_cpu, const buft_list_t * buft_list_input, const buft_list_t * buft_list_output,
         const buft_list_t * buft_list_layer, const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags);
 
+    struct ggml_tensor * create_tensor_as_view(struct ggml_context * ctx, struct ggml_tensor * base, const std::string & name, const std::initializer_list<int64_t> & ne, size_t offset, bool required = true);
+
     void done_getting_tensors(bool partial = false) const;
 
     void init_mappings(bool prefetch = true, llama_mlocks * mlock_mmaps = nullptr);

src/llama-model.cpp

@@ -11,7 +11,6 @@
 #include "llama-kv-cache.h"
 #include "llama-kv-cache-iswa.h"
 #include "llama-kv-cache-dsa.h"
-#include "llama-kv-cache-msa.h"
 #include "llama-kv-cache-dsv4.h"
 #include "llama-memory-hybrid.h"
 #include "llama-memory-hybrid-iswa.h"
@@ -2072,13 +2071,9 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
             {
                 res = nullptr;
             } break;
-        case LLM_ARCH_MINIMAX_M3:
+        case LLM_ARCH_DEEPSEEK32:
             {
-                // sparse (MSA) layers carry an indexer key cache, but leading dense layers do not
-                llama_kv_cache::layer_filter_cb filter_idx =
-                    [&](int32_t il) { return (uint32_t) il >= hparams.n_layer_dense_lead; };
-
-                res = new llama_kv_cache_msa(
+                res = new llama_kv_cache_dsa(
                         *this,
                         params.type_k,
                         params.type_v,
@@ -2091,11 +2086,9 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
                         hparams.n_swa,
                         hparams.swa_type,
                         nullptr,
-                        filter_idx,
                         nullptr);
             } break;
         case LLM_ARCH_GLM_DSA:
-        case LLM_ARCH_DEEPSEEK32:
             {
                 if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP && hparams.n_layer_nextn > 0) {
                     // The NextN/MTP draft head runs dense MLA (no DSA indexer), so the
@@ -2124,11 +2117,10 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
                 } else {
                     // Main context: DSA cache for the trunk layers only - the nextn
                     // layer(s) are never attended by the trunk graph.
-                    llama_kv_cache::layer_filter_cb filter_mla = nullptr;
+                    llama_kv_cache::layer_filter_cb filter = nullptr;
                     if (hparams.n_layer_nextn > 0) {
-                        filter_mla = [&](uint32_t il) { return il < hparams.n_layer(); };
+                        filter = [&](uint32_t il) { return il < hparams.n_layer(); };
                     }
-                    llama_kv_cache::layer_filter_cb filter_lid = [&](uint32_t il) { return il < hparams.n_layer() && (arch != LLM_ARCH_GLM_DSA || hparams.is_indexer_full(il)); };
 
                     res = new llama_kv_cache_dsa(
                             *this,
@@ -2142,89 +2134,19 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
                             1,
                             hparams.n_swa,
                             hparams.swa_type,
-                            filter_mla,
-                            filter_lid,
+                            filter,
                             nullptr);
                 }
             } break;
-        case LLM_ARCH_DEEPSEEK4:
-            {
-                GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE);
-
-                if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) {
-                    const llama_memory_i::layer_filter_cb filter_mtp = [&](int32_t il) {
-                        return il >= (int32_t) hparams.n_layer();
-                    };
-
-                    res = new llama_kv_cache_iswa(
-                            *this,
-                            params.type_k,
-                            params.type_v,
-                            !cparams.flash_attn,
-                            cparams.offload_kqv,
-                            params.swa_full,
-                            cparams.kv_unified,
-                            cparams.n_ctx_seq,
-                            cparams.n_seq_max,
-                            cparams.n_ubatch,
-                            1,
-                            nullptr,
-                            filter_mtp,
-                            nullptr,
-                            nullptr);
-                } else {
-                    res = new llama_kv_cache_dsv4(
-                            *this,
-                            params.type_k,
-                            params.type_v,
-                            !cparams.flash_attn,
-                            cparams.offload_kqv,
-                            params.swa_full,
-                            cparams.kv_unified,
-                            cparams.n_ctx_seq,
-                            cparams.n_seq_max,
-                            cparams.n_ubatch,
-                            1,
-                            cparams.n_rs_seq,
-                            nullptr,
-                            nullptr);
-                }
-            } break;
-        case LLM_ARCH_DFLASH:
-            {
-                // DSV4 DSpark stages store a single MLA-style K per position (window = the draft ring)
-                if (hparams.dsv4_hc_mult > 0) {
-                    GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE);
-
-                    res = new llama_kv_cache_iswa(
-                            *this,
-                            params.type_k,
-                            params.type_v,
-                            !cparams.flash_attn,
-                            cparams.offload_kqv,
-                            params.swa_full,
-                            cparams.kv_unified,
-                            cparams.n_ctx_seq,
-                            cparams.n_seq_max,
-                            cparams.n_ubatch,
-                            1,
-                            nullptr,
-                            nullptr,
-                            nullptr,
-                            nullptr);
-                    break;
-                }
-            }
-            [[fallthrough]];
         // Models that need standard caching should rely on recurrent/hybrid
         // checks
         default:
             {
-                // The MTP head is dense-attention only on hybrid Qwen3-Next/3.5/3.6, so use a plain
+                // The MTP head is dense-attention only on hybrid Qwen3.5/3.6, so use a plain
                 // attention KV cache for the MTP context instead of the hybrid wrapper.
-                const bool mtp_on_hybrid_qwen =
+                const bool mtp_on_hybrid_qwen35 =
                     params.ctx_type == LLAMA_CONTEXT_TYPE_MTP &&
-                    (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE);
+                    (arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE);
 
                 if (llm_arch_is_recurrent(arch)) {
                     res = new llama_memory_recurrent(
@@ -2236,7 +2158,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
                             cparams.n_seq_max,
                             cparams.n_rs_seq,
                             nullptr);
-                } else if (llm_arch_is_hybrid(arch) && !mtp_on_hybrid_qwen) {
+                } else if (llm_arch_is_hybrid(arch) && !mtp_on_hybrid_qwen35) {
                     // The main difference between hybrid architectures is the
                     // layer filters, so pick the right one here
                     llama_memory_hybrid::layer_filter_cb filter_attn = nullptr;
@@ -2251,7 +2173,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
                         filter_recr = [&](uint32_t il) {
                             return hparams.is_recr(il) && hparams.n_ff(il) == 0;
                         };
-                    } else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE) {
+                    } else if (arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE) {
                         filter_attn = [&](uint32_t il) {
                             return il < hparams.n_layer() && !hparams.is_recr(il);
                         };
@@ -2317,12 +2239,12 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
                         };
                     }
 
-                    if (mtp_on_hybrid_qwen) {
+                    if (mtp_on_hybrid_qwen35) {
                         filter = [&](uint32_t il) { return il >= hparams.n_layer(); };
                     }
 
                     if ((arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_HY_V3 || arch == LLM_ARCH_GLM_DSA ||
-                            arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_DEEPSEEK32) &&
+                            arch == LLM_ARCH_MIMO2) &&
                             hparams.n_layer_nextn > 0) {
                         if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) {
                             filter = [&](uint32_t il) { return il >= hparams.n_layer(); };
@@ -2331,7 +2253,24 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
                         }
                     }
 
-                    if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {
+                    if (arch == LLM_ARCH_DEEPSEEK4) {
+                        GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE);
+
+                        res = new llama_kv_cache_dsv4(
+                                *this,
+                                params.type_k,
+                                params.type_v,
+                                !cparams.flash_attn,
+                                cparams.offload_kqv,
+                                params.swa_full,
+                                cparams.kv_unified,
+                                cparams.n_ctx_seq,
+                                cparams.n_seq_max,
+                                cparams.n_ubatch,
+                                1,
+                                filter,
+                                reuse);
+                    } else if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {
                         GGML_ASSERT(hparams.is_swa_any());
 
                         if (arch == LLM_ARCH_GEMMA4_ASSISTANT) {
@@ -2680,11 +2619,8 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
         case LLM_ARCH_STEP35:
         case LLM_ARCH_TALKIE:
         case LLM_ARCH_MELLUM:
-            return LLAMA_ROPE_TYPE_NEOX;
-
         case LLM_ARCH_DFLASH:
-            // DSV4 DSpark drafters use DeepSeek-V4's normal RoPE; legacy DFlash backbones are NeoX
-            return model->hparams.dsv4_hc_mult > 0 ? LLAMA_ROPE_TYPE_NORM : LLAMA_ROPE_TYPE_NEOX;
+            return LLAMA_ROPE_TYPE_NEOX;
 
         case LLM_ARCH_QWEN2VL:
         case LLM_ARCH_PADDLEOCR:
@@ -2867,8 +2803,7 @@ llama_model_base::llama_model_base(const struct llama_model_params & params) : l
     TENSOR_DUPLICATED     (llama_model_loader::TENSOR_DUPLICATED),
     TENSOR_NOT_REQUIRED   (llama_model_loader::TENSOR_NOT_REQUIRED),
     TENSOR_SKIP           (llama_model_loader::TENSOR_SKIP),
-    TENSOR_SKIP_IF_VIRTUAL(llama_model_loader::TENSOR_SKIP_IF_VIRTUAL),
-    TENSOR_ALLOW_RESHAPE  (llama_model_loader::TENSOR_ALLOW_RESHAPE) {}
+    TENSOR_SKIP_IF_VIRTUAL(llama_model_loader::TENSOR_SKIP_IF_VIRTUAL) {}
 
 ggml_tensor * llama_model_base::create_tensor(const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags) {
     GGML_ASSERT(ml != nullptr);

src/llama-model.h

@@ -719,7 +719,6 @@ struct llama_model_base : public llama_model {
     const int TENSOR_NOT_REQUIRED;
     const int TENSOR_SKIP;
     const int TENSOR_SKIP_IF_VIRTUAL;
-    const int TENSOR_ALLOW_RESHAPE;
 
     explicit llama_model_base(const llama_model_params & params);
     virtual ~llama_model_base() = default;

src/llama-sampler.cpp

@@ -2638,8 +2638,7 @@ struct llama_sampler * llama_sampler_init_grammar_lazy_patterns(
 
 // penalties
 
-struct llama_sampler_penalties : public llama_sampler_backend {
-    const int32_t n_vocab;
+struct llama_sampler_penalties {
     const int32_t penalty_last_n;
     const float   penalty_repeat;
     const float   penalty_freq;
@@ -2649,50 +2648,10 @@ struct llama_sampler_penalties : public llama_sampler_backend {
 
     // a frequency map to count token occurrences
     std::unordered_map<llama_token, int> token_count;
-
-    // backend graph inputs
-    ggml_tensor * inp_token_ids = nullptr;
-    ggml_tensor * inp_counts    = nullptr;
-
-    // backend helpers
-    int32_t n_max   = 0;
-    bool has_candidates = false;
-
-    std::vector<int32_t> host_token_ids;
-    std::vector<int32_t> host_counts;
-
-    static bool is_disabled(
-            int32_t penalty_last_n,
-            float   penalty_repeat,
-            float   penalty_freq,
-            float   penalty_present) {
-        return penalty_last_n == 0 ||
-            (penalty_repeat == 1.0f && penalty_freq == 0.0f && penalty_present == 0.0f);
-    }
-
-    bool is_disabled() const {
-        return is_disabled(penalty_last_n, penalty_repeat, penalty_freq, penalty_present);
-    }
-
-    llama_sampler_penalties(
-            int32_t n_vocab,
-            int32_t penalty_last_n,
-            float   penalty_repeat,
-            float   penalty_freq,
-            float   penalty_present)
-        : llama_sampler_backend("penalties")
-        , n_vocab         (n_vocab)
-        , penalty_last_n  (penalty_last_n)
-        , penalty_repeat  (penalty_repeat)
-        , penalty_freq    (penalty_freq)
-        , penalty_present (penalty_present)
-        , prev            (penalty_last_n) {
-    }
 };
 
-static const char * llama_sampler_penalties_name(const struct llama_sampler * smpl) {
-    auto * ctx = (llama_sampler_penalties *) smpl->ctx;
-    return ctx->get_name();
+static const char * llama_sampler_penalties_name(const struct llama_sampler * /*smpl*/) {
+    return "penalties";
 }
 
 static void llama_sampler_penalties_accept(struct llama_sampler * smpl, llama_token token) {
@@ -2729,7 +2688,8 @@ static void llama_sampler_penalties_accept(struct llama_sampler * smpl, llama_to
 static void llama_sampler_penalties_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
     auto * ctx = (llama_sampler_penalties *) smpl->ctx;
 
-    if (ctx->is_disabled()) {
+    if ((ctx->penalty_last_n == 0) ||
+        (ctx->penalty_repeat == 1.0f && ctx->penalty_freq == 0.0f && ctx->penalty_present == 0.0f)) {
         return;
     }
 
@@ -2767,7 +2727,6 @@ static void llama_sampler_penalties_reset(struct llama_sampler * smpl) {
 static struct llama_sampler * llama_sampler_penalties_clone(const struct llama_sampler * smpl) {
     const auto * ctx = (const llama_sampler_penalties *) smpl->ctx;
     auto * result = llama_sampler_init_penalties(
-            ctx->n_vocab,
             ctx->penalty_last_n,
             ctx->penalty_repeat,
             ctx->penalty_freq,
@@ -2777,8 +2736,7 @@ static struct llama_sampler * llama_sampler_penalties_clone(const struct llama_s
     {
         auto * result_ctx = (llama_sampler_penalties *) result->ctx;
 
-        result_ctx->prev        = ctx->prev;
-        result_ctx->token_count = ctx->token_count;
+        result_ctx->prev = ctx->prev;
     }
 
     return result;
@@ -2788,170 +2746,6 @@ static void llama_sampler_penalties_free(struct llama_sampler * smpl) {
     delete (llama_sampler_penalties *) smpl->ctx;
 }
 
-static bool llama_sampler_penalties_backend_init(
-        struct llama_sampler       * smpl,
-        ggml_backend_buffer_type_t   buft) {
-    auto * sctx = (llama_sampler_penalties *) smpl->ctx;
-
-    const bool res = llama_sampler_backend_support(smpl, buft);
-
-    sctx->init(res);
-
-    return res;
-}
-
-static void llama_sampler_penalties_backend_apply(
-        struct llama_sampler      * smpl,
-        struct ggml_context       * ctx,
-        struct ggml_cgraph        * gf,
-        struct llama_sampler_data * data) {
-    GGML_UNUSED(gf);
-
-    auto * sctx = (llama_sampler_penalties *) smpl->ctx;
-
-    if (sctx->is_disabled()) {
-        return;
-    }
-
-    GGML_ASSERT(sctx->n_vocab > 0);
-
-    sctx->has_candidates = data->candidates != nullptr;
-    sctx->n_max   = std::min(sctx->penalty_last_n, sctx->n_vocab);
-
-    sctx->inp_token_ids = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, sctx->n_max);
-    ggml_set_name(sctx->inp_token_ids, "penalties_token_ids");
-    ggml_set_input(sctx->inp_token_ids);
-
-    sctx->inp_counts = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, sctx->n_max);
-    ggml_set_name(sctx->inp_counts, "penalties_counts");
-    ggml_set_input(sctx->inp_counts);
-
-    if ((int32_t) sctx->host_token_ids.size() != sctx->n_max) {
-        sctx->host_token_ids.assign(sctx->n_max, 0);
-        sctx->host_counts.assign(sctx->n_max, 0);
-    }
-
-    // flatten
-    ggml_tensor * logits = ggml_reshape_1d(ctx, data->logits, ggml_nelements(data->logits));
-    ggml_tensor * gathered = logits;
-    ggml_tensor * counts_f32 = ggml_cast(ctx, sctx->inp_counts, GGML_TYPE_F32);
-
-    if (sctx->has_candidates) {
-        ggml_tensor * candidates = ggml_reshape_1d(
-                ctx, data->candidates, ggml_nelements(data->candidates));
-        const int64_t n_candidates = candidates->ne[0];
-        GGML_ASSERT(n_candidates == ggml_nelements(logits));
-
-        ggml_tensor * counts_rows = ggml_fill(
-                ctx, ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1, sctx->n_vocab), 0.0f);
-        ggml_tensor * scatter_rows = ggml_reshape_2d(ctx, counts_f32, 1, sctx->n_max);
-        counts_rows = ggml_set_rows(ctx, counts_rows, scatter_rows, sctx->inp_token_ids);
-        counts_f32 = ggml_get_rows(ctx, counts_rows, candidates);
-        counts_f32 = ggml_reshape_1d(ctx, counts_f32, n_candidates);
-    } else {
-        ggml_tensor * logits_rows = ggml_reshape_2d(ctx, logits, 1, ggml_nelements(logits));
-        gathered = ggml_get_rows(ctx, logits_rows, sctx->inp_token_ids);
-        gathered = ggml_reshape_1d(ctx, gathered, sctx->n_max);
-    }
-
-    ggml_tensor * active_mask = ggml_step(ctx, counts_f32);
-    ggml_tensor * inactive_mask = ggml_sub(ctx, ggml_fill(ctx, active_mask, 1.0f), active_mask);
-
-    ggml_tensor * penalized = gathered;
-
-    if (sctx->penalty_repeat != 1.0f) {
-        ggml_tensor * pos_mask = ggml_step(ctx, penalized);
-        ggml_tensor * neg_mask = ggml_sub(ctx, ggml_fill(ctx, pos_mask, 1.0f), pos_mask);
-
-        ggml_tensor * pos_scale = ggml_scale(ctx, pos_mask, 1.0f/sctx->penalty_repeat);
-        ggml_tensor * neg_scale = ggml_scale(ctx, neg_mask, sctx->penalty_repeat);
-        ggml_tensor * repeat_scale = ggml_add(ctx, pos_scale, neg_scale);
-
-        // scale inactive entries with 1 to avoid -INF * 0 = NaN for values masked by top-p
-        repeat_scale = ggml_mul(ctx, repeat_scale, active_mask);
-        repeat_scale = ggml_add(ctx, repeat_scale, inactive_mask);
-        penalized = ggml_mul(ctx, gathered, repeat_scale);
-    }
-
-    if (sctx->penalty_freq != 0.0f) {
-        ggml_tensor * penalty_freq = ggml_scale(ctx, counts_f32, sctx->penalty_freq);
-        penalized = ggml_sub(ctx, penalized, penalty_freq);
-    }
-
-    if (sctx->penalty_present != 0.0f) {
-        ggml_tensor * penalty_present = ggml_scale(ctx, active_mask, sctx->penalty_present);
-        penalized = ggml_sub(ctx, penalized, penalty_present);
-    }
-
-    if (sctx->has_candidates) {
-        data->logits = penalized;
-    } else {
-        ggml_tensor * logits_rows = ggml_reshape_2d(ctx, logits, 1, ggml_nelements(logits));
-        ggml_tensor * scatter_rows = ggml_reshape_2d(ctx, penalized, 1, sctx->n_max);
-        logits_rows = ggml_set_rows(ctx, logits_rows, scatter_rows, sctx->inp_token_ids);
-        data->logits = ggml_reshape_1d(ctx, logits_rows, ggml_nelements(logits));
-    }
-}
-
-static void llama_sampler_penalties_backend_set_input(struct llama_sampler * smpl) {
-    auto * sctx = (llama_sampler_penalties *) smpl->ctx;
-
-    if (!sctx->inp_token_ids || !sctx->inp_counts || sctx->n_max <= 0 || sctx->n_vocab <= 0) {
-        return;
-    }
-
-    if (sctx->is_disabled()) {
-        return;
-    }
-
-    // fill active entries from the map
-    int32_t n_active = 0;
-
-    for (const auto & it : sctx->token_count) {
-        GGML_ASSERT(n_active < sctx->n_max);
-        sctx->host_token_ids[n_active] = it.first;
-        sctx->host_counts   [n_active] = it.second;
-        ++n_active;
-    }
-
-    // Sorting is required because backend_apply uses ggml_set_rows (a scatter-back operation)
-    std::vector<std::pair<int32_t, int32_t>> entries;
-    entries.reserve(n_active);
-    for (int32_t i = 0; i < n_active; ++i) {
-        entries.emplace_back(sctx->host_token_ids[i], sctx->host_counts[i]);
-    }
-    std::sort(entries.begin(), entries.end(), [](const auto & a, const auto & b) {
-        return a.first < b.first;
-    });
-    for (int32_t i = 0; i < n_active; ++i) {
-        sctx->host_token_ids[i] = entries[i].first;
-        sctx->host_counts   [i] = entries[i].second;
-    }
-
-    // Padding: Finds a filler token id that is not present in token_count.
-    // Use it to do padding for the arrays, it avoids resizing every time.
-    // The arrays must always have exactly n_max entries (the GPU tensor is a fixed size).
-    int32_t filler = 0;
-    if (n_active < sctx->n_max) {
-        while (sctx->token_count.find(filler) != sctx->token_count.end()) {
-            ++filler;
-        }
-        GGML_ASSERT(filler < sctx->n_vocab);
-    }
-
-    // Fill the rest of the arrays with the filler token id and count 0.
-    // Inactive slots are padded with a unique dummy token ID (count = 0).
-    // The uniqueness matters because ggml_set_rows with duplicate indices can produce non-deterministic or incorrect results.
-    // Using a filler token with count 0 that isn't in the active set is safe, because the active_mask step in backend_apply filters them out via ggml_step(counts_f32)
-    for (int32_t i = n_active; i < sctx->n_max; ++i) {
-        sctx->host_token_ids[i] = filler;
-        sctx->host_counts   [i] = 0;
-    }
-
-    ggml_backend_tensor_set(sctx->inp_token_ids, sctx->host_token_ids.data(), 0, sctx->n_max * sizeof(int32_t));
-    ggml_backend_tensor_set(sctx->inp_counts,    sctx->host_counts.data(),    0, sctx->n_max * sizeof(int32_t));
-}
-
 static struct llama_sampler_i llama_sampler_penalties_i = {
     /* .name              = */ llama_sampler_penalties_name,
     /* .accept            = */ llama_sampler_penalties_accept,
@@ -2959,33 +2753,35 @@ static struct llama_sampler_i llama_sampler_penalties_i = {
     /* .reset             = */ llama_sampler_penalties_reset,
     /* .clone             = */ llama_sampler_penalties_clone,
     /* .free              = */ llama_sampler_penalties_free,
-    /* .backend_init      = */ llama_sampler_penalties_backend_init,
+    /* .backend_init      = */ nullptr,
     /* .backend_accept    = */ nullptr,
-    /* .backend_apply     = */ llama_sampler_penalties_backend_apply,
-    /* .backend_set_input = */ llama_sampler_penalties_backend_set_input,
+    /* .backend_apply     = */ nullptr,
+    /* .backend_set_input = */ nullptr,
 };
 
 struct llama_sampler * llama_sampler_init_penalties(
-        int32_t n_vocab,
         int32_t penalty_last_n,
         float penalty_repeat,
         float penalty_freq,
         float penalty_present) {
     penalty_last_n = std::max(penalty_last_n, 0);
 
-    if (llama_sampler_penalties::is_disabled(
-                penalty_last_n, penalty_repeat, penalty_freq, penalty_present)) {
+    const bool is_empty = (penalty_last_n == 0 || (penalty_repeat == 1.0f && penalty_freq == 0.0f && penalty_present == 0.0f));
+
+    if (is_empty) {
         return llama_sampler_init_empty("?penalties");
     }
 
     return llama_sampler_init(
         /* .iface = */ &llama_sampler_penalties_i,
-        /* .ctx   = */ new llama_sampler_penalties(
-            n_vocab,
-            penalty_last_n,
-            penalty_repeat,
-            penalty_freq,
-            penalty_present)
+        /* .ctx   = */ new llama_sampler_penalties {
+            /* .penalty_last_n  = */ penalty_last_n,
+            /* .penalty_repeat  = */ penalty_repeat,
+            /* .penalty_freq    = */ penalty_freq,
+            /* .penalty_present = */ penalty_present,
+            /* .prev            = */ ring_buffer<llama_token>(penalty_last_n),
+            /* .token_count     = */ {},
+        }
     );
 }
 

src/llama-vocab.cpp

@@ -1373,10 +1373,8 @@ struct llm_tokenizer_plamo2 : llm_tokenizer {
             if (vocab.is_byte(token_id)) {
                 if (entry.text.length() == 6 && entry.text.substr(0, 3) == "<0x" && entry.text.back() == '>') {
                     std::string hex_str = entry.text.substr(3, 2);
-                    if (std::isxdigit(static_cast<unsigned char>(hex_str[0])) && std::isxdigit(static_cast<unsigned char>(hex_str[1]))) {
-                        int byte_val = std::stoi(hex_str, nullptr, 16);
-                        bytes_[byte_val] = static_cast<llama_token>(token_id);
-                    }
+                    int byte_val = std::stoi(hex_str, nullptr, 16);
+                    bytes_[byte_val] = static_cast<llama_token>(token_id);
                 }
                 continue;
             }
@@ -2534,12 +2532,6 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
             const std::string & key = kv(std::get<0>(it));
             int32_t & id = std::get<1>(it);
 
-            if (id >= 0 && static_cast<size_t>(id) >= id_to_token.size()) {
-                LLAMA_LOG_WARN("%s: default special token '%s' = %d out of vocab range, disabling\n",
-                    __func__, key.c_str(), id);
-                id = LLAMA_TOKEN_NULL;
-            }
-
             uint32_t new_id;
             if (!ml.get_key(std::get<0>(it), new_id, false)) {
                 continue;
@@ -3627,15 +3619,12 @@ int32_t llama_vocab::impl::token_to_piece(llama_token token, char * buf, int32_t
                 if (vocab.is_byte(token)) {
                     // Handle byte tokens like <0xXX>
                     if (token_text.length() == 6 && token_text.substr(0, 3) == "<0x" && token_text.back() == '>') {
-                        std::string hex_str = token_text.substr(3, 2);
-                        if (std::isxdigit(static_cast<unsigned char>(hex_str[0])) && std::isxdigit(static_cast<unsigned char>(hex_str[1]))) {
-                            int hex_val = std::stoi(hex_str, nullptr, 16);
-                            if (length < 1) {
-                                return -1;
-                            }
-                            buf[0] = static_cast<char>(hex_val);
-                            return 1;
+                        int hex_val = std::stoi(token_text.substr(3, 2), nullptr, 16);
+                        if (length < 1) {
+                            return -1;
                         }
+                        buf[0] = static_cast<char>(hex_val);
+                        return 1;
                     }
                 }
 

src/models/deepseek2.cpp

@@ -37,11 +37,6 @@ void llama_model_deepseek2::load_arch_hparams(llama_model_loader & ml) {
         hparams.rope_yarn_log_mul /= 0.1f;
     }
 
-    // NextN/MTP
-    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
-    GGML_ASSERT(hparams.n_layer_nextn == 0 ||
-        hparams.n_layer() + hparams.n_layer_nextn == hparams.n_layer_all);
-
     // (optional) temperature tuning - used by mistral-large
     ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_SCALE,  hparams.f_attn_temp_scale,       false);
     ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_LENGTH, hparams.n_attn_temp_floor_scale, false); // FIXME why not use temperature_length?
@@ -57,20 +52,10 @@ void llama_model_deepseek2::load_arch_hparams(llama_model_loader & ml) {
     }
 }
 
-void llama_model_deepseek2::load_arch_tensors(llama_model_loader & ml) {
+void llama_model_deepseek2::load_arch_tensors(llama_model_loader &) {
     LLAMA_LOAD_LOCALS;
     const int64_t n_expert_shared = hparams.n_expert_shared;
 
-    const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
-    const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";
-    const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);
-    const int trunk_flags = mtp_only  ? TENSOR_NOT_REQUIRED : 0;
-    int       mtp_flags   = trunk_only ? TENSOR_NOT_REQUIRED : 0;
-
-    if (!ml.load_mtp) {
-        mtp_flags |= TENSOR_SKIP;
-    }
-
     const bool is_mla = hparams.is_mla();
 
     // note: these are the actual head sizes you get when treating as MHA or after "decompression" using wv_b for MLA
@@ -96,45 +81,44 @@ void llama_model_deepseek2::load_arch_tensors(llama_model_loader & ml) {
         output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
     }
 
-    for (int i = 0; i < n_layer_all; ++i) {
+    for (int i = 0; i < n_layer; ++i) {
         auto & layer = layers[i];
-        const int flags = i < n_layer ? trunk_flags : mtp_flags;
 
-        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, flags);
+        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
         if (q_lora_rank > 0) {
-            layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, flags);
+            layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, 0);
         }
 
-        layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, "weight", i), {kv_lora_rank}, flags);
+        layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, "weight", i), {kv_lora_rank}, 0);
 
         if (q_lora_rank > 0) {
-            layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, flags);
-            layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * n_embd_head_k_mla}, flags);
+            layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, 0);
+            layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * n_embd_head_k_mla}, 0);
         } else {
-            layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_head * n_embd_head_k_mla}, flags);
+            layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_head * n_embd_head_k_mla}, 0);
         }
 
-        layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, "weight", i), {n_embd, kv_lora_rank + n_embd_head_qk_rope}, flags);
+        layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, "weight", i), {n_embd, kv_lora_rank + n_embd_head_qk_rope}, 0);
 
         // note: only old legacy GGUF files will have the unsplit wkv_b tensor in
         if (is_mla) {
-            layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K_B, "weight", i), {n_embd_head_qk_nope, kv_lora_rank, n_head}, flags);
-            layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V_B, "weight", i), {kv_lora_rank, n_embd_head_v_mla, n_head}, flags);
+            layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K_B, "weight", i), {n_embd_head_qk_nope, kv_lora_rank, n_head}, 0);
+            layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V_B, "weight", i), {kv_lora_rank, n_embd_head_v_mla, n_head}, 0);
         } else {
-            layer.wkv_b = create_tensor(tn(LLM_TENSOR_ATTN_KV_B, "weight", i), {kv_lora_rank, n_head * (n_embd_head_qk_nope + n_embd_head_v_mla)}, flags);
+            layer.wkv_b = create_tensor(tn(LLM_TENSOR_ATTN_KV_B, "weight", i), {kv_lora_rank, n_head * (n_embd_head_qk_nope + n_embd_head_v_mla)}, 0);
         }
 
-        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head * n_embd_head_v_mla, n_embd}, flags);
+        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head * n_embd_head_v_mla, n_embd}, 0);
 
-        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, flags);
+        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
 
         if (i < (int) hparams.n_layer_dense_lead) {
-            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd,   n_ff}, flags);
-            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {  n_ff, n_embd}, flags);
-            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   "weight", i), {n_embd,   n_ff}, flags);
+            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd,   n_ff}, 0);
+            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {  n_ff, n_embd}, 0);
+            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   "weight", i), {n_embd,   n_ff}, 0);
         } else {
-            layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, flags);
-            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED | flags);
+            layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
+            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED);
 
             if (n_expert == 0) {
                 throw std::runtime_error("n_expert must be > 0");
@@ -144,281 +128,21 @@ void llama_model_deepseek2::load_arch_tensors(llama_model_loader & ml) {
             }
 
             // MoE branch
-            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp,   n_embd, n_expert}, flags);
-            create_tensor_gate_up_exps(layer, i, n_embd, n_ff_exp, n_expert, flags);
+            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp,   n_embd, n_expert}, 0);
+            create_tensor_gate_up_exps(layer, i, n_embd, n_ff_exp, n_expert, 0);
 
             // Shared expert branch
-            layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, flags);
-            layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {        n_ff_exp * n_expert_shared, n_embd}, flags);
-            layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   "weight", i), {n_embd, n_ff_exp * n_expert_shared}, flags);
-        }
-
-        // NextN/MTP tensors
-        if (i >= n_layer) {
-            layer.nextn.eh_proj          = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), { 2 * n_embd, n_embd }, mtp_flags);
-            layer.nextn.enorm            = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), { n_embd }, mtp_flags);
-            layer.nextn.hnorm            = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", i), { n_embd }, mtp_flags);
-            layer.nextn.embed_tokens     = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", i), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED | flags);
-            layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED | flags);
-            layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), { n_embd }, TENSOR_NOT_REQUIRED | flags);
+            layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0);
+            layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {        n_ff_exp * n_expert_shared, n_embd}, 0);
+            layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0);
         }
     }
 }
 
 std::unique_ptr<llm_graph_context> llama_model_deepseek2::build_arch_graph(const llm_graph_params & params) const {
-    if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) {
-        return std::make_unique<graph_mtp>(*this, params);
-    }
     return std::make_unique<graph>(*this, params);
 }
 
-llama_model_deepseek2::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) :
-    llm_graph_context(params) {
-    GGML_ASSERT(hparams.n_layer_nextn > 0 && "GLM4 MTP requires n_layer_nextn > 0");
-    GGML_ASSERT(hparams.n_layer_nextn == 1 && "GLM4 MTP currently only supports a single MTP block");
-    GGML_ASSERT(hparams.is_mla() && "GLM4 MTP requires MLA");
-    GGML_ASSERT(hparams.f_attn_temp_scale == 0.0f && "GLM4 MTP does not support attention temperature scaling");
-
-    // The appended MTP block is stored immediately after the main decoder layers.
-    const int il = hparams.n_layer();
-    const auto & layer = model.layers[il];
-
-    GGML_ASSERT(layer.nextn.eh_proj && "MTP block missing nextn.eh_proj");
-    GGML_ASSERT(layer.nextn.enorm   && "MTP block missing nextn.enorm");
-    GGML_ASSERT(layer.nextn.hnorm   && "MTP block missing nextn.hnorm");
-
-    GGML_ASSERT((uint32_t) il >= hparams.n_layer_dense_lead && "GLM4 MTP block expected to use MoE FFN");
-
-    const int64_t n_embd_head_k_mla   = hparams.n_embd_head_k_mla();
-    const int64_t n_embd_head_qk_rope = hparams.n_rot();
-    const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope;
-    const int64_t kv_lora_rank        = hparams.n_lora_kv;
-
-    GGML_ASSERT(n_embd_head_qk_nope >= 1);
-    GGML_ASSERT(hparams.n_lora_q > 0);
-    GGML_ASSERT(layer.wq_a);
-    GGML_ASSERT(layer.attn_q_a_norm);
-    GGML_ASSERT(layer.wq_b);
-    GGML_ASSERT(layer.wkv_a_mqa);
-    GGML_ASSERT(layer.attn_kv_a_norm);
-    GGML_ASSERT(layer.wk_b);
-
-    const bool has_split_exps =
-            layer.ffn_up_exps   != nullptr &&
-            layer.ffn_gate_exps != nullptr;
-
-    const bool has_fused_exps = layer.ffn_gate_up_exps != nullptr;
-
-    GGML_ASSERT(has_split_exps || has_fused_exps);
-    GGML_ASSERT(layer.ffn_norm);
-    GGML_ASSERT(layer.ffn_gate_inp);
-    GGML_ASSERT(layer.ffn_down_exps);
-    GGML_ASSERT(layer.ffn_gate_shexp);
-    GGML_ASSERT(layer.ffn_down_shexp);
-    GGML_ASSERT(layer.ffn_up_shexp);
-
-    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd);
-
-    inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
-    ggml_set_input(inp->tokens);
-
-    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens);
-    ggml_set_input(inp->embd);
-
-    ggml_tensor * tok_embd;
-    if (ubatch.token) {
-        ggml_tensor * tok_embd_w = layer.nextn.embed_tokens
-                ? layer.nextn.embed_tokens
-                : model.tok_embd;
-
-        tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens);
-    } else {
-        tok_embd = inp->embd;
-    }
-    cb(tok_embd, "mtp_tok_embd", il);
-
-    inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);
-    ggml_set_input(inp->h);
-    ggml_set_name(inp->h, "mtp_h_input");
-
-    ggml_tensor * h_embd = inp->h;
-
-    res->add_input(std::move(inp));
-
-    ggml_tensor * inp_pos     = build_inp_pos();
-    ggml_tensor * inp_out_ids = build_inp_out_ids();
-
-    auto * inp_attn_k = build_attn_inp_k();
-
-    ggml_tensor * h_norm = build_norm(h_embd, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il);
-    cb(h_norm, "mtp_hnorm", il);
-
-    ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il);
-    cb(e_norm, "mtp_enorm", il);
-
-    ggml_tensor * concat = ggml_concat(ctx0, e_norm, h_norm, 0);
-    cb(concat, "mtp_concat", il);
-
-    ggml_tensor * cur = build_lora_mm(layer.nextn.eh_proj, concat, layer.nextn.eh_proj_s);
-    cb(cur, "mtp_eh_proj", il);
-
-    ggml_tensor * inpSA = cur;
-
-    cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
-    cb(cur, "mtp_attn_norm", il);
-
-    ggml_tensor * q = ggml_mul_mat(ctx0, layer.wq_a, cur);
-    cb(q, "mtp_q_a", il);
-
-    q = build_norm(q, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il);
-    cb(q, "mtp_q_a_norm", il);
-
-    q = ggml_mul_mat(ctx0, layer.wq_b, q);
-    cb(q, "mtp_q_b", il);
-
-    ggml_tensor * q_nope =
-        ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens,
-                ggml_row_size(q->type, n_embd_head_k_mla),
-                ggml_row_size(q->type, n_embd_head_k_mla) * n_head, 0);
-    cb(q_nope, "mtp_q_nope", il);
-
-    ggml_tensor * q_pe =
-        ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens,
-                ggml_row_size(q->type, n_embd_head_k_mla),
-                ggml_row_size(q->type, n_embd_head_k_mla) * n_head,
-                ggml_row_size(q->type, n_embd_head_qk_nope));
-    cb(q_pe, "mtp_q_pe", il);
-
-    ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, layer.wkv_a_mqa, cur);
-    cb(kv_cmpr_pe, "mtp_kv_cmpr_pe", il);
-
-    ggml_tensor * kv_cmpr =
-        ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens,
-                ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), 0);
-    cb(kv_cmpr, "mtp_kv_cmpr", il);
-
-    ggml_tensor * k_pe =
-        ggml_view_3d(ctx0, kv_cmpr_pe, n_embd_head_qk_rope, 1, n_tokens,
-                ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),
-                ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),
-                ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));
-    cb(k_pe, "mtp_k_pe", il);
-
-    kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);
-    cb(kv_cmpr, "mtp_kv_cmpr_norm", il);
-
-    GGML_ASSERT(ext_factor >= 0.0f);
-
-    const float attn_factor_org =
-            attn_factor * (1.0f + 0.1f * logf(1.0f / freq_scale));
-
-    const float mscale =
-            attn_factor_org * (1.0f + 0.1f * hparams.rope_yarn_log_mul * logf(1.0f / freq_scale));
-
-    const float kq_scale =
-            1.0f * mscale * mscale / sqrtf(float(n_embd_head_k_mla));
-
-    q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr,
-            n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
-            ext_factor, attn_factor, beta_fast, beta_slow);
-    cb(q_pe, "mtp_q_pe_rope", il);
-
-    k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr,
-            n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
-            ext_factor, attn_factor, beta_fast, beta_slow);
-    cb(k_pe, "mtp_k_pe_rope", il);
-
-    q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3);
-    cb(q_nope, "mtp_q_nope_perm", il);
-
-    ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, layer.wk_b, q_nope);
-    cb(q_nope_absorbed, "mtp_q_nope_absorbed", il);
-
-    q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3);
-    cb(q_nope_absorbed, "mtp_q_nope_absorbed_perm", il);
-
-    ggml_tensor * Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0);
-    cb(Qcur, "mtp_Qcur", il);
-
-    kv_cmpr = ggml_reshape_3d(ctx0, kv_cmpr, hparams.n_lora_kv, 1, n_tokens);
-    cb(kv_cmpr, "mtp_kv_cmpr_reshape", il);
-
-    ggml_tensor * Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0);
-    cb(Kcur, "mtp_Kcur", il);
-
-    ggml_tensor * Vcur = kv_cmpr;
-    cb(Vcur, "mtp_Vcur", il);
-
-    cur = build_attn(inp_attn_k,
-            layer.wo, nullptr, layer.wo_s,
-            Qcur, Kcur, Vcur, nullptr, nullptr, layer.wv_b, kq_scale, il);
-    cb(cur, "mtp_attn_out", il);
-
-    ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
-    cb(ffn_inp, "mtp_ffn_inp", il);
-
-    cur = build_norm(ffn_inp, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
-    cb(cur, "mtp_ffn_norm", il);
-
-    ggml_tensor * moe_out = build_moe_ffn(cur,
-            layer.ffn_gate_inp,
-            layer.ffn_up_exps,
-            layer.ffn_gate_exps,
-            layer.ffn_down_exps,
-            layer.ffn_exp_probs_b,
-            n_expert, n_expert_used,
-            LLM_FFN_SILU, hparams.expert_weights_norm,
-            hparams.expert_weights_scale,
-            (llama_expert_gating_func_type) hparams.expert_gating_func,
-            il,
-            nullptr,
-            layer.ffn_gate_up_exps);
-    cb(moe_out, "mtp_ffn_moe_out", il);
-
-    ggml_tensor * ffn_shexp = build_ffn(cur,
-            layer.ffn_up_shexp, nullptr, nullptr,
-            layer.ffn_gate_shexp, nullptr, nullptr,
-            layer.ffn_down_shexp, nullptr, nullptr,
-            nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il);
-    cb(ffn_shexp, "mtp_ffn_shexp", il);
-
-    cur = ggml_add(ctx0, moe_out, ffn_shexp);
-    cb(cur, "mtp_ffn_out", il);
-
-    cur = ggml_add(ctx0, cur, ffn_inp);
-    cb(cur, "mtp_post_ffn", il);
-
-    ggml_tensor * head_norm_w = layer.nextn.shared_head_norm
-            ? layer.nextn.shared_head_norm
-            : model.output_norm;
-    GGML_ASSERT(head_norm_w && "GLM4 MTP: missing both nextn.shared_head_norm and output_norm");
-
-    cur = build_norm(cur, head_norm_w, nullptr, LLM_NORM_RMS, -1);
-    cb(cur, "h_nextn", -1);
-    res->t_h_nextn = cur;
-
-    if (inp_out_ids) {
-        cur = ggml_get_rows(ctx0, cur, inp_out_ids);
-    }
-    cb(cur, "mtp_shared_head_norm", -1);
-
-    ggml_tensor * head_w = layer.nextn.shared_head_head
-            ? layer.nextn.shared_head_head
-            : model.output;
-
-    ggml_tensor * head_s = layer.nextn.shared_head_head
-            ? layer.nextn.shared_head_head_s
-            : model.output_s;
-
-    GGML_ASSERT(head_w && "GLM4 MTP: missing LM head (nextn.shared_head_head or model.output)");
-
-    cur = build_lora_mm(head_w, cur, head_s);
-    cb(cur, "result_output", -1);
-
-    res->t_logits = cur;
-    ggml_build_forward_expand(gf, cur);
-}
-
 llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_params & params) :
     llm_graph_context(params) {
     // lite variants include DeepSeek-V2-Lite, GigaChat3-10B-A1.8B
@@ -641,7 +365,7 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p
                             Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
             }
         }
-        if (il == n_layer - 1 && inp_out_ids && (!cparams.embeddings_nextn || cparams.embeddings_nextn_masked)) {
+        if (il == n_layer - 1 && inp_out_ids) {
             cur   = ggml_get_rows(ctx0, cur, inp_out_ids);
             inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
         }
@@ -701,13 +425,6 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p
 
     cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
 
-    cb(cur, "h_nextn", -1);
-    res->t_h_nextn = cur;
-
-    if (cparams.embeddings_nextn && !cparams.embeddings_nextn_masked && inp_out_ids) {
-        cur = ggml_get_rows(ctx0, cur, inp_out_ids);
-    }
-
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 

src/models/deepseek32.cpp

@@ -44,24 +44,13 @@ void llama_model_deepseek32::load_arch_hparams(llama_model_loader & ml) {
     GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer");
 
     switch (hparams.n_layer()) {
-        case 61: type = LLM_TYPE_685B_A37B; break;
+        case 62: type = LLM_TYPE_685B_A37B; break;
         default: type = LLM_TYPE_UNKNOWN;
     }
 }
 
-void llama_model_deepseek32::load_arch_tensors(llama_model_loader & ml) {
+void llama_model_deepseek32::load_arch_tensors(llama_model_loader &) {
     LLAMA_LOAD_LOCALS;
-
-    const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
-    const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";
-    const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);
-    const int trunk_flags = mtp_only   ? TENSOR_NOT_REQUIRED : 0;
-    int       mtp_flags   = trunk_only ? TENSOR_NOT_REQUIRED : 0;
-
-    if (!ml.load_mtp) {
-        mtp_flags |= TENSOR_SKIP;
-    }
-
     const bool is_mla = hparams.is_mla();
     if (!is_mla) {
         throw std::runtime_error("DEEPSEEK32 architecture requires MLA");
@@ -91,7 +80,12 @@ void llama_model_deepseek32::load_arch_tensors(llama_model_loader & ml) {
     }
 
     for (int i = 0; i < n_layer_all; ++i) {
-        const int flags = (i >= n_layer) ? mtp_flags : trunk_flags;
+        int flags = 0;
+        if (i >= n_layer) {
+            // skip all tensors in the NextN layers
+            // TODO @ngxson : TENSOR_NOT_REQUIRED was a hack, need to remove it later
+            flags |= TENSOR_SKIP | TENSOR_NOT_REQUIRED;
+        }
 
         auto & layer = layers[i];
 
@@ -144,7 +138,7 @@ void llama_model_deepseek32::load_arch_tensors(llama_model_loader & ml) {
             layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   "weight", i), {n_embd, n_ff_exp * n_expert_shared}, flags);
         }
 
-        // NextN/MTP tensors - conditionally load for last nextn_predict_layers
+        // NextN/MTP tensors (preserved but unused) - conditionally load for last nextn_predict_layers
         if (i >= n_layer) {
             layer.nextn.eh_proj          = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), { 2 * n_embd, n_embd }, flags);
             layer.nextn.enorm            = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), { n_embd }, flags);
@@ -159,9 +153,6 @@ void llama_model_deepseek32::load_arch_tensors(llama_model_loader & ml) {
 }
 
 std::unique_ptr<llm_graph_context> llama_model_deepseek32::build_arch_graph(const llm_graph_params & params) const {
-    if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) {
-        return std::make_unique<graph_mtp>(*this, params);
-    }
     return std::make_unique<graph>(*this, params);
 }
 
@@ -439,9 +430,7 @@ llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_
                         Qcur, Kcur, Vcur, nullptr, nullptr, model.layers[il].wv_b, top_k, kq_scale, il);
             }
         }
-        // when unmasked nextn embeddings are requested, t_h_nextn must keep all rows,
-        // so the early output masking has to be skipped (it is applied after the final norm instead)
-        if (il == n_layer - 1 && inp_out_ids && (!cparams.embeddings_nextn || cparams.embeddings_nextn_masked)) {
+        if (il == n_layer - 1 && inp_out_ids) {
             cur   = ggml_get_rows(ctx0, cur, inp_out_ids);
             inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
         }
@@ -504,14 +493,6 @@ llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_
 
     cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
 
-    // post-norm hidden state feeds the NextN/MTP draft head
-    cb(cur, "h_nextn", -1);
-    res->t_h_nextn = cur;
-
-    if (cparams.embeddings_nextn && !cparams.embeddings_nextn_masked && inp_out_ids) {
-        cur = ggml_get_rows(ctx0, cur, inp_out_ids);
-    }
-
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
@@ -523,243 +504,3 @@ llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_
 
     ggml_build_forward_expand(gf, cur);
 }
-
-// LLM_GRAPH_TYPE_DECODER_MTP draft head for DeepSeek V3.2 (DEEPSEEK32).
-// Semantics mirror the deepseek-family NextN/MTP layer:
-//   enorm(embed) + hnorm(prev_hidden) -> concat(e, h) -> eh_proj ->
-//   full deepseek32 decoder block (dense MLA attention + sigmoid-gated MoE FFN
-//   with shared expert, exactly as the trunk deepseek2 graph builds it) ->
-//   shared_head_norm (fallback output_norm) -> shared LM head.
-// The DSA indexer is not used at runtime.
-llama_model_deepseek32::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params)
-    : llm_graph_context(params) {
-    GGML_ASSERT(hparams.n_layer_nextn > 0 && "DEEPSEEK32 MTP requires n_layer_nextn > 0");
-    GGML_ASSERT(hparams.n_layer_nextn == 1 && "DEEPSEEK32 MTP currently only supports a single MTP block");
-    GGML_ASSERT(hparams.is_mla() && "DEEPSEEK32 MTP requires MLA");
-
-    const int il = hparams.n_layer() + cparams.nextn_layer_offset;
-    GGML_ASSERT(cparams.nextn_layer_offset >= 0 &&
-                cparams.nextn_layer_offset < (int) hparams.n_layer_nextn &&
-                "nextn_layer_offset out of range [0, n_layer_nextn)");
-    const auto & layer = model.layers[il];
-
-    GGML_ASSERT(layer.nextn.eh_proj && "MTP block missing nextn.eh_proj");
-    GGML_ASSERT(layer.nextn.enorm   && "MTP block missing nextn.enorm");
-    GGML_ASSERT(layer.nextn.hnorm   && "MTP block missing nextn.hnorm");
-    GGML_ASSERT(layer.ffn_gate_inp  && "MTP block missing ffn_gate_inp");
-
-    // note: these are the actual head sizes you get when treating as MHA or after "decompression" using wv_b for MLA
-    const int64_t n_embd_head_k = hparams.n_embd_head_k_mla();
-
-    const int64_t n_embd_head_qk_rope = hparams.n_rot();
-    const int64_t n_embd_head_qk_nope = n_embd_head_k - n_embd_head_qk_rope;
-
-    const uint32_t kv_lora_rank = hparams.n_lora_kv;
-
-    // We have to pre-scale kq_scale and attn_factor to make the YaRN RoPE work correctly.
-    // See the deepseek2 trunk graph for the detailed explanation - this must match it EXACTLY.
-    GGML_ASSERT(ext_factor >= 0.0f);
-    const float attn_factor_org = attn_factor * (1.0f + 0.1f * logf(1.0f / freq_scale));
-
-    const float mscale   = attn_factor_org * (1.0f + 0.1f * hparams.rope_yarn_log_mul * logf(1.0f / freq_scale));
-    const float kq_scale = 1.0f * mscale * mscale / sqrtf(float(n_embd_head_k));
-
-    // TODO: extract in a common llm_graph_context::build_inp_embd_h()
-    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd);
-
-    inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
-    ggml_set_input(inp->tokens);
-
-    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens);
-    ggml_set_input(inp->embd);
-
-    ggml_tensor * tok_embd;
-    if (ubatch.token) {
-        ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;
-
-        tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens);
-    } else {
-        tok_embd = inp->embd;
-    }
-    cb(tok_embd, "mtp_tok_embd", il);
-
-    inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);
-    ggml_set_input(inp->h);
-    ggml_set_name(inp->h, "mtp_h_input");
-
-    ggml_tensor * h_embd = inp->h;
-
-    res->add_input(std::move(inp));
-
-    ggml_tensor * inp_pos     = build_inp_pos();
-    ggml_tensor * inp_out_ids = build_inp_out_ids();
-
-    // MLA with the absorption optimization uses a K-only cache (V is a view of K)
-    auto * inp_attn = build_attn_inp_k();
-
-    ggml_tensor * h_norm = build_norm(h_embd, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il);
-    cb(h_norm, "mtp_hnorm", il);
-
-    ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il);
-    cb(e_norm, "mtp_enorm", il);
-
-    ggml_tensor * concat = ggml_concat(ctx0, e_norm, h_norm, /*dim=*/ 0);
-    cb(concat, "mtp_concat", il);
-
-    ggml_tensor * cur = build_lora_mm(layer.nextn.eh_proj, concat, layer.nextn.eh_proj_s);
-    cb(cur, "mtp_eh_proj", il);
-
-    ggml_tensor * inpSA = cur;
-
-    cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
-    cb(cur, "mtp_attn_norm", il);
-
-    // self-attention: dense MLA, same construction as the deepseek2 trunk graph
-    {
-        ggml_tensor * q = ggml_mul_mat(ctx0, layer.wq_a, cur);
-        cb(q, "mtp_q", il);
-
-        q = build_norm(q, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il);
-        cb(q, "mtp_q", il);
-
-        q = ggml_mul_mat(ctx0, layer.wq_b, q);
-        cb(q, "mtp_q", il);
-
-        // split into {n_embd_head_qk_nope, n_head, n_tokens}
-        ggml_tensor * q_nope =
-            ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k),
-                         ggml_row_size(q->type, n_embd_head_k) * n_head, 0);
-        cb(q_nope, "mtp_q_nope", il);
-
-        // and {n_embd_head_qk_rope, n_head, n_tokens}
-        ggml_tensor * q_pe = ggml_view_3d(
-            ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k),
-            ggml_row_size(q->type, n_embd_head_k) * n_head, ggml_row_size(q->type, n_embd_head_qk_nope));
-        cb(q_pe, "mtp_q_pe", il);
-
-        ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, layer.wkv_a_mqa, cur);
-        cb(kv_cmpr_pe, "mtp_kv_cmpr_pe", il);
-
-        // split into {kv_lora_rank, n_tokens}
-        ggml_tensor * kv_cmpr =
-            ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens,
-                         ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), 0);
-        cb(kv_cmpr, "mtp_kv_cmpr", il);
-
-        // and {n_embd_head_qk_rope, 1, n_tokens}
-        ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_cmpr_pe, n_embd_head_qk_rope, 1, n_tokens,
-                                          ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),
-                                          ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),
-                                          ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));
-        cb(k_pe, "mtp_k_pe", il);
-
-        q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
-                             ext_factor, attn_factor, beta_fast, beta_slow);
-        cb(q_pe, "mtp_q_pe", il);
-
-        k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
-                             ext_factor, attn_factor, beta_fast, beta_slow);
-        cb(k_pe, "mtp_k_pe", il);
-
-        kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);
-        cb(kv_cmpr, "mtp_kv_cmpr", il);
-
-        // {n_embd_head_qk_nope, n_tokens, n_head}
-        q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3);
-        cb(q_nope, "mtp_q_nope_perm", il);
-
-        // {n_embd_head_qk_nope, kv_lora_rank, n_head} x {n_embd_head_qk_nope, n_tokens, n_head}
-        ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, layer.wk_b, q_nope);
-        cb(q_nope_absorbed, "mtp_q_nope_absorbed", il);
-
-        // {kv_lora_rank, n_head, n_tokens}
-        q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3);
-        cb(q_nope_absorbed, "mtp_q_nope_absorbed_perm", il);
-
-        // {n_embd_head_qk_rope + kv_lora_rank, n_head, n_tokens}
-        // note: rope must go first for in-place context shifting in build_rope_shift()
-        ggml_tensor * Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0);
-        cb(Qcur, "mtp_Qcur", il);
-
-        kv_cmpr = ggml_reshape_3d(ctx0, kv_cmpr, kv_lora_rank, 1, n_tokens);
-        cb(kv_cmpr, "mtp_kv_cmpr_reshape", il);
-
-        // {n_embd_head_qk_rope + kv_lora_rank, 1, n_tokens}
-        ggml_tensor * Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0);
-        cb(Kcur, "mtp_Kcur", il);
-
-        // {kv_lora_rank, 1, n_tokens}
-        ggml_tensor * Vcur = kv_cmpr;
-        cb(Vcur, "mtp_Vcur", il);
-
-        // note: MLA with the absorption optimization converts into MQA (ie: GQA with 1 group)
-        cur = build_attn(inp_attn,
-                layer.wo, NULL, layer.wo_s,
-                Qcur, Kcur, Vcur, nullptr, nullptr, layer.wv_b, kq_scale, il);
-        cb(cur, "mtp_attn_out", il);
-    }
-
-    ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
-    cb(ffn_inp, "mtp_ffn_inp", il);
-
-    cur = build_norm(ffn_inp, layer.ffn_norm, NULL, LLM_NORM_RMS, il);
-    cb(cur, "mtp_ffn_norm", il);
-
-    // MoE FFN with shared expert - same construction as the deepseek2 trunk graph
-    ggml_tensor * moe_out = build_moe_ffn(cur,
-        layer.ffn_gate_inp,
-        layer.ffn_up_exps,
-        layer.ffn_gate_exps,
-        layer.ffn_down_exps,
-        layer.ffn_exp_probs_b,
-        n_expert, n_expert_used,
-        LLM_FFN_SILU, hparams.expert_weights_norm,
-        hparams.expert_weights_scale,
-        (llama_expert_gating_func_type) hparams.expert_gating_func,
-        il,
-        nullptr,
-        layer.ffn_gate_up_exps,
-        layer.ffn_up_exps_s,
-        layer.ffn_gate_exps_s,
-        layer.ffn_down_exps_s);
-    cb(moe_out, "mtp_ffn_moe_out", il);
-
-    // FFN shared expert
-    ggml_tensor * ffn_shexp =
-        build_ffn(cur,
-            layer.ffn_up_shexp, NULL, layer.ffn_up_shexp_s,
-            layer.ffn_gate_shexp, NULL, layer.ffn_gate_shexp_s,
-            layer.ffn_down_shexp, NULL, layer.ffn_down_shexp_s,
-            NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);
-    cb(ffn_shexp, "mtp_ffn_shexp", il);
-
-    cur = ggml_add(ctx0, moe_out, ffn_shexp);
-    cb(cur, "mtp_ffn_out", il);
-
-    cur = ggml_add(ctx0, cur, ffn_inp);
-    cb(cur, "mtp_post_ffn", il);
-
-    // shared_head_norm applied after the decoder block, before the shared LM head.
-    // The post-norm hidden state seeds the next MTP step.
-    ggml_tensor * head_norm_w = layer.nextn.shared_head_norm
-            ? layer.nextn.shared_head_norm
-            : model.output_norm;
-    GGML_ASSERT(head_norm_w && "DEEPSEEK32 MTP: missing both nextn.shared_head_norm and output_norm");
-    cur = build_norm(cur, head_norm_w, nullptr, LLM_NORM_RMS, -1);
-
-    cb(cur, "h_nextn", -1);
-    res->t_h_nextn = cur;
-
-    cur = ggml_get_rows(ctx0, cur, inp_out_ids);
-    cb(cur, "mtp_shared_head_norm", -1);
-
-    ggml_tensor * head_w = layer.nextn.shared_head_head ? layer.nextn.shared_head_head : model.output;
-    ggml_tensor * head_s = layer.nextn.shared_head_head ? layer.nextn.shared_head_head_s : model.output_s;
-    GGML_ASSERT(head_w && "DEEPSEEK32 MTP: missing LM head (nextn.shared_head_head or model.output)");
-    cur = build_lora_mm(head_w, cur, head_s);
-    cb(cur, "result_output", -1);
-
-    res->t_logits = cur;
-    ggml_build_forward_expand(gf, cur);
-}
-

src/models/deepseek4.cpp

@@ -16,16 +16,6 @@ static float dsv4_rope_attn_factor(float freq_scale, float ext_factor) {
 }
 
 void llama_model_deepseek4::load_arch_hparams(llama_model_loader & ml) {
-    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
-    if (hparams.n_layer_nextn > 0 && hparams.n_layer_nextn < hparams.n_layer_all) {
-        const uint32_t n_layer_main = hparams.n_layer_all - hparams.n_layer_nextn;
-        const std::string mtp_probe = "blk." + std::to_string(n_layer_main) + ".nextn.eh_proj.weight";
-        if (ml.get_weight(mtp_probe.c_str()) == nullptr) {
-            hparams.n_layer_nextn = 0;
-        }
-    }
-    GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < block_count");
-
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
     ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK,       hparams.n_lora_q);
     ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW,    hparams.n_swa);
@@ -34,8 +24,8 @@ void llama_model_deepseek4::load_arch_hparams(llama_model_loader & ml) {
     ml.get_key(LLM_KV_EXPERT_SHARED_COUNT,         hparams.n_expert_shared);
     ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,        hparams.expert_weights_scale);
     ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,         hparams.expert_weights_norm);
-    ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP,     hparams.swiglu_clamp_exp,   hparams.n_layer_all);
-    if (!ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP,   hparams.swiglu_clamp_shexp, hparams.n_layer_all, 0)) {
+    ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP,     hparams.swiglu_clamp_exp,   hparams.n_layer());
+    if (!ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP,   hparams.swiglu_clamp_shexp, hparams.n_layer(), 0)) {
         hparams.swiglu_clamp_shexp = hparams.swiglu_clamp_exp;
     }
 
@@ -51,11 +41,9 @@ void llama_model_deepseek4::load_arch_hparams(llama_model_loader & ml) {
     ml.get_key(LLM_KV_HYPER_CONNECTION_EPSILON,             hparams.dsv4_hc_eps);
     ml.get_key(LLM_KV_HASH_LAYER_COUNT,                     hparams.dsv4_hash_layer_count);
 
-    hparams.n_embd_out_impl = hparams.dsv4_hc_mult * hparams.n_embd;
-
     uint32_t n_compress_ratios = 0;
     ml.get_arr_n(LLM_KV_ATTENTION_COMPRESS_RATIOS, n_compress_ratios);
-    if (n_compress_ratios < hparams.n_layer_all) {
+    if (n_compress_ratios < hparams.n_layer()) {
         throw std::runtime_error("DeepSeek-V4 compress_ratios is shorter than block_count");
     }
     ml.get_arr(LLM_KV_ATTENTION_COMPRESS_RATIOS, hparams.dsv4_compress_ratios);
@@ -66,9 +54,6 @@ void llama_model_deepseek4::load_arch_hparams(llama_model_loader & ml) {
     }
     hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
     hparams.set_swa_pattern(0);
-    for (uint32_t il = hparams.n_layer(); il < hparams.n_layer_all; ++il) {
-        hparams.is_swa_impl[il] = true;
-    }
 
     switch (hparams.n_layer()) {
         case 43: type = LLM_TYPE_UNKNOWN; break;
@@ -76,7 +61,7 @@ void llama_model_deepseek4::load_arch_hparams(llama_model_loader & ml) {
     }
 }
 
-void llama_model_deepseek4::load_arch_tensors(llama_model_loader & ml) {
+void llama_model_deepseek4::load_arch_tensors(llama_model_loader &) {
     LLAMA_LOAD_LOCALS;
 
     const int64_t q_lora_rank     = hparams.n_lora_q;
@@ -90,10 +75,6 @@ void llama_model_deepseek4::load_arch_tensors(llama_model_loader & ml) {
     const int64_t hc_dim      = hc_mult * n_embd;
     const int64_t hc_mix_dim  = (2 + hc_mult) * hc_mult;
 
-    const bool mtp_only = (n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
-    const int trunk_flags = mtp_only    ? TENSOR_NOT_REQUIRED : 0;
-    const int mtp_flags   = ml.load_mtp ? 0 : TENSOR_SKIP;
-
     tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
 
     output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
@@ -103,84 +84,69 @@ void llama_model_deepseek4::load_arch_tensors(llama_model_loader & ml) {
     hc_head_base  = create_tensor(tn(LLM_TENSOR_HC_HEAD_BASE, "weight"),  {hc_mult}, 0);
     hc_head_scale = create_tensor(tn(LLM_TENSOR_HC_HEAD_SCALE, "weight"), {1}, 0);
 
-    for (int i = 0; i < n_layer_all; ++i) {
+    for (int i = 0; i < n_layer; ++i) {
         auto & layer = layers[i];
-        const int flags = i < n_layer ? trunk_flags : mtp_flags;
 
-        layer.attn_norm     = create_tensor(tn(LLM_TENSOR_ATTN_NORM,     "weight", i), {n_embd}, flags);
-        layer.attn_sinks    = create_tensor(tn(LLM_TENSOR_ATTN_SINKS,    "weight", i), {n_head}, flags);
-        layer.wq_a          = create_tensor(tn(LLM_TENSOR_ATTN_Q_A,      "weight", i), {n_embd, q_lora_rank}, flags);
-        layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, flags);
-        layer.wq_b          = create_tensor(tn(LLM_TENSOR_ATTN_Q_B,      "weight", i), {q_lora_rank, n_head * n_embd_head}, flags);
-        layer.wkv           = create_tensor(tn(LLM_TENSOR_ATTN_KV,       "weight", i), {n_embd, n_embd_head}, flags);
-        layer.attn_kv_norm  = create_tensor(tn(LLM_TENSOR_ATTN_KV_NORM,  "weight", i), {n_embd_head}, flags);
-        // for wo_a, the shape in the file is (n_head * n_embd_head / o_groups, o_lora_rank*o_groups)
-        // so we reshape here, to avoid reshaping the tensor in the graph
-        layer.wo_a          = create_tensor(tn(LLM_TENSOR_ATTN_OUT_A,    "weight", i), {n_head * n_embd_head / o_groups, o_lora_rank, o_groups}, flags | TENSOR_ALLOW_RESHAPE);
-        layer.wo_b          = create_tensor(tn(LLM_TENSOR_ATTN_OUT_B,    "weight", i), {o_groups * o_lora_rank, n_embd}, flags);
+        layer.attn_norm     = create_tensor(tn(LLM_TENSOR_ATTN_NORM,     "weight", i), {n_embd}, 0);
+        layer.attn_sinks    = create_tensor(tn(LLM_TENSOR_ATTN_SINKS,    "weight", i), {n_head}, 0);
+        layer.wq_a          = create_tensor(tn(LLM_TENSOR_ATTN_Q_A,      "weight", i), {n_embd, q_lora_rank}, 0);
+        layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, 0);
+        layer.wq_b          = create_tensor(tn(LLM_TENSOR_ATTN_Q_B,      "weight", i), {q_lora_rank, n_head * n_embd_head}, 0);
+        layer.wkv           = create_tensor(tn(LLM_TENSOR_ATTN_KV,       "weight", i), {n_embd, n_embd_head}, 0);
+        layer.attn_kv_norm  = create_tensor(tn(LLM_TENSOR_ATTN_KV_NORM,  "weight", i), {n_embd_head}, 0);
+        layer.wo_a          = create_tensor(tn(LLM_TENSOR_ATTN_OUT_A,    "weight", i), {n_head * n_embd_head / o_groups, o_lora_rank * o_groups}, 0);
+        layer.wo_b          = create_tensor(tn(LLM_TENSOR_ATTN_OUT_B,    "weight", i), {o_groups * o_lora_rank, n_embd}, 0);
 
-        layer.hc_attn_fn    = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN,    "weight", i), {hc_dim, hc_mix_dim}, flags);
-        layer.hc_attn_base  = create_tensor(tn(LLM_TENSOR_HC_ATTN_BASE,  "weight", i), {hc_mix_dim}, flags);
-        layer.hc_attn_scale = create_tensor(tn(LLM_TENSOR_HC_ATTN_SCALE, "weight", i), {3}, flags);
-        layer.hc_ffn_fn     = create_tensor(tn(LLM_TENSOR_HC_FFN_FN,     "weight", i), {hc_dim, hc_mix_dim}, flags);
-        layer.hc_ffn_base   = create_tensor(tn(LLM_TENSOR_HC_FFN_BASE,   "weight", i), {hc_mix_dim}, flags);
-        layer.hc_ffn_scale  = create_tensor(tn(LLM_TENSOR_HC_FFN_SCALE,  "weight", i), {3}, flags);
+        layer.hc_attn_fn    = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN,    "weight", i), {hc_dim, hc_mix_dim}, 0);
+        layer.hc_attn_base  = create_tensor(tn(LLM_TENSOR_HC_ATTN_BASE,  "weight", i), {hc_mix_dim}, 0);
+        layer.hc_attn_scale = create_tensor(tn(LLM_TENSOR_HC_ATTN_SCALE, "weight", i), {3}, 0);
+        layer.hc_ffn_fn     = create_tensor(tn(LLM_TENSOR_HC_FFN_FN,     "weight", i), {hc_dim, hc_mix_dim}, 0);
+        layer.hc_ffn_base   = create_tensor(tn(LLM_TENSOR_HC_FFN_BASE,   "weight", i), {hc_mix_dim}, 0);
+        layer.hc_ffn_scale  = create_tensor(tn(LLM_TENSOR_HC_FFN_SCALE,  "weight", i), {3}, 0);
 
         const int64_t ratio = hparams.dsv4_compress_ratios[i];
         if (ratio != 0) {
             const int64_t coff = ratio == 4 ? 2 : 1;
 
-            layer.attn_comp_wkv   = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_WKV,   "weight", i), {n_embd, coff * n_embd_head}, flags);
-            layer.attn_comp_wgate = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_WGATE, "weight", i), {n_embd, coff * n_embd_head}, flags);
-            layer.attn_comp_ape   = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_APE,   "weight", i), {coff * n_embd_head, ratio}, flags);
-            layer.attn_comp_norm  = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_NORM,  "weight", i), {n_embd_head}, flags);
+            layer.attn_comp_wkv   = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_WKV,   "weight", i), {n_embd, coff * n_embd_head}, 0);
+            layer.attn_comp_wgate = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_WGATE, "weight", i), {n_embd, coff * n_embd_head}, 0);
+            layer.attn_comp_ape   = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_APE,   "weight", i), {coff * n_embd_head, ratio}, 0);
+            layer.attn_comp_norm  = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_NORM,  "weight", i), {n_embd_head}, 0);
 
             if (ratio == 4) {
                 const int64_t n_embd_indexer = hparams.indexer_head_size;
 
-                layer.indexer_proj     = create_tensor(tn(LLM_TENSOR_INDEXER_PROJ,     "weight", i), {n_embd, hparams.indexer_n_head}, flags);
-                layer.indexer_attn_q_b = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_Q_B, "weight", i), {q_lora_rank, hparams.indexer_n_head * n_embd_indexer}, flags);
+                layer.indexer_proj     = create_tensor(tn(LLM_TENSOR_INDEXER_PROJ,     "weight", i), {n_embd, hparams.indexer_n_head}, 0);
+                layer.indexer_attn_q_b = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_Q_B, "weight", i), {q_lora_rank, hparams.indexer_n_head * n_embd_indexer}, 0);
 
-                layer.indexer_comp_wkv   = create_tensor(tn(LLM_TENSOR_INDEXER_COMPRESSOR_WKV,   "weight", i), {n_embd, 2 * n_embd_indexer}, flags);
-                layer.indexer_comp_wgate = create_tensor(tn(LLM_TENSOR_INDEXER_COMPRESSOR_WGATE, "weight", i), {n_embd, 2 * n_embd_indexer}, flags);
-                layer.indexer_comp_ape   = create_tensor(tn(LLM_TENSOR_INDEXER_COMPRESSOR_APE,   "weight", i), {2 * n_embd_indexer, ratio}, flags);
-                layer.indexer_comp_norm  = create_tensor(tn(LLM_TENSOR_INDEXER_COMPRESSOR_NORM,  "weight", i), {n_embd_indexer}, flags);
+                layer.indexer_comp_wkv   = create_tensor(tn(LLM_TENSOR_INDEXER_COMPRESSOR_WKV,   "weight", i), {n_embd, 2 * n_embd_indexer}, 0);
+                layer.indexer_comp_wgate = create_tensor(tn(LLM_TENSOR_INDEXER_COMPRESSOR_WGATE, "weight", i), {n_embd, 2 * n_embd_indexer}, 0);
+                layer.indexer_comp_ape   = create_tensor(tn(LLM_TENSOR_INDEXER_COMPRESSOR_APE,   "weight", i), {2 * n_embd_indexer, ratio}, 0);
+                layer.indexer_comp_norm  = create_tensor(tn(LLM_TENSOR_INDEXER_COMPRESSOR_NORM,  "weight", i), {n_embd_indexer}, 0);
             } else if (ratio != 128) {
                 throw std::runtime_error("DeepSeek-V4 loader only supports compression ratios 0, 4, and 128");
             }
         }
 
-        layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, flags);
+        layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
         if ((uint32_t) i < hparams.dsv4_hash_layer_count) {
-            layer.ffn_gate_tid2eid = create_tensor(tn(LLM_TENSOR_FFN_GATE_TID2EID, "weight", i), {n_expert_used, n_vocab}, flags);
+            layer.ffn_gate_tid2eid = create_tensor(tn(LLM_TENSOR_FFN_GATE_TID2EID, "weight", i), {n_expert_used, n_vocab}, 0);
         } else {
-            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, flags);
+            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0);
         }
-        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, flags);
+        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
 
-        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd,   n_ff_exp, n_expert}, flags);
-        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd,   n_expert}, flags);
-        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   "weight", i), {n_embd,   n_ff_exp, n_expert}, flags);
+        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd,   n_ff_exp, n_expert}, 0);
+        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd,   n_expert}, 0);
+        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   "weight", i), {n_embd,   n_ff_exp, n_expert}, 0);
 
-        layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd,                     n_ff_exp * n_expert_shared}, flags);
-        layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_exp * n_expert_shared, n_embd                    }, flags);
-        layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   "weight", i), {n_embd,                     n_ff_exp * n_expert_shared}, flags);
-
-        if (i >= n_layer) {
-            layer.nextn.eh_proj          = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ,          "weight", i), {2 * n_embd, n_embd}, flags);
-            layer.nextn.enorm            = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM,            "weight", i), {n_embd},             flags);
-            layer.nextn.hnorm            = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM,            "weight", i), {n_embd},             flags);
-            layer.nextn.embed_tokens     = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS,     "weight", i), {n_embd, n_vocab},    TENSOR_NOT_REQUIRED | flags);
-            layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), {n_embd, n_vocab},    TENSOR_NOT_REQUIRED | flags);
-            layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), {n_embd},             TENSOR_NOT_REQUIRED | flags);
-        }
+        layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd,                     n_ff_exp * n_expert_shared}, 0);
+        layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_exp * n_expert_shared, n_embd                    }, 0);
+        layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   "weight", i), {n_embd,                     n_ff_exp * n_expert_shared}, 0);
     }
 }
 
 std::unique_ptr<llm_graph_context> llama_model_deepseek4::build_arch_graph(const llm_graph_params & params) const {
-    if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) {
-        return std::make_unique<graph_mtp>(*this, params);
-    }
     return std::make_unique<graph>(*this, params);
 }
 
@@ -209,69 +175,18 @@ static ggml_tensor * dsv4_append_zero_row(ggml_context * ctx, ggml_tensor * t, b
     return ggml_concat(ctx, t, row, 1);
 }
 
-struct dsv4_state_tensors {
-    ggml_tensor * kv;
-    ggml_tensor * score;
-};
-
-static dsv4_state_tensors dsv4_build_state_restore(
-        ggml_context * ctx,
-        const llm_graph_input_dsv4::comp_input & inp,
-        const llama_dsv4_comp_state * state,
-        int32_t il) {
-    dsv4_state_tensors restored = {
-        state->get_kv_all(ctx, il),
-        state->get_score_all(ctx, il),
-    };
-
-    if (inp.state_restore_src_idxs == nullptr || inp.state_restore_dst_idxs == nullptr) {
-        return restored;
+static ggml_tensor * dsv4_with_zero_dep(ggml_context * ctx, ggml_tensor * t, ggml_tensor * dep) {
+    if (dep == nullptr) {
+        return t;
     }
 
-    ggml_tensor * kv_rows = ggml_get_rows(ctx, restored.kv, inp.state_restore_src_idxs);
-    restored.kv = state->cpy_kv(ctx, kv_rows, inp.state_restore_dst_idxs, il);
-
-    ggml_tensor * score_rows = ggml_get_rows(ctx, restored.score, inp.state_restore_src_idxs);
-    restored.score = state->cpy_score(ctx, score_rows, inp.state_restore_dst_idxs, il);
-
-    return restored;
-}
-
-static dsv4_state_tensors dsv4_build_state_snapshot(
-        ggml_context * ctx,
-        const llm_graph_input_dsv4::comp_input & inp,
-        const llama_dsv4_comp_state * state,
-        ggml_tensor * source_kv,
-        ggml_tensor * source_score,
-        int32_t il) {
-    if (inp.state_snapshot_src_idxs == nullptr || inp.state_snapshot_dst_idxs == nullptr ||
-            source_kv == nullptr || source_score == nullptr) {
-        return {};
-    }
-
-    ggml_tensor * kv_rows = ggml_get_rows(ctx, source_kv, inp.state_snapshot_src_idxs);
-    ggml_tensor * kv = state->cpy_kv(ctx, kv_rows, inp.state_snapshot_dst_idxs, il);
-
-    ggml_tensor * score_rows = ggml_get_rows(ctx, source_score, inp.state_snapshot_src_idxs);
-    ggml_tensor * score = state->cpy_score(ctx, score_rows, inp.state_snapshot_dst_idxs, il);
-
-    return { kv, score };
+    ggml_tensor * zero = ggml_scale(ctx, ggml_sum(ctx, dep), 0.0f);
+    return ggml_add(ctx, t, zero);
 }
 
 static constexpr int64_t DSV4_CSA_RATIO  = 4;
 static constexpr int64_t DSV4_HCA_RATIO  = 128;
 
-// mean over the hyper-connection streams: [n_embd, hc, n_tokens] -> [n_embd, n_tokens]
-static ggml_tensor * dsv4_hc_mean(ggml_context * ctx, ggml_tensor * x) {
-    const int64_t hc = x->ne[1];
-
-    ggml_tensor * acc = ggml_view_2d(ctx, x, x->ne[0], x->ne[2], x->nb[2], 0);
-    for (int64_t s = 1; s < hc; ++s) {
-        acc = ggml_add(ctx, acc, ggml_view_2d(ctx, x, x->ne[0], x->ne[2], x->nb[2], s*x->nb[1]));
-    }
-    return ggml_scale(ctx, acc, 1.0f/hc);
-}
-
 static ggml_tensor * dsv4_hc_affine(
         ggml_context * ctx,
         ggml_tensor  * x,
@@ -889,29 +804,8 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention(
         ggml_tensor * cur,
         ggml_tensor * inp_pos,
         int il) const {
-    return build_attention_impl(model, inp_dsv4, nullptr, cur, inp_pos, il);
-}
-
-ggml_tensor * llama_model_deepseek4::graph::build_attention(
-        const llama_model & model,
-        llm_graph_input_attn_k_iswa * inp_mtp,
-        ggml_tensor * cur,
-        ggml_tensor * inp_pos,
-        int il) const {
-    return build_attention_impl(model, nullptr, inp_mtp, cur, inp_pos, il);
-}
-
-ggml_tensor * llama_model_deepseek4::graph::build_attention_impl(
-        const llama_model & model,
-        llm_graph_input_dsv4 * inp_dsv4,
-        llm_graph_input_attn_k_iswa * inp_mtp,
-        ggml_tensor * cur,
-        ggml_tensor * inp_pos,
-        int il) const {
-    GGML_ASSERT((inp_dsv4 == nullptr) != (inp_mtp == nullptr));
-
     const auto & layer = model.layers[il];
-    llm_graph_input_dsv4_raw * inp_attn = inp_dsv4 ? inp_dsv4->get_raw() : nullptr;
+    llm_graph_input_dsv4_raw * inp_attn = inp_dsv4->get_raw();
 
     const int64_t n_embd_head      = hparams.n_embd_head_k();
     const int64_t n_embd_head_rope = hparams.n_rot();
@@ -979,12 +873,9 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl(
     cb(kv, "kv", il);
 
     const int64_t ratio = hparams.dsv4_compress_ratios[il];
-    GGML_ASSERT(inp_dsv4 || ratio == 0);
 
     ggml_tensor * hca_state_kv    = nullptr;
     ggml_tensor * hca_state_score = nullptr;
-    ggml_tensor * hca_source_kv   = nullptr;
-    ggml_tensor * hca_source_score = nullptr;
     if (ratio == DSV4_HCA_RATIO && inp_dsv4->get_hca().state_pos) {
         hca_state_kv = build_lora_mm(layer.attn_comp_wkv, cur);
         cb(hca_state_kv, "hca_state_kv", il);
@@ -1015,16 +906,10 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl(
 
         GGML_ASSERT(inp_dsv4->get_csa().state_write_idxs);
 
-        const auto * csa_state = inp_dsv4->mctx->get_csa_state();
-        const dsv4_state_tensors csa_restored = dsv4_build_state_restore(
-                ctx0, inp_dsv4->get_csa(), csa_state, il);
-        ggml_tensor * csa_base_kv = dsv4_view_2d(
-                ctx0, csa_restored.kv, csa_restored.kv->ne[0], csa_state->get_n_rows(), 0);
-        ggml_tensor * csa_base_score = dsv4_view_2d(
-                ctx0, csa_restored.score, csa_restored.score->ne[0], csa_state->get_n_rows(), 0);
-
-        ggml_tensor * csa_source_kv = ggml_concat(ctx0, csa_base_kv, csa_state_kv, 1);
-        ggml_tensor * csa_source_score = ggml_concat(ctx0, csa_base_score, csa_state_score, 1);
+        ggml_tensor * csa_source_kv = ggml_concat(ctx0,
+                inp_dsv4->mctx->get_csa_state()->get_kv(ctx0, il), csa_state_kv, 1);
+        ggml_tensor * csa_source_score = ggml_concat(ctx0,
+                inp_dsv4->mctx->get_csa_state()->get_score(ctx0, il), csa_state_score, 1);
 
         ggml_tensor * kv_comp_csa_state = build_overlap_compressed_kv_from_state(
                 csa_source_kv,
@@ -1045,19 +930,8 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl(
         ggml_build_forward_expand(gf, inp_dsv4->mctx->get_csa()->cpy_k(ctx0,
                     kv_comp_csa_state, inp_dsv4->get_csa().state_write_idxs, il));
 
-        ggml_tensor * csa_snapshot_source_kv = ggml_concat(ctx0,
-                csa_restored.kv, csa_state_kv, 1);
-        ggml_tensor * csa_snapshot_source_score = ggml_concat(ctx0,
-                csa_restored.score, csa_state_score, 1);
-
-        const dsv4_state_tensors csa_snapshot = dsv4_build_state_snapshot(
-                ctx0, inp_dsv4->get_csa(), csa_state, csa_snapshot_source_kv, csa_snapshot_source_score, il);
-        if (csa_snapshot.kv != nullptr) {
-            ggml_build_forward_expand(gf, csa_snapshot.kv);
-        }
-        if (csa_snapshot.score != nullptr) {
-            ggml_build_forward_expand(gf, csa_snapshot.score);
-        }
+        csa_state_kv    = dsv4_with_zero_dep(ctx0, csa_state_kv,    kv_comp_csa_state);
+        csa_state_score = dsv4_with_zero_dep(ctx0, csa_state_score, kv_comp_csa_state);
 
         ggml_tensor * csa_persist_kv = ggml_get_rows(ctx0, csa_state_kv, inp_dsv4->get_csa().state_persist_src_idxs);
         ggml_tensor * csa_persist_score = ggml_get_rows(ctx0, csa_state_score, inp_dsv4->get_csa().state_persist_src_idxs);
@@ -1084,16 +958,10 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl(
 
         GGML_ASSERT(inp_dsv4->get_lid().state_write_idxs);
 
-        const auto * lid_state = inp_dsv4->mctx->get_lid_state();
-        const dsv4_state_tensors lid_restored = dsv4_build_state_restore(
-                ctx0, inp_dsv4->get_lid(), lid_state, il);
-        ggml_tensor * lid_base_kv = dsv4_view_2d(
-                ctx0, lid_restored.kv, lid_restored.kv->ne[0], lid_state->get_n_rows(), 0);
-        ggml_tensor * lid_base_score = dsv4_view_2d(
-                ctx0, lid_restored.score, lid_restored.score->ne[0], lid_state->get_n_rows(), 0);
-
-        ggml_tensor * lid_source_kv = ggml_concat(ctx0, lid_base_kv, lid_state_kv, 1);
-        ggml_tensor * lid_source_score = ggml_concat(ctx0, lid_base_score, lid_state_score, 1);
+        ggml_tensor * lid_source_kv = ggml_concat(ctx0,
+                inp_dsv4->mctx->get_lid_state()->get_kv(ctx0, il), lid_state_kv, 1);
+        ggml_tensor * lid_source_score = ggml_concat(ctx0,
+                inp_dsv4->mctx->get_lid_state()->get_score(ctx0, il), lid_state_score, 1);
 
         ggml_tensor * kv_comp_lid_state = build_overlap_compressed_kv_from_state(
                 lid_source_kv,
@@ -1114,19 +982,8 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl(
         ggml_build_forward_expand(gf, inp_dsv4->mctx->get_lid()->cpy_k(ctx0,
                     kv_comp_lid_state, inp_dsv4->get_lid().state_write_idxs, il));
 
-        ggml_tensor * lid_snapshot_source_kv = ggml_concat(ctx0,
-                lid_restored.kv, lid_state_kv, 1);
-        ggml_tensor * lid_snapshot_source_score = ggml_concat(ctx0,
-                lid_restored.score, lid_state_score, 1);
-
-        const dsv4_state_tensors lid_snapshot = dsv4_build_state_snapshot(
-                ctx0, inp_dsv4->get_lid(), lid_state, lid_snapshot_source_kv, lid_snapshot_source_score, il);
-        if (lid_snapshot.kv != nullptr) {
-            ggml_build_forward_expand(gf, lid_snapshot.kv);
-        }
-        if (lid_snapshot.score != nullptr) {
-            ggml_build_forward_expand(gf, lid_snapshot.score);
-        }
+        lid_state_kv    = dsv4_with_zero_dep(ctx0, lid_state_kv,    kv_comp_lid_state);
+        lid_state_score = dsv4_with_zero_dep(ctx0, lid_state_score, kv_comp_lid_state);
 
         ggml_tensor * lid_persist_kv = ggml_get_rows(ctx0, lid_state_kv, inp_dsv4->get_lid().state_persist_src_idxs);
         ggml_tensor * lid_persist_score = ggml_get_rows(ctx0, lid_state_score, inp_dsv4->get_lid().state_persist_src_idxs);
@@ -1140,21 +997,15 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl(
         ggml_build_forward_expand(gf, lid_state_score);
     }
 
-    const llama_dsv4_comp_state * hca_state = nullptr;
-    dsv4_state_tensors hca_restored = {};
+    ggml_tensor * hca_state_dep = nullptr;
     if (ratio == DSV4_HCA_RATIO && inp_dsv4->get_hca().state_write_idxs) {
         GGML_ASSERT(hca_state_kv);
         GGML_ASSERT(hca_state_score);
 
-        hca_state = inp_dsv4->mctx->get_hca_state();
-        hca_restored = dsv4_build_state_restore(ctx0, inp_dsv4->get_hca(), hca_state, il);
-        ggml_tensor * hca_base_kv = dsv4_view_2d(
-                ctx0, hca_restored.kv, hca_restored.kv->ne[0], hca_state->get_n_rows(), 0);
-        ggml_tensor * hca_base_score = dsv4_view_2d(
-                ctx0, hca_restored.score, hca_restored.score->ne[0], hca_state->get_n_rows(), 0);
-
-        hca_source_kv = ggml_concat(ctx0, hca_base_kv, hca_state_kv, 1);
-        hca_source_score = ggml_concat(ctx0, hca_base_score, hca_state_score, 1);
+        ggml_tensor * hca_source_kv = ggml_concat(ctx0,
+                inp_dsv4->mctx->get_hca_state()->get_kv(ctx0, il), hca_state_kv, 1);
+        ggml_tensor * hca_source_score = ggml_concat(ctx0,
+                inp_dsv4->mctx->get_hca_state()->get_score(ctx0, il), hca_state_score, 1);
 
         ggml_tensor * kv_comp_hca = build_hca_compressed_kv_from_state(
                 hca_source_kv,
@@ -1173,41 +1024,15 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl(
 
         ggml_build_forward_expand(gf, inp_dsv4->mctx->get_hca()->cpy_k(ctx0,
                     kv_comp_hca, inp_dsv4->get_hca().state_write_idxs, il));
+        hca_state_dep = kv_comp_hca;
     }
 
     if (ratio == DSV4_HCA_RATIO && inp_dsv4->get_hca().state_pos) {
         GGML_ASSERT(hca_state_kv);
         GGML_ASSERT(hca_state_score);
 
-        if (hca_state == nullptr) {
-            hca_state = inp_dsv4->mctx->get_hca_state();
-        }
-        if (hca_restored.kv == nullptr) {
-            hca_restored = dsv4_build_state_restore(ctx0, inp_dsv4->get_hca(), hca_state, il);
-        }
-        if (hca_source_kv == nullptr || hca_source_score == nullptr) {
-            ggml_tensor * hca_base_kv = dsv4_view_2d(
-                    ctx0, hca_restored.kv, hca_restored.kv->ne[0], hca_state->get_n_rows(), 0);
-            ggml_tensor * hca_base_score = dsv4_view_2d(
-                    ctx0, hca_restored.score, hca_restored.score->ne[0], hca_state->get_n_rows(), 0);
-
-            hca_source_kv = ggml_concat(ctx0, hca_base_kv, hca_state_kv, 1);
-            hca_source_score = ggml_concat(ctx0, hca_base_score, hca_state_score, 1);
-        }
-
-        ggml_tensor * hca_snapshot_source_kv = ggml_concat(ctx0,
-                hca_restored.kv, hca_state_kv, 1);
-        ggml_tensor * hca_snapshot_source_score = ggml_concat(ctx0,
-                hca_restored.score, hca_state_score, 1);
-
-        const dsv4_state_tensors hca_snapshot = dsv4_build_state_snapshot(
-                ctx0, inp_dsv4->get_hca(), hca_state, hca_snapshot_source_kv, hca_snapshot_source_score, il);
-        if (hca_snapshot.kv != nullptr) {
-            ggml_build_forward_expand(gf, hca_snapshot.kv);
-        }
-        if (hca_snapshot.score != nullptr) {
-            ggml_build_forward_expand(gf, hca_snapshot.score);
-        }
+        hca_state_kv    = dsv4_with_zero_dep(ctx0, hca_state_kv,    hca_state_dep);
+        hca_state_score = dsv4_with_zero_dep(ctx0, hca_state_score, hca_state_dep);
 
         ggml_tensor * hca_persist_kv = ggml_get_rows(ctx0, hca_state_kv, inp_dsv4->get_hca().state_persist_src_idxs);
         ggml_tensor * hca_persist_score = ggml_get_rows(ctx0, hca_state_score, inp_dsv4->get_hca().state_persist_src_idxs);
@@ -1222,14 +1047,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl(
     }
 
     ggml_tensor * out = nullptr;
-    if (inp_mtp) {
-        out = build_attn(inp_mtp,
-                nullptr, nullptr, nullptr,
-                q, kv, nullptr,
-                nullptr, layer.attn_sinks, nullptr,
-                1.0f/sqrtf(float(n_embd_head)), il);
-        cb(out, "attn_raw", il);
-    } else if (ratio == DSV4_CSA_RATIO &&
+    if (ratio == DSV4_CSA_RATIO &&
             inp_dsv4->get_csa().kq_mask &&
             inp_dsv4->get_lid().kq_mask &&
             inp_dsv4->get_lid().k_rot) {
@@ -1260,7 +1078,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl(
 
     out = ggml_reshape_3d(ctx0, out, o_group_dim, n_groups, nt);
     out = ggml_permute(ctx0, out, 0, 2, 1, 3);
-    ggml_tensor * oa = ggml_mul_mat(ctx0, layer.wo_a, out);
+    ggml_tensor * oa = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, layer.wo_a, layer.wo_a->ne[0], o_lora_rank, n_groups), out);
     cb(oa, "attn_wo_a", il);
     oa = ggml_permute(ctx0, oa, 0, 2, 1, 3);
     oa = ggml_cont_2d(ctx0, oa, o_lora_rank*n_groups, nt);
@@ -1288,12 +1106,6 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p
     cb(inpL, "hc_init", -1);
 
     for (int il = 0; il < n_layer; ++il) {
-        if ((size_t) il < cparams.embeddings_layer_inp.size() && cparams.embeddings_layer_inp[il]) {
-            res->t_layer_inp[il] = dsv4_hc_mean(ctx0, inpL);
-            cb(res->t_layer_inp[il], "layer_inp", il);
-            ggml_build_forward_expand(gf, res->t_layer_inp[il]);
-        }
-
         ggml_tensor * residual = inpL;
         ggml_tensor * post = nullptr;
         ggml_tensor * comb = nullptr;
@@ -1370,23 +1182,10 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p
         cb(inpL, "l_last", il);
     }
 
-    if ((size_t) n_layer < cparams.embeddings_layer_inp.size() && cparams.embeddings_layer_inp[n_layer]) {
-        res->t_layer_inp[n_layer] = dsv4_hc_mean(ctx0, inpL);
-        cb(res->t_layer_inp[n_layer], "layer_inp", n_layer);
-        ggml_build_forward_expand(gf, res->t_layer_inp[n_layer]);
-    }
-
-    ggml_tensor * flat = ggml_reshape_2d(ctx0, inpL, n_embd*hc, n_tokens);
-    ggml_tensor * flat_out = inp_out_ids ? ggml_get_rows(ctx0, flat, inp_out_ids) : flat;
-
-    if (cparams.embeddings_nextn) {
-        ggml_tensor * h_nextn = cparams.embeddings_nextn_masked ? flat_out : inpL;
-        cb(h_nextn, "h_nextn", -1);
-        res->t_h_nextn = h_nextn;
-    }
-
     if (inp_out_ids) {
-        inpL = ggml_reshape_3d(ctx0, flat_out, n_embd, hc, n_outputs);
+        ggml_tensor * flat = ggml_reshape_2d(ctx0, inpL, n_embd*hc, n_tokens);
+        flat = ggml_get_rows(ctx0, flat, inp_out_ids);
+        inpL = ggml_reshape_3d(ctx0, flat, n_embd, hc, n_outputs);
     }
 
     cur = build_hc_head(inpL, model.hc_head_fn, model.hc_head_scale, model.hc_head_base);
@@ -1402,145 +1201,3 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p
 
     ggml_build_forward_expand(gf, cur);
 }
-
-
-llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) :
-    graph(params) {
-    GGML_ASSERT(hparams.n_layer_nextn > 0 && "DEEPSEEK4 MTP requires n_layer_nextn > 0");
-    GGML_ASSERT(hparams.n_layer_nextn == 1 && "DEEPSEEK4 MTP currently only supports a single MTP block");
-    GGML_ASSERT(cparams.nextn_layer_offset >= 0 &&
-            cparams.nextn_layer_offset < (int) hparams.n_layer_nextn &&
-            "nextn_layer_offset out of range [0, n_layer_nextn)");
-    GGML_ASSERT(ubatch.token && "DEEPSEEK4 MTP requires token input");
-
-    const int64_t hc = hparams.dsv4_hc_mult;
-    GGML_ASSERT(hparams.n_embd_out() == (uint32_t) (n_embd*hc) && "DEEPSEEK4 MTP hidden width mismatch");
-
-    const int il = hparams.n_layer() + cparams.nextn_layer_offset;
-    const auto & layer = model.layers[il];
-
-    GGML_ASSERT(layer.nextn.eh_proj && "MTP block missing nextn.eh_proj");
-    GGML_ASSERT(layer.nextn.enorm   && "MTP block missing nextn.enorm");
-    GGML_ASSERT(layer.nextn.hnorm   && "MTP block missing nextn.hnorm");
-
-    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_out());
-
-    inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
-    ggml_set_input(inp->tokens);
-
-    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_out(), n_tokens);
-    ggml_set_input(inp->embd);
-
-    inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_out(), n_tokens);
-    ggml_set_input(inp->h);
-    ggml_set_name(inp->h, "mtp_h_input");
-
-    ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;
-    ggml_tensor * tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens);
-    cb(tok_embd, "mtp_tok_embd", il);
-
-    ggml_tensor * h_state = ggml_reshape_3d(ctx0, inp->h, n_embd, hc, n_tokens);
-    cb(h_state, "mtp_h_state", il);
-
-    res->add_input(std::move(inp));
-
-    ggml_tensor * inp_pos = build_inp_pos();
-    ggml_tensor * inp_out_ids = build_inp_out_ids();
-    llm_graph_input_attn_k_iswa * inp_attn = build_attn_inp_k_iswa();
-
-    ggml_tensor * h_norm = build_norm(h_state, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il);
-    cb(h_norm, "mtp_hnorm", il);
-
-    ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il);
-    e_norm = ggml_reshape_3d(ctx0, e_norm, n_embd, 1, n_tokens);
-    e_norm = ggml_repeat_4d(ctx0, e_norm, n_embd, hc, n_tokens, 1);
-    cb(e_norm, "mtp_enorm", il);
-
-    ggml_tensor * concat = ggml_concat(ctx0, e_norm, h_norm, 0);
-    cb(concat, "mtp_concat", il);
-
-    ggml_tensor * inpL = build_lora_mm(layer.nextn.eh_proj, concat, layer.nextn.eh_proj_s);
-    cb(inpL, "mtp_eh_proj", il);
-
-    ggml_tensor * residual = inpL;
-    ggml_tensor * post = nullptr;
-    ggml_tensor * comb = nullptr;
-
-    ggml_tensor * cur = build_hc_pre(inpL,
-            layer.hc_attn_fn,
-            layer.hc_attn_scale,
-            layer.hc_attn_base,
-            &post, &comb, il);
-    cb(cur, "mtp_hc_attn_pre", il);
-
-    cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
-    cb(cur, "mtp_attn_norm", il);
-
-    cur = build_attention(model, inp_attn, cur, inp_pos, il);
-
-    inpL = build_hc_post(cur, residual, post, comb, il);
-    cb(inpL, "mtp_hc_attn_post", il);
-
-    residual = inpL;
-    cur = build_hc_pre(inpL,
-            layer.hc_ffn_fn,
-            layer.hc_ffn_scale,
-            layer.hc_ffn_base,
-            &post, &comb, il);
-    cb(cur, "mtp_hc_ffn_pre", il);
-
-    cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
-    cb(cur, "mtp_ffn_norm", il);
-
-    GGML_ASSERT((uint32_t) il >= hparams.dsv4_hash_layer_count && "DEEPSEEK4 MTP does not support hash-routed MTP blocks");
-    ggml_tensor * moe_out = build_moe_ffn(cur,
-            layer.ffn_gate_inp,
-            layer.ffn_up_exps,
-            layer.ffn_gate_exps,
-            layer.ffn_down_exps,
-            layer.ffn_exp_probs_b,
-            n_expert, hparams.n_expert_used,
-            LLM_FFN_SILU, hparams.expert_weights_norm,
-            hparams.expert_weights_scale,
-            (llama_expert_gating_func_type) hparams.expert_gating_func,
-            il);
-    cb(moe_out, "mtp_ffn_moe_out", il);
-
-    ggml_tensor * ffn_shexp = build_ffn(cur,
-            layer.ffn_up_shexp, nullptr, nullptr,
-            layer.ffn_gate_shexp, nullptr, nullptr,
-            layer.ffn_down_shexp, nullptr, nullptr,
-            nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il);
-    cb(ffn_shexp, "mtp_ffn_shexp", il);
-
-    cur = ggml_add(ctx0, moe_out, ffn_shexp);
-    cb(cur, "mtp_ffn_out", il);
-
-    inpL = build_hc_post(cur, residual, post, comb, il);
-    inpL = build_cvec(inpL, il);
-    cb(inpL, "mtp_l_out", il);
-
-    ggml_tensor * flat = ggml_reshape_2d(ctx0, inpL, n_embd*hc, n_tokens);
-    ggml_tensor * h_nextn = ggml_get_rows(ctx0, flat, inp_out_ids);
-    cb(h_nextn, "h_nextn", -1);
-    res->t_h_nextn = h_nextn;
-
-    inpL = ggml_reshape_3d(ctx0, h_nextn, n_embd, hc, n_outputs);
-
-    cur = build_hc_head(inpL, model.hc_head_fn, model.hc_head_scale, model.hc_head_base);
-    cb(cur, "mtp_hc_head", -1);
-
-    ggml_tensor * head_norm_w = layer.nextn.shared_head_norm ? layer.nextn.shared_head_norm : model.output_norm;
-    GGML_ASSERT(head_norm_w && "DEEPSEEK4 MTP missing shared head norm");
-    cur = build_norm(cur, head_norm_w, nullptr, LLM_NORM_RMS, -1);
-    cb(cur, "mtp_shared_head_norm", -1);
-    res->t_embd = cur;
-
-    ggml_tensor * head_w = layer.nextn.shared_head_head ? layer.nextn.shared_head_head : model.output;
-    GGML_ASSERT(head_w && "DEEPSEEK4 MTP missing LM head");
-    cur = ggml_mul_mat(ctx0, head_w, cur);
-    cb(cur, "result_output", -1);
-
-    res->t_logits = cur;
-    ggml_build_forward_expand(gf, cur);
-}

src/models/dflash.cpp

@@ -20,48 +20,6 @@ void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) {
     }
     LLAMA_LOG_INFO("]\n");
 
-    // DeepSeek-V4 DSpark backbone: stages are full DSV4 blocks, uniform sliding window (the draft KV ring)
-    ml.get_key(LLM_KV_HYPER_CONNECTION_COUNT, hparams.dsv4_hc_mult, false);
-    if (hparams.dsv4_hc_mult > 0) {
-        ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK,                hparams.n_lora_q);
-        ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW,             hparams.n_swa);
-        ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,           hparams.n_ff_exp);
-        ml.get_key(LLM_KV_EXPERT_SHARED_COUNT,                  hparams.n_expert_shared);
-        ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,                 hparams.expert_weights_scale);
-        ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,                  hparams.expert_weights_norm);
-        ml.get_key(LLM_KV_EXPERT_GATING_FUNC,                   hparams.expert_gating_func);
-        ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP,              hparams.swiglu_clamp_exp, hparams.n_layer_all);
-        if (!ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP,       hparams.swiglu_clamp_shexp, hparams.n_layer_all, 0)) {
-            hparams.swiglu_clamp_shexp = hparams.swiglu_clamp_exp;
-        }
-        ml.get_key(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT,         hparams.dsv4_o_group_count);
-        ml.get_key(LLM_KV_ATTENTION_OUTPUT_LORA_RANK,           hparams.dsv4_o_lora_rank);
-        ml.get_key(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, hparams.dsv4_hc_sinkhorn_iters);
-        ml.get_key(LLM_KV_HYPER_CONNECTION_EPSILON,             hparams.dsv4_hc_eps);
-        ml.get_arr(LLM_KV_ATTENTION_COMPRESS_RATIOS,            hparams.dsv4_compress_ratios, false);
-
-        if (hparams.expert_gating_func != LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS) {
-            throw std::runtime_error("DSpark DSV4 draft expects sqrtsoftplus MoE scoring");
-        }
-        for (uint32_t il = 0; il < hparams.n_layer_all; ++il) {
-            if (hparams.dsv4_compress_ratios[il] != 0) {
-                throw std::runtime_error("DSpark DSV4 draft expects uncompressed attention on all stages");
-            }
-        }
-
-        GGML_ASSERT(hparams.n_swa > 0);
-        hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
-        hparams.set_swa_pattern(0);
-        for (uint32_t il = 0; il < hparams.n_layer_all; ++il) {
-            hparams.is_swa_impl[il] = true;
-        }
-        hparams.rope_freq_base_train_swa  = hparams.rope_freq_base_train;
-        hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
-
-        type = LLM_TYPE_UNKNOWN;
-        return;
-    }
-
     // optional interleaved sliding-window attention with per-layer pattern array.
     // DFlash has a single rope, so the SWA rope == main rope.
     if (ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false) && hparams.n_swa > 0) {
@@ -100,56 +58,6 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) {
     output_norm_enc = create_tensor(tn(LLM_TENSOR_ENC_OUTPUT_NORM, "weight"), { n_embd }, 0); // encoder hidden_norm (after fc)
     output_norm     = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM,    "weight"), { n_embd }, 0); // decoder final norm
 
-    if (hparams.dsv4_hc_mult > 0) {
-        const int64_t q_lora_rank     = hparams.n_lora_q;
-        const int64_t n_ff_exp        = hparams.n_ff_exp;
-        const int64_t n_expert_shared = hparams.n_expert_shared;
-        const int64_t n_embd_head     = hparams.n_embd_head_k();
-        const int64_t o_groups        = hparams.dsv4_o_group_count;
-        const int64_t o_lora_rank     = hparams.dsv4_o_lora_rank;
-        const int64_t hc_mult         = hparams.dsv4_hc_mult;
-        const int64_t hc_dim          = hc_mult * n_embd;
-        const int64_t hc_mix_dim      = (2 + hc_mult) * hc_mult;
-
-        hc_head_fn    = create_tensor(tn(LLM_TENSOR_HC_HEAD_FN,    "weight"), {hc_dim, hc_mult}, 0);
-        hc_head_base  = create_tensor(tn(LLM_TENSOR_HC_HEAD_BASE,  "weight"), {hc_mult}, 0);
-        hc_head_scale = create_tensor(tn(LLM_TENSOR_HC_HEAD_SCALE, "weight"), {1}, 0);
-
-        for (int i = 0; i < n_layer; ++i) {
-            auto & layer = layers[i];
-
-            layer.attn_norm     = create_tensor(tn(LLM_TENSOR_ATTN_NORM,     "weight", i), {n_embd}, 0);
-            layer.attn_sinks    = create_tensor(tn(LLM_TENSOR_ATTN_SINKS,    "weight", i), {n_head}, 0);
-            layer.wq_a          = create_tensor(tn(LLM_TENSOR_ATTN_Q_A,      "weight", i), {n_embd, q_lora_rank}, 0);
-            layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, 0);
-            layer.wq_b          = create_tensor(tn(LLM_TENSOR_ATTN_Q_B,      "weight", i), {q_lora_rank, n_head * n_embd_head}, 0);
-            layer.wkv           = create_tensor(tn(LLM_TENSOR_ATTN_KV,       "weight", i), {n_embd, n_embd_head}, 0);
-            layer.attn_kv_norm  = create_tensor(tn(LLM_TENSOR_ATTN_KV_NORM,  "weight", i), {n_embd_head}, 0);
-            layer.wo_a          = create_tensor(tn(LLM_TENSOR_ATTN_OUT_A,    "weight", i), {n_head * n_embd_head / o_groups, o_lora_rank * o_groups}, 0);
-            layer.wo_b          = create_tensor(tn(LLM_TENSOR_ATTN_OUT_B,    "weight", i), {o_groups * o_lora_rank, n_embd}, 0);
-
-            layer.hc_attn_fn    = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN,    "weight", i), {hc_dim, hc_mix_dim}, 0);
-            layer.hc_attn_base  = create_tensor(tn(LLM_TENSOR_HC_ATTN_BASE,  "weight", i), {hc_mix_dim}, 0);
-            layer.hc_attn_scale = create_tensor(tn(LLM_TENSOR_HC_ATTN_SCALE, "weight", i), {3}, 0);
-            layer.hc_ffn_fn     = create_tensor(tn(LLM_TENSOR_HC_FFN_FN,     "weight", i), {hc_dim, hc_mix_dim}, 0);
-            layer.hc_ffn_base   = create_tensor(tn(LLM_TENSOR_HC_FFN_BASE,   "weight", i), {hc_mix_dim}, 0);
-            layer.hc_ffn_scale  = create_tensor(tn(LLM_TENSOR_HC_FFN_SCALE,  "weight", i), {3}, 0);
-
-            layer.ffn_gate_inp    = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,    "weight", i), {n_embd, n_expert}, 0);
-            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias",   i), {n_expert}, 0);
-            layer.ffn_norm        = create_tensor(tn(LLM_TENSOR_FFN_NORM,        "weight", i), {n_embd}, 0);
-
-            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd,   n_ff_exp, n_expert}, 0);
-            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd,   n_expert}, 0);
-            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   "weight", i), {n_embd,   n_ff_exp, n_expert}, 0);
-
-            layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd,                     n_ff_exp * n_expert_shared}, 0);
-            layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_exp * n_expert_shared, n_embd                    }, 0);
-            layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   "weight", i), {n_embd,                     n_ff_exp * n_expert_shared}, 0);
-        }
-        return;
-    }
-
     for (int i = 0; i < n_layer; ++i) {
         auto & layer = layers[i];
 
@@ -176,9 +84,6 @@ std::unique_ptr<llm_graph_context> llama_model_dflash::build_arch_graph(const ll
             return std::make_unique<graph<true>>(*this, params);
         case LLM_GRAPH_TYPE_DEFAULT:
         case LLM_GRAPH_TYPE_DECODER:
-            if (hparams.dsv4_hc_mult > 0) {
-                return std::make_unique<graph_dsv4>(*this, params);
-            }
             return std::make_unique<graph<false>>(*this, params);
         default:
             GGML_ABORT("invalid graph type");
@@ -498,178 +403,3 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra
         build_dspark_markov_head(*this, model, inp_tokens);
     }
 }
-
-// DSV4 DSpark decoder, dual-mode by batch type (see the DFlash decoder above):
-//   * embd batch  -> project main_x through each stage's wkv and inject K into the ring cache
-//   * token batch -> noise block through 3 full DSV4 stages (hc + MLA + MoE), markov + confidence heads
-llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_graph_params & params) :
-    llama_model_deepseek4::graph(params) {
-    const int64_t n_embd_head      = hparams.n_embd_head_k();
-    const int64_t n_embd_head_rope = hparams.n_rot();
-    const int64_t n_embd_head_nope = n_embd_head - n_embd_head_rope;
-
-    ggml_tensor * inp_pos = build_inp_pos();
-
-    llm_graph_input_attn_k_iswa * inp_attn = build_attn_inp_k_iswa();
-
-    // KV cache injection: fused target features from the encoder
-    if (ubatch.embd) {
-        auto inp = std::make_unique<llm_graph_input_embd>(n_embd);
-
-        inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_tokens);
-        ggml_set_input(inp->embd);
-
-        ggml_tensor * inp_g = inp->embd;
-        cb(inp_g, "inp_g_embeddings", -1);
-
-        res->add_input(std::move(inp));
-
-        for (int il = 0; il < n_layer; ++il) {
-            const auto & layer = model.layers[il];
-
-            // main-track KV: kv_norm(wkv(main_x)) with rope on the trailing dims, same
-            // rope parameters as the uncompressed layers in build_attention_impl
-            ggml_tensor * kv = build_lora_mm(layer.wkv, inp_g);
-            kv = build_norm(kv, layer.attn_kv_norm, nullptr, LLM_NORM_RMS, il);
-            kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, n_tokens);
-
-            ggml_tensor * kv_nope = ggml_view_3d(ctx0, kv, n_embd_head_nope, 1, n_tokens,
-                    ggml_row_size(kv->type, n_embd_head),
-                    ggml_row_size(kv->type, n_embd_head),
-                    0);
-            ggml_tensor * kv_pe = ggml_view_3d(ctx0, kv, n_embd_head_rope, 1, n_tokens,
-                    ggml_row_size(kv->type, n_embd_head),
-                    ggml_row_size(kv->type, n_embd_head),
-                    ggml_row_size(kv->type, n_embd_head_nope));
-            kv_pe = ggml_rope_ext(ctx0, kv_pe, inp_pos, nullptr, n_embd_head_rope, rope_type, 0,
-                    freq_base, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
-            kv = ggml_concat(ctx0, kv_nope, kv_pe, 0);
-            cb(kv, "kv_injected", il);
-
-            if (inp_attn->self_k_rot_swa) {
-                kv = llama_mul_mat_hadamard(ctx0, kv, inp_attn->self_k_rot_swa);
-            }
-            ggml_build_forward_expand(gf, inp_attn->mctx->get_swa()->cpy_k(ctx0, kv, inp_attn->get_k_idxs_swa(), il));
-        }
-
-        res->t_embd = inp_g;
-
-        ggml_build_forward_expand(gf, inp_g);
-        return;
-    }
-
-    // tok_embd from the target model (shared via ctx_other)
-    auto * tok_embd = model.tok_embd;
-    if (tok_embd == nullptr) {
-        GGML_ASSERT(cparams.ctx_other != nullptr);
-        const auto * model_other = llama_get_model(cparams.ctx_other);
-
-        GGML_ASSERT(model_other->tok_embd != nullptr && "DSpark decoder requires the target model's token embeddings");
-        tok_embd = model_other->tok_embd;
-    }
-
-    auto inp = std::make_unique<llm_graph_input_embd>(n_embd);
-
-    inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
-    ggml_set_input(inp->tokens);
-
-    ggml_tensor * inp_tokens = inp->tokens;
-
-    ggml_tensor * inpL = ggml_get_rows(ctx0, tok_embd, inp->tokens);
-    cb(inpL, "inp_noise_embd", -1);
-
-    res->add_input(std::move(inp));
-
-    const int64_t hc = hparams.dsv4_hc_mult;
-    inpL = ggml_reshape_3d(ctx0, inpL, n_embd, 1, n_tokens);
-    inpL = ggml_repeat_4d(ctx0, inpL, n_embd, hc, n_tokens, 1);
-    cb(inpL, "hc_init", -1);
-
-    for (int il = 0; il < n_layer; ++il) {
-        const auto & layer = model.layers[il];
-
-        ggml_tensor * residual = inpL;
-        ggml_tensor * post = nullptr;
-        ggml_tensor * comb = nullptr;
-
-        ggml_tensor * cur = build_hc_pre(inpL,
-                layer.hc_attn_fn,
-                layer.hc_attn_scale,
-                layer.hc_attn_base,
-                &post, &comb, il);
-        cb(cur, "hc_attn_pre", il);
-
-        cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
-        cb(cur, "attn_norm", il);
-
-        cur = build_attention(model, inp_attn, cur, inp_pos, il);
-
-        inpL = build_hc_post(cur, residual, post, comb, il);
-        cb(inpL, "hc_attn_post", il);
-
-        residual = inpL;
-        cur = build_hc_pre(inpL,
-                layer.hc_ffn_fn,
-                layer.hc_ffn_scale,
-                layer.hc_ffn_base,
-                &post, &comb, il);
-        cb(cur, "hc_ffn_pre", il);
-
-        cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
-        cb(cur, "ffn_norm", il);
-
-        ggml_tensor * moe_out = build_moe_ffn(cur,
-                layer.ffn_gate_inp,
-                layer.ffn_up_exps,
-                layer.ffn_gate_exps,
-                layer.ffn_down_exps,
-                layer.ffn_exp_probs_b,
-                n_expert, hparams.n_expert_used,
-                LLM_FFN_SILU, hparams.expert_weights_norm,
-                hparams.expert_weights_scale,
-                (llama_expert_gating_func_type) hparams.expert_gating_func,
-                il);
-        cb(moe_out, "ffn_moe_out", il);
-
-        ggml_tensor * ffn_shexp = build_ffn(cur,
-                layer.ffn_up_shexp, nullptr, nullptr,
-                layer.ffn_gate_shexp, nullptr, nullptr,
-                layer.ffn_down_shexp, nullptr, nullptr,
-                nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il);
-        cb(ffn_shexp, "ffn_shexp", il);
-
-        cur = ggml_add(ctx0, moe_out, ffn_shexp);
-        cb(cur, "ffn_out", il);
-
-        inpL = build_hc_post(cur, residual, post, comb, il);
-        cb(inpL, "l_out", il);
-    }
-
-    ggml_tensor * cur = build_hc_head(inpL, model.hc_head_fn, model.hc_head_scale, model.hc_head_base);
-    cb(cur, "hc_head", -1);
-
-    // confidence head input: the reference scores the pre-norm collapsed hidden state
-    res->t_embd = cur;
-
-    cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);
-    cb(cur, "result_norm", -1);
-
-    // lm_head from the target model (shared via ctx_other)
-    auto * output = model.output;
-    if (output == nullptr) {
-        GGML_ASSERT(cparams.ctx_other != nullptr);
-        const auto * model_other = llama_get_model(cparams.ctx_other);
-        GGML_ASSERT(model_other->output != nullptr && "DSpark decoder requires the target model's output projection");
-        output = model_other->output;
-    }
-
-    cur = build_lora_mm(output, cur);
-    cb(cur, "result_output", -1);
-    res->t_logits = cur;
-
-    ggml_build_forward_expand(gf, cur);
-
-    if (model.dspark_markov_w1) {
-        build_dspark_markov_head(*this, model, inp_tokens);
-    }
-}

src/models/mimo2.cpp

@@ -30,11 +30,7 @@ void llama_model_mimo2::load_arch_tensors(llama_model_loader & ml) {
 
     const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";
     const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);
-    int mtp_flags         = trunk_only ? TENSOR_NOT_REQUIRED : 0;
-
-    if (!ml.load_mtp) {
-        mtp_flags |= TENSOR_SKIP;
-    }
+    const int  mtp_flags  = trunk_only ? TENSOR_NOT_REQUIRED : 0;
 
     tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
 

src/models/minimax-m3.cpp

@@ -1,5 +1,5 @@
 #include "models.h"
-#include "llama-kv-cache-msa.h"
+#include "llama-kv-cache.h"
 #include <cmath>
 #include <vector>
 #include <cstdint>
@@ -7,8 +7,7 @@
 // MiniMax-M3: MiniMax-M2 style GQA (per-head QK-norm, partial rotary) with
 // DeepSeek-V3 leading-dense + routed/shared experts (sigmoid gating, routed scaling),
 // swigluoai activation, and MiniMax Sparse Attention (MSA). MTP is not in released model weights.
-// MSA blocks are defined over token positions. The graph translates between position space (block
-// selection) and cell space (K/V/indexer storage) via per-ubatch pos<->cell maps populated from llama_kv_cells
+// Notes: Blocks are anchored to absolute KV cache slots.
 
 void llama_model_minimax_m3::load_arch_hparams(llama_model_loader & ml) {
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
@@ -24,6 +23,7 @@ void llama_model_minimax_m3::load_arch_hparams(llama_model_loader & ml) {
     ml.get_key(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE,    hparams.indexer_block_size);
     ml.get_key(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS,  hparams.indexer_local_blocks);
     msa_p = { (int) hparams.indexer_block_size, (int) hparams.indexer_top_k, (int) hparams.indexer_local_blocks };
+    hparams.indexer_kv = true;
 
     switch (hparams.n_layer()) {
         case 60: type = LLM_TYPE_428B_A23B; break;
@@ -86,83 +86,43 @@ std::unique_ptr<llm_graph_context> llama_model_minimax_m3::build_arch_graph(cons
     return std::make_unique<graph>(*this, params);
 }
 
-class llm_graph_input_msa : public llm_graph_input_i {
+// per-query local-force bias for MSA selection
+// local window always wins a slot
+class llm_graph_input_msa_local : public llm_graph_input_i {
 public:
-    llm_graph_input_msa(const llama_kv_cache_msa_context * mctx, int blk, int local) :
-        mctx(mctx), blk(blk), local(local) {}
+    llm_graph_input_msa_local(int blk, int local, int64_t nblk) : blk(blk), local(local), nblk(nblk) {}
 
     void set_input(const llama_ubatch * ubatch) override {
-        if (pos_slot_i) { mctx->set_input_pos_slot(pos_slot_i, ubatch); }
-        if (pos_slot_f) { mctx->set_input_pos_slot(pos_slot_f, ubatch); }
-        if (cell_blk)   { mctx->set_input_cell_pos(cell_blk, ubatch, blk); }
-        if (pos_mask)   { mctx->set_input_pos_mask(pos_mask, ubatch); }
-
-        // local-force bias over position blocks
-        if (bias && ubatch->pos) {
-            const int64_t n_tokens = ubatch->n_tokens;
-            const int64_t nblk     = bias->ne[0];
-            std::vector<float> data((size_t) nblk * n_tokens, 0.0f);
-            for (int64_t i = 0; i < n_tokens; ++i) {
-                const int64_t L = ubatch->pos[i] / blk;
-                for (int l = 0; l < local && L - l >= 0; ++l) {
-                    if (L - l < nblk) {
-                        data[(size_t) i * nblk + (L - l)] = 1e30f;
-                    }
+        if (!bias || !ubatch->pos) {
+            return;
+        }
+        const int64_t n_tokens = ubatch->n_tokens;
+        std::vector<float> data((size_t) nblk * n_tokens, 0.0f);
+        for (int64_t i = 0; i < n_tokens; ++i) {
+            const int64_t L = ubatch->pos[i] / blk;
+            for (int l = 0; l < local && L - l >= 0; ++l) {
+                if (L - l < nblk) {
+                    data[(size_t) i * nblk + (L - l)] = 1e30f;
                 }
             }
-            ggml_backend_tensor_set(bias, data.data(), 0, data.size() * sizeof(float));
         }
+        ggml_backend_tensor_set(bias, data.data(), 0, data.size() * sizeof(float));
     }
 
-    // valid as long as the tensor dims still match the new ubatch/cache window and the
-    // ubatch is in the same regime (decode graphs have pos_slot_f, batch graphs cell_blk)
+    // valid as long as the bias tensor dims still match the new ubatch/cache window
     bool can_reuse(const llm_graph_params & params) override {
-        const auto * mctx_new = static_cast<const llama_kv_cache_msa_context *>(params.mctx);
-
-        this->mctx = mctx_new;
-
-        const int64_t n_ps = GGML_PAD((int64_t) mctx_new->get_n_pos(), blk);
-        const int64_t ns   = params.cparams.kv_unified ? 1 : params.ubatch.n_seqs_unq;
-
-        const bool decode = params.ubatch.n_tokens == ns;   // one token per stream
+        const auto * mctx = static_cast<const llama_kv_cache_context *>(params.mctx);
 
         bool res = true;
-
-        res &= bias->ne[0] * blk == n_ps;
-        res &= bias->ne[1]       == params.ubatch.n_tokens;
-
-        res &= pos_mask->ne[0] == n_ps;
-        res &= pos_mask->ne[1] == params.ubatch.n_tokens;
-
-        res &= pos_slot_i->ne[0] == n_ps;
-        res &= pos_slot_i->ne[1] == ns;
-
-        res &= decode == (pos_slot_f != nullptr);
-        res &= decode == (cell_blk   == nullptr);
-
-        if (pos_slot_f) {
-            res &= pos_slot_f->ne[0] == n_ps;
-            res &= pos_slot_f->ne[1] == ns;
-        }
-
-        if (cell_blk) {
-            res &= cell_blk->ne[0] == (int64_t) mctx_new->get_base()->get_n_kv();
-            res &= cell_blk->ne[1] == ns;
-        }
-
+        res &= bias->ne[1] == params.ubatch.n_tokens;
+        res &= bias->ne[0] * blk == (int64_t) mctx->get_n_kv();
         return res;
     }
 
-    ggml_tensor * bias       = nullptr; // F32 [nblk, n_tokens] local-force bias (position blocks)
-    ggml_tensor * pos_mask   = nullptr; // F32 [n_ps, n_tokens] 0/-inf visibility, by position
-    ggml_tensor * pos_slot_i = nullptr; // I32 [n_ps, ns]       pos -> cell (get_rows index)
-    ggml_tensor * pos_slot_f = nullptr; // F32 [n_ps, ns]       pos -> cell (gatherable values, decode)
-    ggml_tensor * cell_blk   = nullptr; // I32 [n_kv, ns]       cell -> position block (batch)
-
-    const llama_kv_cache_msa_context * mctx;
-
-    int blk;
-    int local;
+    ggml_tensor * bias = nullptr;
+    int     blk;
+    int     local;
+    int64_t nblk;
 };
 
 // One FA call for all GQA groups (and at multi-stream decode, all streams) by mapping them onto the FA sequence dim (ne[3])
@@ -213,9 +173,7 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_
     inpL = build_inp_embd(model.tok_embd);
 
     ggml_tensor * inp_pos = build_inp_pos();
-
-    // ==========================================
-    // TODO: avoid such kind of complexity in the model graphs
+    auto inp_attn = build_attn_inp_kv();
 
     // MSA calls ggml_flash_attn_ext directly and assumes the non-transposed V layout that
     // llama.cpp only provides when flash attention is enabled. Block selection is anchored
@@ -227,8 +185,6 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_
     const bool streams_ok  = cparams.n_seq_max == 1 || !cparams.kv_unified;
     const bool msa_enabled = fa_on && streams_ok;
 
-    auto * inp_attn = build_attn_inp_kv_msa(msa_enabled);
-
     static bool warned_no_fa = false;
     if (!fa_on && !warned_no_fa) {
         LLAMA_LOG_WARN("%s: flash attention disabled; MSA requires it -> running DENSE attention "
@@ -241,54 +197,36 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_
                        "-> running DENSE attention. Output may be degraded. Drop --kv-unified to enable MSA.\n", __func__);
         warned_unified = true;
     }
-    // ==========================================
 
     // hoisted per-graph MSA state (shared by every sparse layer)
-    llm_graph_input_msa * msa = nullptr;
+    llm_graph_input_msa_local * msa_loc = nullptr;
     ggml_tensor * msa_kqm = nullptr;
-    ggml_tensor * msa_mf  = nullptr;   // F32 copy of the FA mask for the final mask add
-    int64_t n_kv = 0, n_ps = 0, nblk = 0, ns = 1, n_tps = 0;
+    ggml_tensor * msa_mf  = nullptr;
+    int64_t n_kv = 0, nblk = 0, ns = 1, n_tps = 0;
     bool msa_decode = false;           // gather (1 token per stream) vs mask
     const int     blk = mm.msa_p.blk;
     const int64_t Hd  = hparams.indexer_n_head;   // one indexer head per GQA group
 
     if (msa_enabled) {
-        const auto * mctx_msa = static_cast<const llama_kv_cache_msa_context *>(mctx);
-
         msa_kqm = inp_attn->get_kq_mask();
         n_kv  = msa_kqm->ne[0];
         n_tps = msa_kqm->ne[1];        // tokens per stream
         ns    = msa_kqm->ne[3];        // streams in this ubatch
         GGML_ASSERT(msa_kqm->type == GGML_TYPE_F16 && "MSA requires the FA (f16) mask");
         GGML_ASSERT(n_tps*ns == n_tokens);
-
-        // the position axis covers every position currently in the cache and is padded to whole blocks
-        n_ps = GGML_PAD((int64_t) mctx_msa->get_n_pos(), blk);
-        nblk = n_ps / blk;
+        GGML_ASSERT(n_kv % blk == 0 &&
+            "MSA: KV/mask n_kv must be a multiple of indexer.block_size (128); "
+            "the flash-attention KV padding must be a multiple of the block size. "
+            "A non-multiple would silently drop the partial tail block.");
+        nblk = n_kv / blk;
         msa_decode = n_tps == 1;
 
-        auto inp = std::make_unique<llm_graph_input_msa>(mctx_msa, blk, mm.msa_p.local);
+        msa_mf = ggml_cast(ctx0, msa_kqm, GGML_TYPE_F32);
 
-        inp->bias = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, nblk, n_tokens);  // stream-grouped tokens
-        ggml_set_input(inp->bias);
-
-        inp->pos_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_ps, n_tokens);
-        ggml_set_input(inp->pos_mask);
-
-        inp->pos_slot_i = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_ps, ns);
-        ggml_set_input(inp->pos_slot_i);
-
-        if (msa_decode) {
-            inp->pos_slot_f = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_ps, ns);
-            ggml_set_input(inp->pos_slot_f);
-        } else {
-            inp->cell_blk = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_kv, ns);
-            ggml_set_input(inp->cell_blk);
-
-            msa_mf = ggml_cast(ctx0, msa_kqm, GGML_TYPE_F32);
-        }
-
-        msa = (llm_graph_input_msa *) res->add_input(std::move(inp));
+        auto loc = std::make_unique<llm_graph_input_msa_local>(blk, mm.msa_p.local, nblk);
+        loc->bias = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, nblk, n_tokens);  // stream-grouped tokens
+        ggml_set_input(loc->bias);
+        msa_loc = (llm_graph_input_msa_local *) res->add_input(std::move(loc));
     }
 
     ggml_tensor * inp_out_ids = build_inp_out_ids();
@@ -345,11 +283,9 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_
                 ik = ggml_rope_ext(ctx0, ik, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
                                    freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
 
-                const auto * mctx_msa_l = static_cast<const llama_kv_cache_msa_context *>(mctx);
-                const auto * mctx_cur = mctx_msa_l->get_base();
-                const auto * mctx_idx = mctx_msa_l->get_idx();
-                ggml_build_forward_expand(gf, mctx_idx->cpy_k(ctx0, ik, inp_attn->get_k_idxs_idx(), il));
-                ggml_tensor * ik_kv = mctx_idx->get_k(ctx0, il);
+                const auto * mctx_cur = inp_attn->mctx;
+                ggml_build_forward_expand(gf, mctx_cur->cpy_k_idx(ctx0, ik, inp_attn->get_k_idxs(), il));
+                ggml_tensor * ik_kv = mctx_cur->get_k_idx(ctx0, il);
 
                 if (inp_attn->self_k_rot) {
                     Qcur = llama_mul_mat_hadamard(ctx0, Qcur, inp_attn->self_k_rot);
@@ -380,52 +316,42 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_
 
                 if (msa_decode) {
                     // decode: batched over streams top-k + gather, one grouped FA
-                    // gather the indexer keys through the pos -> cell map
-                    ggml_tensor * ik3 = ggml_view_3d(ctx0, ik_kv, n_idx_dim, n_kv, ns,
-                            ik_kv->nb[2], ik_kv->nb[3], 0);
-                    ggml_tensor * ikp = ggml_get_rows(ctx0, ik3, msa->pos_slot_i);   // [n_idx_dim, n_ps, ns]
+                    // scores: per-stream batched matmul over the stream dim (ne[3]).
+                    // the cache views are not contiguous across streams (stride = kv_size, not n_kv)
+                    ggml_tensor * ikv4 = ggml_view_4d(ctx0, ik_kv, n_idx_dim, n_kv, 1, ns,
+                            ik_kv->nb[2], ik_kv->nb[3], ik_kv->nb[3], 0);
                     ggml_tensor * iq4 = ggml_reshape_4d(ctx0, iq, n_idx_dim, Hd, 1, ns);
-                    ggml_tensor * sc  = ggml_mul_mat(ctx0,
-                            ggml_reshape_4d(ctx0, ikp, n_idx_dim, n_ps, 1, ns), iq4);
+                    ggml_tensor * sc  = ggml_mul_mat(ctx0, ikv4, iq4);
                     ggml_mul_mat_set_prec(sc, GGML_PREC_F32);
-                    // unmapped positions come out -inf, so they can never rank into the top-k
-                    sc = ggml_add_inplace(ctx0, sc,
-                            ggml_reshape_4d(ctx0, msa->pos_mask, n_ps, 1, 1, ns));
+                    sc = ggml_add_inplace(ctx0, sc, msa_mf);
                     ggml_tensor * bs = ggml_pool_2d(ctx0, sc, GGML_OP_POOL_MAX, blk, 1, blk, 1, 0, 0);
                     cb(bs, "msa_bs", il);
 
                     ggml_tensor * bsf = ggml_add(ctx0, bs,
-                            ggml_reshape_4d(ctx0, msa->bias, nblk, 1, 1, ns));
-                    ggml_tensor * idx = ggml_top_k(ctx0, bsf, K);   // position blocks
+                            ggml_reshape_4d(ctx0, msa_loc->bias, nblk, 1, 1, ns));
+                    ggml_tensor * idx = ggml_top_k(ctx0, bsf, K);
 
-                    // pos idx:  tj[t,k,h,s] = blk*idx[k,h,s] + t   (positions - mask gather)
-                    // cell idx: cs[t,k,h,s] = pos_slot[tj]         (pos -> cell translation)
-                    // row idx:  tr[t,k,h,s] = cs*HKV + h           (per-stream K/V gather)
+                    // token idx: tj[t,k,h,s] = blk*idx[k,h,s] + t   (for the mask gather)
+                    // row   idx: tr[t,k,h,s] = tj*HKV + h           (for the per-stream K/V gather)
                     ggml_tensor * a = ggml_scale(ctx0, ggml_cast(ctx0, idx, GGML_TYPE_F32), (float) blk);
                     a = ggml_reshape_4d(ctx0, a, 1, K, Hd, ns);
                     ggml_tensor * tj = ggml_add(ctx0,
                             ggml_repeat_4d(ctx0, a, blk, K, Hd, ns),
                             ggml_reshape_3d(ctx0, ggml_arange(ctx0, 0.0f, (float) blk, 1.0f), blk, 1, 1));
-
-                    ggml_tensor * tokj = ggml_cast(ctx0, ggml_reshape_2d(ctx0, tj, (int64_t) blk*K*Hd, ns), GGML_TYPE_I32);
-
-                    ggml_tensor * cs = ggml_get_rows(ctx0,
-                            ggml_reshape_3d(ctx0, msa->pos_slot_f, 1, n_ps, ns), tokj);   // [1, blk*K*Hd, ns]
-                    cs = ggml_reshape_4d(ctx0, cs, blk, K, Hd, ns);
-
                     ggml_tensor * tr = ggml_add(ctx0,
-                            ggml_scale(ctx0, cs, (float) HKV),
+                            ggml_scale(ctx0, tj, (float) HKV),
                             ggml_reshape_3d(ctx0, ggml_arange(ctx0, 0.0f, (float) HKV, 1.0f), 1, 1, Hd));
 
+                    ggml_tensor * tokj = ggml_cast(ctx0, ggml_reshape_2d(ctx0, tj, (int64_t) blk*K*Hd, ns), GGML_TYPE_I32);
                     ggml_tensor * tokr = ggml_cast(ctx0, ggml_reshape_2d(ctx0, tr, (int64_t) blk*K*Hd, ns), GGML_TYPE_I32);
 
                     ggml_tensor * k3 = ggml_view_3d(ctx0, k, D, HKV*n_kv, ns, k->nb[1], k->nb[3], 0);
                     ggml_tensor * v3 = ggml_view_3d(ctx0, v, D, HKV*n_kv, ns, v->nb[1], v->nb[3], 0);
-                    ggml_tensor * mp = ggml_reshape_3d(ctx0, msa->pos_mask, 1, n_ps, ns);
+                    ggml_tensor * m3 = ggml_reshape_3d(ctx0, msa_kqm, 1, n_kv, ns);
 
                     ggml_tensor * kg = ggml_get_rows(ctx0, k3, tokr);
                     ggml_tensor * vg = ggml_get_rows(ctx0, v3, tokr);
-                    ggml_tensor * mg = ggml_get_rows(ctx0, mp, tokj);
+                    ggml_tensor * mg = ggml_get_rows(ctx0, m3, tokj);
 
                     // fold (group, stream) onto the FA channel dim
                     const ggml_type kt = ggml_is_quantized(k->type) ? GGML_TYPE_F16 : k->type;
@@ -446,16 +372,12 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_
                                 iq->nb[1], iq->nb[2], st*n_tps*iq->nb[2]);
                         ggml_tensor * ik_s = ggml_view_2d(ctx0, ik_kv, n_idx_dim, n_kv,
                                 ik_kv->nb[2], st*ik_kv->nb[3]);
-                        ggml_tensor * psl_s = ggml_view_1d(ctx0, msa->pos_slot_i, n_ps,
-                                st*msa->pos_slot_i->nb[1]);
-                        ggml_tensor * pm_s = ggml_view_3d(ctx0, msa->pos_mask, n_ps, 1, n_tps,
-                                msa->pos_mask->nb[1], msa->pos_mask->nb[1], st*n_tps*msa->pos_mask->nb[1]);
-                        ggml_tensor * cb_s = ggml_view_1d(ctx0, msa->cell_blk, n_kv,
-                                st*msa->cell_blk->nb[1]);
-                        ggml_tensor * mf_s = ggml_view_3d(ctx0, msa_mf, n_kv, n_tps, 1,
-                                msa_mf->nb[1], msa_mf->nb[3], st*msa_mf->nb[3]);
-                        ggml_tensor * bias_s = ggml_view_3d(ctx0, msa->bias, nblk, 1, n_tps,
-                                msa->bias->nb[1], msa->bias->nb[1], st*n_tps*msa->bias->nb[1]);
+                        ggml_tensor * mf_s = ggml_view_3d(ctx0, msa_mf, n_kv, 1, n_tps,
+                                msa_mf->nb[1], msa_mf->nb[1], st*msa_mf->nb[3]);
+                        ggml_tensor * km_s = ggml_view_3d(ctx0, msa_kqm, n_kv, n_tps, 1,
+                                msa_kqm->nb[1], msa_kqm->nb[3], st*msa_kqm->nb[3]);
+                        ggml_tensor * bias_s = ggml_view_3d(ctx0, msa_loc->bias, nblk, 1, n_tps,
+                                msa_loc->bias->nb[1], msa_loc->bias->nb[1], st*n_tps*msa_loc->bias->nb[1]);
                         ggml_tensor * q_s = ggml_view_3d(ctx0, Qcur, D, n_head, n_tps,
                                 Qcur->nb[1], Qcur->nb[2], st*n_tps*Qcur->nb[2]);
                         ggml_tensor * k_s = ggml_view_4d(ctx0, k, D, HKV, n_kv, 1,
@@ -463,16 +385,14 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_
                         ggml_tensor * v_s = ggml_view_4d(ctx0, v, D, HKV, n_kv, 1,
                                 v->nb[1], v->nb[2], v->nb[3], st*v->nb[3]);
 
-                        // block scores: the indexer keys are gathered through the pos -> cell map first
+                        // block scores: bs = maxpool_blk(idx_q * idx_k^T + causal mask)
                         // scores are unscaled, only the top-k ordering matters
-                        ggml_tensor * ikp = ggml_get_rows(ctx0, ik_s, psl_s);   // [n_idx_dim, n_ps]
-                        ggml_tensor * sc = ggml_mul_mat(ctx0, ikp,
+                        ggml_tensor * sc = ggml_mul_mat(ctx0, ik_s,
                                 ggml_reshape_2d(ctx0, iq_s, n_idx_dim, Hd*n_tps));
                         // indexer scores run in F32
                         ggml_mul_mat_set_prec(sc, GGML_PREC_F32);
-                        sc = ggml_reshape_3d(ctx0, sc, n_ps, Hd, n_tps);
-                        // unmapped positions (holes, padding, empty cells) come out -inf
-                        sc = ggml_add_inplace(ctx0, sc, pm_s);
+                        sc = ggml_reshape_3d(ctx0, sc, n_kv, Hd, n_tps);
+                        sc = ggml_add_inplace(ctx0, sc, mf_s);
                         ggml_tensor * bs = ggml_pool_2d(ctx0, sc, GGML_OP_POOL_MAX, blk, 1, blk, 1, 0, 0);
                         cb(bs, "msa_bs", il);
 
@@ -496,16 +416,14 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_
                         bm = ggml_cont(ctx0, ggml_permute(ctx0, bm, 0, 2, 1, 3)); // [nblk, n_tps, Hd]
                         cb(bm, "msa_block_mask", il);
 
-                        // expand block -> cell granularity through the cell -> position block
-                        // map, then combine with the causal mask. empty cells are masked by the causal mask.
-                        ggml_tensor * bm2 = ggml_cont(ctx0, ggml_transpose(ctx0,
-                                ggml_reshape_2d(ctx0, bm, nblk, n_tps*Hd)));       // [n_tps*Hd, nblk]
-                        ggml_tensor * bmc = ggml_get_rows(ctx0, bm2, cb_s);        // [n_tps*Hd, n_kv] F32
-                        ggml_tensor * bmx = ggml_cont(ctx0, ggml_transpose(ctx0, bmc));
+                        // expand block -> token granularity (j = bk*blk + t),
+                        // then combine with the causal mask in place
+                        ggml_tensor * bmx = ggml_repeat_4d(ctx0,
+                                ggml_reshape_3d(ctx0, bm, 1, nblk, n_tps*Hd),
+                                blk, nblk, n_tps*Hd, 1);
                         bmx = ggml_reshape_3d(ctx0, bmx, n_kv, n_tps, Hd);
-                        ggml_tensor * mask4 = ggml_add_inplace(ctx0, bmx, mf_s);
-                        mask4 = ggml_cast(ctx0,
-                                ggml_reshape_4d(ctx0, mask4, n_kv, n_tps, 1, Hd), GGML_TYPE_F16);
+                        ggml_tensor * mask4 = ggml_add_inplace(ctx0, bmx, km_s);
+                        mask4 = ggml_reshape_4d(ctx0, mask4, n_kv, n_tps, 1, Hd);
                         cb(mask4, "msa_mask4", il);
 
                         // cache views with groups on ne[3];

src/models/models.h

@@ -1084,10 +1084,6 @@ struct llama_model_deepseek2 : public llama_model_base {
         graph(const llama_model & model, const llm_graph_params & params);
     };
 
-    struct graph_mtp : public llm_graph_context {
-        graph_mtp(const llama_model & model, const llm_graph_params & params);
-    };
-
     std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
 };
 
@@ -1101,10 +1097,6 @@ struct llama_model_deepseek32 : public llama_model_base {
         graph(const llama_model & model, const llm_graph_params & params);
     };
 
-    struct graph_mtp : public llm_graph_context {
-        graph_mtp(const llama_model & model, const llm_graph_params & params);
-    };
-
     std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
 };
 
@@ -1115,7 +1107,6 @@ struct llama_model_deepseek4 : public llama_model_base {
     void load_arch_tensors(llama_model_loader & ml) override;
 
     struct graph : public llm_graph_context {
-        graph(const llm_graph_params & params) : llm_graph_context(params) {}
         graph(const llama_model & model, const llm_graph_params & params);
 
         ggml_tensor * build_hc_pre(
@@ -1147,21 +1138,6 @@ struct llama_model_deepseek4 : public llama_model_base {
                 ggml_tensor * inp_pos,
                 int il) const;
 
-        ggml_tensor * build_attention(
-                const llama_model & model,
-                llm_graph_input_attn_k_iswa * inp_mtp,
-                ggml_tensor * cur,
-                ggml_tensor * inp_pos,
-                int il) const;
-
-        ggml_tensor * build_attention_impl(
-                const llama_model & model,
-                llm_graph_input_dsv4 * inp_dsv4,
-                llm_graph_input_attn_k_iswa * inp_mtp,
-                ggml_tensor * cur,
-                ggml_tensor * inp_pos,
-                int il) const;
-
         ggml_tensor * build_hca_compressed_kv_from_state(
                 ggml_tensor * kv_state,
                 ggml_tensor * score_state,
@@ -1237,10 +1213,6 @@ struct llama_model_deepseek4 : public llama_model_base {
                 int il) const;
     };
 
-    struct graph_mtp : public graph {
-        graph_mtp(const llama_model & model, const llm_graph_params & params);
-    };
-
     std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
 };
 
@@ -1300,10 +1272,6 @@ struct llama_model_dflash : public llama_model_base {
         ggml_tensor * build_inp_embd_enc() const;
     };
 
-    struct graph_dsv4 : public llama_model_deepseek4::graph {
-        graph_dsv4(const llama_model & model, const llm_graph_params & params);
-    };
-
     std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
 };
 
@@ -2041,10 +2009,6 @@ struct llama_model_qwen3next : public llama_model_base {
         const llama_model & model;
     };
 
-    struct graph_mtp : public llm_graph_context {
-        graph_mtp(const llama_model & model, const llm_graph_params & params);
-    };
-
     std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
 };
 

src/models/qwen3next.cpp

@@ -13,11 +13,7 @@ void llama_model_qwen3next::load_arch_hparams(llama_model_loader & ml) {
     ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
     ml.get_key(LLM_KV_SSM_GROUP_COUNT,    hparams.ssm_n_group);
 
-    // NextN/MTP: extra decoder block appended beyond the main stack
-    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
-    GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all");
-
-    // Mark recurrent layers (linear attention layers).
+    // Mark recurrent layers (linear attention layers)
     if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) {
         uint32_t full_attn_interval = 4;
         ml.get_key(LLM_KV_FULL_ATTENTION_INTERVAL, full_attn_interval, false);
@@ -32,17 +28,13 @@ void llama_model_qwen3next::load_arch_hparams(llama_model_loader & ml) {
     }
 }
 
-void llama_model_qwen3next::load_arch_tensors(llama_model_loader & ml) {
+void llama_model_qwen3next::load_arch_tensors(llama_model_loader &) {
     LLAMA_LOAD_LOCALS;
 
     if (n_expert == 0) {
         throw std::runtime_error(arch_name() + " model cannot have zero experts");
     }
 
-    const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
-    const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;
-    int mtp_flags = !ml.load_mtp ? TENSOR_SKIP : 0;
-
     tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0);
 
     // output
@@ -69,73 +61,49 @@ void llama_model_qwen3next::load_arch_tensors(llama_model_loader & ml) {
     const int64_t qkvz_dim = key_dim * 2 + value_dim * 2;
     const int64_t ba_dim   = n_v_heads * 2;
 
-    auto load_block_trunk = [&](int il, int flags) {
-        auto & layer = layers[il];
-        const uint32_t n_ff_shexp = hparams.n_ff_shexp > 0 ? hparams.n_ff_shexp : hparams.n_ff(il);
+    for (int i = 0; i < n_layer; ++i) {
+        auto & layer = layers[i];
+        const uint32_t n_ff_shexp = hparams.n_ff_shexp > 0 ? hparams.n_ff_shexp : hparams.n_ff(i);
 
-        layer.attn_norm      = create_tensor(tn(LLM_TENSOR_ATTN_NORM,      "weight", il), { n_embd }, flags);
-        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", il), { n_embd }, flags);
+        layer.attn_norm      = create_tensor(tn(LLM_TENSOR_ATTN_NORM,      "weight", i), { n_embd }, 0);
+        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), { n_embd }, 0);
 
-        if (!hparams.is_recr(il)) {
+        if (!hparams.is_recr(i)) {
             // Attention layers
-            create_tensor_qkv(layer, il, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, flags);
-            layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), { n_embd_head_k * n_head, n_embd }, flags);
+            create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, 0);
+            layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, 0);
+
             // Q/K normalization for attention layers
-            layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", il), { n_embd_head_k }, flags);
-            layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", il), { n_embd_head_k }, flags);
+            layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), { n_embd_head_k }, 0);
+            layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), { n_embd_head_k }, 0);
         } else {
             // Linear attention (gated delta net) specific tensors
             // Create tensors with calculated dimensions
             // note: ssm_in is used by legacy GGUF
-            layer.ssm_in         = create_tensor(tn(LLM_TENSOR_SSM_IN,         "weight", il), { n_embd, qkvz_dim }, TENSOR_NOT_REQUIRED | flags);
-            layer.wqkv           = create_tensor(tn(LLM_TENSOR_ATTN_QKV,       "weight", il), { n_embd, key_dim * 2 + value_dim }, TENSOR_NOT_REQUIRED | flags);
-            layer.wqkv_gate      = create_tensor(tn(LLM_TENSOR_ATTN_GATE,      "weight", il), { n_embd, value_dim }, TENSOR_NOT_REQUIRED | flags);
-            layer.ssm_conv1d     = create_tensor(tn(LLM_TENSOR_SSM_CONV1D,     "weight", il), { hparams.ssm_d_conv, conv_dim }, flags);
-            layer.ssm_dt         = create_tensor(tn(LLM_TENSOR_SSM_DT,         "bias",   il), { hparams.ssm_dt_rank }, flags);
-            layer.ssm_a          = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN,             il), { hparams.ssm_dt_rank }, flags);
-            layer.ssm_beta_alpha = create_tensor(tn(LLM_TENSOR_SSM_BETA_ALPHA, "weight", il), { n_embd, ba_dim }, flags);
-            layer.ssm_norm       = create_tensor(tn(LLM_TENSOR_SSM_NORM,       "weight", il), { head_v_dim }, flags);
-            layer.ssm_out        = create_tensor(tn(LLM_TENSOR_SSM_OUT,        "weight", il), { value_dim, n_embd }, flags);
+            layer.ssm_in         = create_tensor(tn(LLM_TENSOR_SSM_IN,         "weight", i), { n_embd, qkvz_dim }, TENSOR_NOT_REQUIRED);
+            layer.wqkv           = create_tensor(tn(LLM_TENSOR_ATTN_QKV,       "weight", i), { n_embd, key_dim * 2 + value_dim }, TENSOR_NOT_REQUIRED);
+            layer.wqkv_gate      = create_tensor(tn(LLM_TENSOR_ATTN_GATE,      "weight", i), { n_embd, value_dim }, TENSOR_NOT_REQUIRED);
+            layer.ssm_conv1d     = create_tensor(tn(LLM_TENSOR_SSM_CONV1D,     "weight", i), { hparams.ssm_d_conv, conv_dim }, 0);
+            layer.ssm_dt         = create_tensor(tn(LLM_TENSOR_SSM_DT,         "bias",   i), { hparams.ssm_dt_rank }, 0);
+            layer.ssm_a          = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN,             i), { hparams.ssm_dt_rank }, 0);
+            layer.ssm_beta_alpha = create_tensor(tn(LLM_TENSOR_SSM_BETA_ALPHA, "weight", i), { n_embd, ba_dim }, 0);
+            layer.ssm_norm       = create_tensor(tn(LLM_TENSOR_SSM_NORM,       "weight", i), { head_v_dim }, 0);
+            layer.ssm_out        = create_tensor(tn(LLM_TENSOR_SSM_OUT,        "weight", i), { value_dim, n_embd }, 0);
         }
 
-        layer.ffn_gate_inp  = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,  "weight", il), { n_embd, n_expert }, flags);
-        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), { n_ff_exp, n_embd, n_expert }, flags);
-        create_tensor_gate_up_exps(layer, il, n_embd, n_ff_exp, n_expert, flags);
+        layer.ffn_gate_inp  = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,  "weight", i), { n_embd, n_expert }, 0);
+        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff_exp, n_embd, n_expert }, 0);
+        create_tensor_gate_up_exps(layer, i, n_embd, n_ff_exp, n_expert, 0);
 
         // Shared experts
-        layer.ffn_gate_inp_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP_SHEXP, "weight", il), { n_embd }, flags);
-        layer.ffn_gate_shexp     = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP,     "weight", il), { n_embd, n_ff_shexp }, flags);
-        layer.ffn_up_shexp       = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,       "weight", il), { n_embd, n_ff_shexp }, flags);
-        layer.ffn_down_shexp     = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP,     "weight", il), { n_ff_shexp, n_embd }, flags);
-    };
-
-    auto load_block_mtp = [&](int il) {
-        // MTP head is identical to the trunk block (full attention + FFN)
-        load_block_trunk(il, mtp_flags);
-
-        auto & layer = layers[il];
-
-        // NextN-specific tensors that define the MTP block.
-        layer.nextn.eh_proj          = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ,          "weight", il), { 2 * n_embd, n_embd }, mtp_flags);
-        layer.nextn.enorm            = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM,            "weight", il), { n_embd },             mtp_flags);
-        layer.nextn.hnorm            = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM,            "weight", il), { n_embd },             mtp_flags);
-        layer.nextn.embed_tokens     = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS,     "weight", il), { n_embd, n_vocab },    mtp_flags | TENSOR_NOT_REQUIRED);
-        layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", il), { n_embd, n_vocab },    mtp_flags | TENSOR_NOT_REQUIRED);
-        layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", il), { n_embd },             mtp_flags | TENSOR_NOT_REQUIRED);
-    };
-
-    for (int i = 0; i < n_layer; i++) {
-        load_block_trunk(i, trunk_flags);
-    }
-    for (int i = n_layer; i < n_layer_all; i++) {
-        load_block_mtp(i);
+        layer.ffn_gate_inp_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP_SHEXP, "weight", i), { n_embd }, 0);
+        layer.ffn_gate_shexp     = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP,     "weight", i), { n_embd, n_ff_shexp }, 0);
+        layer.ffn_up_shexp       = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,       "weight", i), { n_embd, n_ff_shexp }, 0);
+        layer.ffn_down_shexp     = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP,     "weight", i), { n_ff_shexp, n_embd }, 0);
     }
 }
 
 std::unique_ptr<llm_graph_context> llama_model_qwen3next::build_arch_graph(const llm_graph_params & params) const {
-    if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) {
-        return std::make_unique<graph_mtp>(*this, params);
-    }
     return std::make_unique<graph>(*this, params);
 }
 
@@ -152,7 +120,6 @@ llama_model_qwen3next::graph::graph(const llama_model & model, const llm_graph_p
     ggml_tensor * inp_pos     = build_inp_pos();
     ggml_tensor * inp_out_ids = build_inp_out_ids();
 
-    // MTP/NextN layers are loaded as extra decoder blocks but not executed in the main pass.
     for (int il = 0; il < n_layer; ++il) {
         res->t_layer_inp[il] = inpL;
 
@@ -172,7 +139,7 @@ llama_model_qwen3next::graph::graph(const llama_model & model, const llm_graph_p
             cur = build_layer_attn(inp->get_attn(), cur, inp_pos, il);
         }
 
-        if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) {
+        if (il == n_layer - 1 && inp_out_ids) {
             cur   = ggml_get_rows(ctx0, cur, inp_out_ids);
             inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
         }
@@ -204,16 +171,9 @@ llama_model_qwen3next::graph::graph(const llama_model & model, const llm_graph_p
     }
     cur = inpL;
 
-    // post-norm hidden state is input to both the LM head and the MTP head
+    // Final norm
     cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);
 
-    cb(cur, "h_nextn", -1);
-    res->t_h_nextn = cur;
-
-    if (!cparams.embeddings_nextn_masked && inp_out_ids) {
-        cur = ggml_get_rows(ctx0, cur, inp_out_ids);
-    }
-
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
@@ -226,6 +186,14 @@ llama_model_qwen3next::graph::graph(const llama_model & model, const llm_graph_p
     ggml_build_forward_expand(gf, cur);
 }
 
+// utility to get one slice from the third dimension
+// input dim:  [x, y, c, b]
+// output dim: [x, y, 1, b]
+static ggml_tensor * get_slice_2d(ggml_context * ctx0, ggml_tensor * t, int64_t c) {
+    return ggml_view_4d(ctx0, t, t->ne[0], t->ne[1], 1, t->ne[3],
+        t->nb[1], t->nb[2], t->nb[3], t->nb[2] * c);
+}
+
 ggml_tensor * llama_model_qwen3next::graph::build_norm_gated(
         ggml_tensor * input,
         ggml_tensor * weights,
@@ -248,7 +216,7 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_attn(
     // Order: joint QG projection, QG split, Q norm, KV projection, K norm, RoPE, attention
 
     // Qwen3Next uses a single Q projection that outputs query + gate
-    ggml_tensor * Qcur_full = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s);
+    ggml_tensor * Qcur_full = build_lora_mm(model.layers[il].wq, cur);
     cb(Qcur_full, "Qcur_full", il);
 
     Qcur_full = ggml_reshape_4d(ctx0, Qcur_full, n_embd_head * 2, n_head, n_tokens, 1);
@@ -264,10 +232,10 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_attn(
                      Qcur_full->nb[1], Qcur_full->nb[2], Qcur_full->nb[3], n_embd_head * ggml_element_size(Qcur_full));
     cb(gate, "gate", il);
 
-    ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s);
+    ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
     cb(Kcur, "Kcur", il);
 
-    ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s);
+    ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
     cb(Vcur, "Vcur", il);
 
     Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
@@ -306,6 +274,8 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_attn(
     gate = ggml_sigmoid(ctx0, gate);
     cb(gate, "gate_sigmoid", il);
 
+    gate = ggml_reshape_2d(ctx0, gate, n_embd_head * n_head, n_tokens);
+
     cur = ggml_mul(ctx0, cur, gate);
     cb(cur, "attn_gated", il);
 
@@ -580,19 +550,16 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_ffn(ggml_tensor * cur, c
                 LLM_FFN_SILU, true,
                 hparams.expert_weights_scale,
                 LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
-                nullptr, model.layers[il].ffn_gate_up_exps,
-                model.layers[il].ffn_up_exps_s,
-                model.layers[il].ffn_gate_exps_s,
-                model.layers[il].ffn_down_exps_s);
+                nullptr, model.layers[il].ffn_gate_up_exps);
         cb(moe_out, "ffn_moe_out", il);
 
         // Add shared experts if present - following Qwen3Next reference implementation
         if (model.layers[il].ffn_up_shexp != nullptr) {
             ggml_tensor * ffn_shexp =
                 build_ffn(cur,
-                    model.layers[il].ffn_up_shexp,   NULL, model.layers[il].ffn_up_shexp_s,
-                    model.layers[il].ffn_gate_shexp, NULL, model.layers[il].ffn_gate_shexp_s,
-                    model.layers[il].ffn_down_shexp, NULL, model.layers[il].ffn_down_shexp_s,
+                    model.layers[il].ffn_up_shexp,   NULL, NULL,
+                    model.layers[il].ffn_gate_shexp, NULL, NULL,
+                    model.layers[il].ffn_down_shexp, NULL, NULL,
                     NULL,
                     LLM_FFN_SILU, LLM_FFN_PAR, il);
             cb(ffn_shexp, "ffn_shexp", il);
@@ -626,198 +593,3 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_ffn(ggml_tensor * cur, c
     }
     return cur;
 }
-
-// LLM_GRAPH_TYPE_DECODER_MTP draft head for Qwen3-Next
-llama_model_qwen3next::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params)
-    : llm_graph_context(params) {
-    GGML_ASSERT(hparams.n_layer_nextn > 0 && "QWEN3NEXT MTP requires n_layer_nextn > 0");
-    GGML_ASSERT(hparams.n_layer_nextn == 1 && "QWEN3NEXT MTP currently only supports a single MTP block");
-
-    const int64_t n_embd_head = hparams.n_embd_head_v();
-    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
-
-    const int il = hparams.n_layer();
-    const auto & layer = model.layers[il];
-
-    GGML_ASSERT(layer.nextn.eh_proj    && "MTP block missing nextn.eh_proj");
-    GGML_ASSERT(layer.nextn.enorm      && "MTP block missing nextn.enorm");
-    GGML_ASSERT(layer.nextn.hnorm      && "MTP block missing nextn.hnorm");
-    GGML_ASSERT(layer.ffn_gate_inp     && "MTP block missing ffn_gate_inp");
-
-    // TODO: extract in a common llm_graph_context::build_inp_embd_h()
-    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd);
-
-    inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
-    ggml_set_input(inp->tokens);
-
-    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens);
-    ggml_set_input(inp->embd);
-
-    // TODO: make static using `ggml_build_forward_select()`
-    //       see llm_graph_context::build_inp_embd() for reference
-    ggml_tensor * tok_embd;
-    if (ubatch.token) {
-        ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;
-
-        tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens);
-    } else {
-        tok_embd = inp->embd;
-    }
-    cb(tok_embd, "mtp_tok_embd", il);
-
-    inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);
-    ggml_set_input(inp->h);
-    ggml_set_name(inp->h, "mtp_h_input");
-
-    ggml_tensor * h_embd = inp->h;
-
-    res->add_input(std::move(inp));
-
-    ggml_tensor * inp_pos     = build_inp_pos();
-    ggml_tensor * inp_out_ids = build_inp_out_ids();
-
-    auto * inp_attn = build_attn_inp_kv();
-
-    ggml_tensor * h_norm = build_norm(h_embd, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il);
-    cb(h_norm, "mtp_hnorm", il);
-
-    ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il);
-    cb(e_norm, "mtp_enorm", il);
-
-    ggml_tensor * concat = ggml_concat(ctx0, e_norm, h_norm, /*dim=*/ 0);
-    cb(concat, "mtp_concat", il);
-
-    ggml_tensor * cur = build_lora_mm(layer.nextn.eh_proj, concat, layer.nextn.eh_proj_s);
-    cb(cur, "mtp_eh_proj", il);
-
-    ggml_tensor * inpSA = cur;
-
-    cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
-    cb(cur, "mtp_attn_norm", il);
-
-    ggml_tensor * Qcur_full = build_lora_mm(layer.wq, cur, layer.wq_s);
-    cb(Qcur_full, "mtp_Qcur_full", il);
-
-    ggml_tensor * Qcur = ggml_view_3d(ctx0, Qcur_full,
-            n_embd_head, n_head, n_tokens,
-            ggml_element_size(Qcur_full) * n_embd_head * 2,
-            ggml_element_size(Qcur_full) * n_embd_head * 2 * n_head,
-            0);
-    Qcur = build_norm(Qcur, layer.attn_q_norm, nullptr, LLM_NORM_RMS, il);
-    cb(Qcur, "mtp_Qcur_normed", il);
-
-    ggml_tensor * Kcur = build_lora_mm(layer.wk, cur, layer.wk_s);
-    Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
-    Kcur = build_norm(Kcur, layer.attn_k_norm, nullptr, LLM_NORM_RMS, il);
-    cb(Kcur, "mtp_Kcur_normed", il);
-
-    ggml_tensor * Vcur = build_lora_mm(layer.wv, cur, layer.wv_s);
-    Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
-
-    Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr,
-            n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
-            ext_factor, attn_factor, beta_fast, beta_slow);
-    Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr,
-            n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
-            ext_factor, attn_factor, beta_fast, beta_slow);
-
-    cb(Qcur, "mtp_Qcur", il);
-    cb(Kcur, "mtp_Kcur", il);
-    cb(Vcur, "mtp_Vcur", il);
-
-    const float kq_scale = hparams.f_attention_scale == 0.0f
-            ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
-
-    cur = build_attn(inp_attn,
-            nullptr, nullptr, nullptr,
-            Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
-    cb(cur, "mtp_attn_pregate", il);
-
-    ggml_tensor * gate = ggml_view_3d(ctx0, Qcur_full,
-            n_embd_head, n_head, n_tokens,
-            ggml_element_size(Qcur_full) * n_embd_head * 2,
-            ggml_element_size(Qcur_full) * n_embd_head * 2 * n_head,
-            ggml_element_size(Qcur_full) * n_embd_head);
-
-    // TODO: CUDA is missing non-contiguous unary ops. when implemented: remove this cont
-    gate = ggml_cont_2d(ctx0, gate, n_embd_head * n_head, n_tokens);
-    cb(gate, "mtp_gate", il);
-
-    cur = ggml_mul(ctx0, cur, ggml_sigmoid(ctx0, gate));
-    cur = build_lora_mm(layer.wo, cur, layer.wo_s);
-    cb(cur, "mtp_attn_out", il);
-
-    if (inp_out_ids) {
-        cur   = ggml_get_rows(ctx0, cur,   inp_out_ids);
-        inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
-    }
-
-    cur = ggml_add(ctx0, cur, inpSA);
-    cb(cur, "mtp_attn_residual", il);
-
-    ggml_tensor * ffn_residual = cur;
-    cur = build_norm(cur, layer.attn_post_norm, nullptr, LLM_NORM_RMS, il);
-    cb(cur, "mtp_attn_post_norm", il);
-
-    // MoE FFN — routed experts plus gated shared expert (mirrors the trunk).
-    ggml_tensor * moe_out =
-        build_moe_ffn(cur,
-            layer.ffn_gate_inp,
-            layer.ffn_up_exps,
-            layer.ffn_gate_exps,
-            layer.ffn_down_exps,
-            nullptr,
-            n_expert, n_expert_used,
-            LLM_FFN_SILU, true,
-            hparams.expert_weights_scale,
-            LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
-            nullptr, layer.ffn_gate_up_exps,
-            layer.ffn_up_exps_s,
-            layer.ffn_gate_exps_s,
-            layer.ffn_down_exps_s);
-    cb(moe_out, "mtp_ffn_moe_out", il);
-
-    if (layer.ffn_up_shexp != nullptr) {
-        ggml_tensor * ffn_shexp =
-            build_ffn(cur,
-                layer.ffn_up_shexp,   nullptr, layer.ffn_up_shexp_s,
-                layer.ffn_gate_shexp, nullptr, layer.ffn_gate_shexp_s,
-                layer.ffn_down_shexp, nullptr, layer.ffn_down_shexp_s,
-                nullptr,
-                LLM_FFN_SILU, LLM_FFN_PAR, il);
-        cb(ffn_shexp, "mtp_ffn_shexp", il);
-
-        ggml_tensor * shared_gate = build_lora_mm(layer.ffn_gate_inp_shexp, cur);
-        shared_gate = ggml_sigmoid(ctx0, shared_gate);
-        cb(shared_gate, "mtp_shared_expert_gate_sigmoid", il);
-
-        ffn_shexp = ggml_mul(ctx0, ffn_shexp, shared_gate);
-        cb(ffn_shexp, "mtp_ffn_shexp_gated", il);
-
-        cur = ggml_add(ctx0, moe_out, ffn_shexp);
-    } else {
-        cur = moe_out;
-    }
-    cb(cur, "mtp_ffn_out", il);
-
-    cur = ggml_add(ctx0, cur, ffn_residual);
-    cb(cur, "mtp_post_ffn", il);
-
-    ggml_tensor * head_norm_w = layer.nextn.shared_head_norm
-            ? layer.nextn.shared_head_norm
-            : model.output_norm;
-    GGML_ASSERT(head_norm_w && "QWEN3NEXT MTP: missing both nextn.shared_head_norm and output_norm");
-    cur = build_norm(cur, head_norm_w, nullptr, LLM_NORM_RMS, -1);
-
-    cb(cur, "h_nextn", -1);
-    res->t_h_nextn = cur;
-
-    ggml_tensor * head_w = layer.nextn.shared_head_head ? layer.nextn.shared_head_head : model.output;
-    ggml_tensor * head_s = layer.nextn.shared_head_head ? layer.nextn.shared_head_head_s : model.output_s;
-    GGML_ASSERT(head_w && "QWEN3NEXT MTP: missing LM head (nextn.shared_head_head or model.output)");
-    cur = build_lora_mm(head_w, cur, head_s);
-    cb(cur, "result_output", -1);
-
-    res->t_logits = cur;
-    ggml_build_forward_expand(gf, cur);
-}

tests/CMakeLists.txt

@@ -258,9 +258,6 @@ llama_build_and_test(test-thread-safety.cpp ARGS -m "${MODEL_DEST}" -ngl 99 -p "
 set_tests_properties(test-thread-safety PROPERTIES FIXTURES_REQUIRED test-download-model)
 
 llama_build_and_test(test-arg-parser.cpp)
-llama_build_and_test(test-model-resolution.cpp)
-# the test serves its repos from an httplib server, and the library links it privately
-target_link_libraries(test-model-resolution PRIVATE cpp-httplib)
 
 if (NOT LLAMA_SANITIZE_ADDRESS AND NOT GGML_SCHED_NO_REALLOC)
   # TODO: repair known memory leaks

tests/test-arg-parser.cpp

@@ -99,34 +99,6 @@ static void test(void) {
     argv = {"binary_name", "-sm", "hello"};
     assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
 
-    {
-        common_params penalty_params;
-
-        argv = {"binary_name", "--repeat-penalty", "0"};
-        assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
-
-        argv = {"binary_name", "--repeat-penalty", "-1"};
-        assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
-
-        argv = {"binary_name", "--repeat-penalty", "nan"};
-        assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
-
-        argv = {"binary_name", "--repeat-penalty", "inf"};
-        assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
-
-        argv = {"binary_name", "--repeat-penalty", "-inf"};
-        assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
-
-        const char * penalty_options[] = {"--frequency-penalty", "--presence-penalty"};
-        const char * nonfinite_values[] = {"nan", "inf", "-inf"};
-        for (const char * option : penalty_options) {
-            for (const char * value : nonfinite_values) {
-                argv = {"binary_name", option, value};
-                assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
-            }
-        }
-    }
-
     // non-existence arg in specific example (--draft cannot be used outside llama-speculative)
     argv = {"binary_name", "--draft", "123"};
     assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_EMBEDDING));

tests/test-backend-ops.cpp

@@ -8069,7 +8069,6 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
     test_cases.emplace_back(new test_dsv4_hc_comb(1, 1));
     test_cases.emplace_back(new test_dsv4_hc_comb(17, 4));
     test_cases.emplace_back(new test_dsv4_hc_comb(257, 8));
-    test_cases.emplace_back(new test_dsv4_hc_comb(17, 20));
 
     test_cases.emplace_back(new test_dsv4_hc_pre(1, 1));
     test_cases.emplace_back(new test_dsv4_hc_pre(31, 17));
@@ -8079,7 +8078,6 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
     test_cases.emplace_back(new test_dsv4_hc_post(1, 1));
     test_cases.emplace_back(new test_dsv4_hc_post(31, 17));
     test_cases.emplace_back(new test_dsv4_hc_post(128, 257));
-    test_cases.emplace_back(new test_dsv4_hc_post(4096, 21));
 
     // glu ops
     for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {
@@ -8194,9 +8192,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
 
     for (ggml_type type_input : {GGML_TYPE_F32}) {
         for (ggml_op_pool pool_type : {GGML_OP_POOL_AVG, GGML_OP_POOL_MAX}) {
-            for (int k0 : {1, 2, 3}) {
-                for (int s0 : {1, 2, 3}) {
-                    for (int p0 : {0, 1, 2, 3}) {
+            for (int k0 : {1, 3}) {
+                for (int s0 : {1, 2}) {
+                    for (int p0 : {0, 1}) {
                         test_cases.emplace_back(new test_pool1d(pool_type, type_input, { 10,  3, 2, 1 }, k0, s0, p0));
                         test_cases.emplace_back(new test_pool1d(pool_type, type_input, { 11,  1, 3, 2 }, k0, s0, p0));
                         test_cases.emplace_back(new test_pool1d(pool_type, type_input, { 128, 2, 1, 3 }, k0, s0, p0));
@@ -8513,7 +8511,6 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
         test_cases.emplace_back(new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {1, 2, 1, 1}));
         test_cases.emplace_back(new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {1, 1, 2, 1}));
         test_cases.emplace_back(new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {1, 1, 1, 2}));
-        test_cases.emplace_back(new test_repeat(GGML_TYPE_F16, {10, 5, 4, ne3}, {2, 1, 1, 1}));
         test_cases.emplace_back(new test_repeat(GGML_TYPE_I32, {10, 5, 4, ne3}, {2, 1, 1, 1}));
         test_cases.emplace_back(new test_repeat(GGML_TYPE_I16, {10, 5, 4, ne3}, {1, 1, 1, 2}));
         test_cases.emplace_back(new test_repeat(GGML_TYPE_BF16, {10, 5, 4, ne3}, {2, 1, 1, 1}));
@@ -9731,12 +9728,6 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
         }
     }
 
-    for (int kv : { 1, 7, 8, 63, 64, 65 }) {
-        for (ggml_type type_K : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0}) {
-            test_cases.emplace_back(new test_lightning_indexer(128, 64, kv, 32, 4, 1, type_K));
-        }
-    }
-
     return test_cases;
 }
 #ifdef _MSC_VER

tests/test-backend-sampler.cpp

@@ -8,15 +8,12 @@
 #endif
 
 #include <algorithm>
-#include <cmath>
 #include <cstdlib>
 #include <cstring>
 #include <fstream>
-#include <functional>
 #include <map>
 #include <string>
 #include <unordered_map>
-#include <unordered_set>
 #include <vector>
 
 struct test_args {
@@ -764,564 +761,6 @@ static void test_backend_logit_bias_sampling(const test_params & params) {
     printf("backend logit bias sampling test PASSED\n");
 }
 
-static void accept_prompt(llama_sampler * smpl, const llama_vocab * vocab, const std::string & prompt) {
-    const llama_token bos = llama_vocab_bos(vocab);
-    if (bos != LLAMA_TOKEN_NULL) {
-        llama_sampler_accept(smpl, bos);
-    }
-
-    std::vector<llama_token> tokens(64);
-    int32_t n_tokens = llama_tokenize(vocab, prompt.c_str(), (int32_t) prompt.size(),
-            tokens.data(), (int32_t) tokens.size(), false, false);
-    if (n_tokens < 0) {
-        tokens.resize(-n_tokens);
-        n_tokens = llama_tokenize(vocab, prompt.c_str(), (int32_t) prompt.size(),
-                tokens.data(), (int32_t) tokens.size(), false, false);
-    }
-
-    for (int32_t i = 0; i < n_tokens; ++i) {
-        llama_sampler_accept(smpl, tokens[i]);
-    }
-}
-
-static std::vector<float> decode_raw_logits(const test_params & params, const std::string & prompt) {
-    const int seq_id = 0;
-    const int n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(params.model.get()));
-    std::vector<llama_sampler_seq_config> empty_configs;
-    test_context ctx(params, empty_configs);
-
-    GGML_ASSERT(ctx.decode({{ seq_id, prompt }}));
-
-    float * logits = llama_get_logits_ith(ctx.ctx.get(), ctx.idx_for_seq(seq_id));
-    GGML_ASSERT(logits != nullptr);
-    return std::vector<float>(logits, logits + n_vocab);
-}
-
-static std::vector<llama_token_data> apply_cpu_sampler(
-        const std::vector<float> & raw_logits,
-        llama_sampler * sampler) {
-    std::vector<llama_token_data> data;
-    data.reserve(raw_logits.size());
-    for (llama_token token = 0; token < (llama_token) raw_logits.size(); ++token) {
-        data.push_back({ token, raw_logits[token], 0.0f });
-    }
-
-    llama_token_data_array cur_p = { data.data(), data.size(), -1, false };
-    llama_sampler_apply(sampler, &cur_p);
-    data.resize(cur_p.size);
-    return data;
-}
-
-using sampler_setup_fn = std::function<void(llama_sampler *)>;
-using sampler_init_fn = std::function<llama_sampler *()>;
-
-enum class penalties_position {
-    before_filter,
-    after_filter,
-};
-
-static void add_filter_and_penalties(
-        llama_sampler * chain,
-        const sampler_init_fn & init_filter,
-        int32_t n_vocab,
-        int32_t penalty_last_n,
-        float penalty_repeat,
-        float penalty_freq,
-        float penalty_present,
-        penalties_position position) {
-    const auto add_penalties = [&]() {
-        llama_sampler_chain_add(chain, llama_sampler_init_penalties(
-                    n_vocab, penalty_last_n, penalty_repeat, penalty_freq, penalty_present));
-    };
-
-    if (position == penalties_position::before_filter) {
-        add_penalties();
-        llama_sampler_chain_add(chain, init_filter());
-    } else {
-        llama_sampler_chain_add(chain, init_filter());
-        add_penalties();
-    }
-}
-
-static llama_sampler_ptr make_sampler_chain(
-        const sampler_setup_fn & add_samplers,
-        const sampler_setup_fn & accept_history) {
-    llama_sampler_ptr chain(llama_sampler_chain_init(llama_sampler_chain_default_params()));
-    add_samplers(chain.get());
-    accept_history(chain.get());
-    return chain;
-}
-
-struct backend_sampler_output {
-    std::vector<float> logits;
-    std::vector<llama_token> candidates;
-};
-
-static backend_sampler_output run_backend_sampler(
-        const test_params & params,
-        const std::string & prompt,
-        llama_sampler * sampler) {
-    const int seq_id = 0;
-    std::vector<llama_sampler_seq_config> configs = {{ seq_id, sampler }};
-    test_context ctx(params, configs);
-
-    GGML_ASSERT(ctx.decode({{ seq_id, prompt }}));
-    llama_synchronize(ctx.ctx.get());
-
-    const int32_t idx = ctx.idx_for_seq(seq_id);
-    const uint32_t n_logits = llama_get_sampled_logits_count_ith(ctx.ctx.get(), idx);
-    const uint32_t n_candidates = llama_get_sampled_candidates_count_ith(ctx.ctx.get(), idx);
-    float * logits = llama_get_sampled_logits_ith(ctx.ctx.get(), idx);
-    llama_token * candidates = llama_get_sampled_candidates_ith(ctx.ctx.get(), idx);
-    GGML_ASSERT(logits != nullptr);
-
-    backend_sampler_output result;
-    result.logits.assign(logits, logits + n_logits);
-    result.candidates.resize(n_logits);
-
-    if (n_candidates == 0) {
-        for (uint32_t i = 0; i < n_logits; ++i) {
-            result.candidates[i] = (llama_token) i;
-        }
-    } else {
-        GGML_ASSERT(candidates != nullptr);
-        GGML_ASSERT(n_candidates == n_logits);
-        std::memcpy(result.candidates.data(), candidates, n_candidates * sizeof(llama_token));
-    }
-
-    return result;
-}
-
-struct sampler_comparison_output {
-    std::vector<llama_token_data> expected;
-    backend_sampler_output actual;
-};
-
-static sampler_comparison_output run_sampler_comparison(
-        const test_params & params,
-        const std::string & prompt,
-        const std::vector<float> & raw_logits,
-        const sampler_setup_fn & add_samplers,
-        const sampler_setup_fn & accept_history) {
-    llama_sampler_ptr cpu_chain = make_sampler_chain(add_samplers, accept_history);
-    llama_sampler_ptr backend_chain = make_sampler_chain(add_samplers, accept_history);
-    return {
-        apply_cpu_sampler(raw_logits, cpu_chain.get()),
-        run_backend_sampler(params, prompt, backend_chain.get()),
-    };
-}
-
-static std::unordered_map<llama_token, float> map_logits(const std::vector<llama_token_data> & data) {
-    std::unordered_map<llama_token, float> result;
-    result.reserve(data.size());
-    for (const auto & item : data) {
-        result[item.id] = item.logit;
-    }
-    return result;
-}
-
-struct sampler_comparison_stats {
-    int n_mismatch = 0;
-    int n_masked = 0;
-    float max_diff = 0.0f;
-};
-
-static sampler_comparison_stats compare_sampler_outputs(
-        const char * name,
-        const std::unordered_map<llama_token, float> & expected,
-        const backend_sampler_output & actual,
-        bool allow_extra_candidates = false) {
-    GGML_ASSERT(actual.logits.size() == actual.candidates.size());
-
-    sampler_comparison_stats result;
-    std::unordered_set<llama_token> seen;
-    seen.reserve(actual.candidates.size());
-
-    for (size_t i = 0; i < actual.logits.size(); ++i) {
-        const llama_token token = actual.candidates[i];
-        const float logit = actual.logits[i];
-        if (!seen.insert(token).second || std::isnan(logit)) {
-            if (result.n_mismatch < 5) {
-                printf("%s token %d has invalid backend output\n", name, token);
-            }
-            ++result.n_mismatch;
-            continue;
-        }
-
-        const auto it = expected.find(token);
-        if (it == expected.end()) {
-            if (std::isinf(logit) && logit < 0.0f) {
-                ++result.n_masked;
-            } else if (!allow_extra_candidates) {
-                if (result.n_mismatch < 5) {
-                    printf("%s token %d was not masked\n", name, token);
-                }
-                ++result.n_mismatch;
-            }
-            continue;
-        }
-
-        const float diff = fabsf(it->second - logit);
-        result.max_diff = std::max(result.max_diff, diff);
-        if (!std::isfinite(logit) || diff > 1e-3f) {
-            if (result.n_mismatch < 5) {
-                printf("%s mismatch token %d: cpu=%.6f backend=%.6f diff=%.6f\n",
-                        name, token, it->second, logit, diff);
-            }
-            ++result.n_mismatch;
-        }
-    }
-
-    for (const auto & item : expected) {
-        if (seen.find(item.first) == seen.end()) {
-            if (result.n_mismatch < 5) {
-                printf("%s missing backend token %d\n", name, item.first);
-            }
-            ++result.n_mismatch;
-        }
-    }
-
-    printf("%s logits: max_diff=%.6f n_masked=%d n_mismatch=%d\n",
-            name, result.max_diff, result.n_masked, result.n_mismatch);
-    return result;
-}
-
-static float find_backend_logit(const backend_sampler_output & output, llama_token token) {
-    for (size_t i = 0; i < output.candidates.size(); ++i) {
-        if (output.candidates[i] == token) {
-            return output.logits[i];
-        }
-    }
-    GGML_ABORT("backend token not found");
-}
-
-static sampler_comparison_output run_penalties_comparison(
-        const test_params & params,
-        int32_t penalty_last_n,
-        float penalty_repeat,
-        float penalty_freq,
-        float penalty_present,
-        const std::string & prompt,
-        const std::function<void(llama_sampler *)> & extra_accept = {}) {
-    const auto * vocab = llama_model_get_vocab(params.model.get());
-    const std::vector<float> raw_logits = decode_raw_logits(params, prompt);
-    const auto add_samplers = [&](llama_sampler * chain) {
-        llama_sampler_chain_add(chain, llama_sampler_init_penalties(
-                    llama_vocab_n_tokens(vocab), penalty_last_n, penalty_repeat, penalty_freq, penalty_present));
-    };
-    const auto accept_history = [&](llama_sampler * chain) {
-        accept_prompt(chain, vocab, prompt);
-        if (extra_accept) {
-            extra_accept(chain);
-        }
-    };
-
-    return run_sampler_comparison(
-            params, prompt, raw_logits, add_samplers, accept_history);
-}
-
-static void compare_penalties_logits(
-        const test_params & params,
-        int32_t penalty_last_n,
-        float penalty_repeat,
-        float penalty_freq,
-        float penalty_present,
-        const std::string & prompt,
-        const std::function<void(llama_sampler *)> & extra_accept = {}) {
-    const sampler_comparison_output output = run_penalties_comparison(
-            params, penalty_last_n, penalty_repeat, penalty_freq, penalty_present, prompt, extra_accept);
-
-    GGML_ASSERT(output.expected.size() == output.actual.logits.size());
-
-    const sampler_comparison_stats stats = compare_sampler_outputs(
-            "penalties", map_logits(output.expected), output.actual);
-    GGML_ASSERT(stats.n_masked == 0);
-    GGML_ASSERT(stats.n_mismatch == 0);
-}
-
-static void test_penalty_parameter_values(const test_params & params) {
-    struct penalty_test_case {
-        const char * name;
-        float repeat;
-        float frequency;
-        float presence;
-    };
-
-    const penalty_test_case cases[] = {
-        { "frequency -1",   1.0f, -1.0f,     0.0f },
-        { "frequency 0",    1.0f,  0.0f,     0.0f },
-        { "frequency 1",    1.0f,  1.0f,     0.0f },
-        { "presence -1",    1.0f,  0.0f,    -1.0f },
-        { "presence 0",     1.0f,  0.0f,     0.0f },
-        { "presence 1",     1.0f,  0.0f,     1.0f },
-        { "repeat 1",       1.0f,  0.0f,     0.0f },
-    };
-
-    int n_failed = 0;
-    for (const auto & test : cases) {
-        const sampler_comparison_output output = run_penalties_comparison(
-                params, 64, test.repeat, test.frequency, test.presence, "Hello Hello world");
-        GGML_ASSERT(output.expected.size() == output.actual.logits.size());
-        const sampler_comparison_stats stats = compare_sampler_outputs(
-                test.name, map_logits(output.expected), output.actual);
-        n_failed += stats.n_mismatch != 0;
-    }
-
-    GGML_ASSERT(n_failed == 0);
-}
-
-static void compare_top_k_penalties_logits(
-        const test_params & params,
-        int32_t k,
-        int32_t penalty_last_n,
-        float penalty_repeat,
-        float penalty_freq,
-        float penalty_present,
-        const std::string & prompt,
-        penalties_position position) {
-    const auto * vocab = llama_model_get_vocab(params.model.get());
-    const std::vector<float> raw_logits = decode_raw_logits(params, prompt);
-    const int n_vocab = (int) raw_logits.size();
-
-    GGML_ASSERT(n_vocab > k);
-
-    const sampler_init_fn init_top_k = [k]() {
-        return llama_sampler_init_top_k(k);
-    };
-    llama_sampler_ptr top_k(init_top_k());
-    const std::vector<llama_token_data> top_k_data = apply_cpu_sampler(raw_logits, top_k.get());
-    GGML_ASSERT(top_k_data.size() == (size_t) k);
-    const llama_token retained_history_token = top_k_data[0].id;
-
-    llama_token excluded_history_token = LLAMA_TOKEN_NULL;
-    for (llama_token token = 0; token < n_vocab; ++token) {
-        const auto it = std::find_if(top_k_data.begin(), top_k_data.end(), [token](const llama_token_data & data) {
-            return data.id == token;
-        });
-        if (it == top_k_data.end()) {
-            excluded_history_token = token;
-            break;
-        }
-    }
-    GGML_ASSERT(excluded_history_token != LLAMA_TOKEN_NULL);
-
-    const auto add_samplers = [&](llama_sampler * chain) {
-        add_filter_and_penalties(chain, init_top_k, n_vocab,
-                penalty_last_n, penalty_repeat, penalty_freq, penalty_present, position);
-    };
-
-    auto accept_history = [&](llama_sampler * smpl) {
-        accept_prompt(smpl, vocab, prompt);
-        llama_sampler_accept(smpl, excluded_history_token);
-        llama_sampler_accept(smpl, excluded_history_token);
-        llama_sampler_accept(smpl, retained_history_token);
-        llama_sampler_accept(smpl, retained_history_token);
-    };
-
-    const sampler_comparison_output output = run_sampler_comparison(
-            params, prompt, raw_logits, add_samplers, accept_history);
-
-    GGML_ASSERT(output.expected.size() == (size_t) k);
-    GGML_ASSERT(output.actual.logits.size() == (size_t) k);
-
-    const std::unordered_map<llama_token, float> expected_logits = map_logits(output.expected);
-
-    if (position == penalties_position::after_filter) {
-        GGML_ASSERT(expected_logits.find(retained_history_token) != expected_logits.end());
-        GGML_ASSERT(fabsf(expected_logits.at(retained_history_token) - raw_logits[retained_history_token]) > 1e-6f);
-        GGML_ASSERT(expected_logits.find(excluded_history_token) == expected_logits.end());
-        GGML_ASSERT(std::find(output.actual.candidates.begin(), output.actual.candidates.end(),
-                    excluded_history_token) == output.actual.candidates.end());
-    } else {
-        const std::unordered_map<llama_token, float> unpenalized_logits = map_logits(top_k_data);
-        bool changed = false;
-        for (const auto & item : expected_logits) {
-            const auto it = unpenalized_logits.find(item.first);
-            if (it == unpenalized_logits.end() || fabsf(it->second - item.second) > 1e-6f) {
-                changed = true;
-                break;
-            }
-        }
-        GGML_ASSERT(changed);
-    }
-
-    const char * name = position == penalties_position::before_filter
-        ? "penalties top-k"
-        : "top-k penalties";
-    const sampler_comparison_stats stats = compare_sampler_outputs(
-            name, expected_logits, output.actual);
-    GGML_ASSERT(stats.n_masked == 0);
-    GGML_ASSERT(stats.n_mismatch == 0);
-}
-
-static void compare_masking_penalties_logits(
-        const test_params & params,
-        const char * filter_name,
-        const sampler_init_fn & init_filter,
-        int32_t penalty_last_n,
-        float penalty_repeat,
-        float penalty_freq,
-        float penalty_present,
-        const std::string & prompt,
-        penalties_position position,
-        bool allow_extra_candidates,
-        bool add_history = true) {
-    const auto * vocab = llama_model_get_vocab(params.model.get());
-    const std::vector<float> raw_logits = decode_raw_logits(params, prompt);
-    const int n_vocab = (int) raw_logits.size();
-    llama_sampler_ptr filter(init_filter());
-    const std::vector<llama_token_data> filtered_data = apply_cpu_sampler(raw_logits, filter.get());
-    GGML_ASSERT(!filtered_data.empty());
-    GGML_ASSERT(filtered_data.size() < (size_t) n_vocab);
-
-    const llama_token penalized_token = filtered_data[0].id;
-    std::unordered_set<llama_token> retained_tokens;
-    retained_tokens.reserve(filtered_data.size());
-    for (const auto & data : filtered_data) {
-        retained_tokens.insert(data.id);
-    }
-
-    llama_token masked_token = LLAMA_TOKEN_NULL;
-    for (llama_token token = 0; token < n_vocab; ++token) {
-        if (retained_tokens.find(token) == retained_tokens.end()) {
-            masked_token = token;
-            break;
-        }
-    }
-    GGML_ASSERT(masked_token != LLAMA_TOKEN_NULL);
-
-    const auto add_samplers = [&](llama_sampler * chain) {
-        add_filter_and_penalties(chain, init_filter, n_vocab,
-                penalty_last_n, penalty_repeat, penalty_freq, penalty_present, position);
-    };
-    auto accept_history = [&](llama_sampler * smpl) {
-        if (!add_history) {
-            return;
-        }
-        accept_prompt(smpl, vocab, prompt);
-        llama_sampler_accept(smpl, penalized_token);
-        llama_sampler_accept(smpl, penalized_token);
-        llama_sampler_accept(smpl, masked_token);
-        llama_sampler_accept(smpl, masked_token);
-    };
-
-    const sampler_comparison_output output = run_sampler_comparison(
-            params, prompt, raw_logits, add_samplers, accept_history);
-
-    GGML_ASSERT(output.actual.logits.size() == (size_t) n_vocab);
-
-    const std::unordered_map<llama_token, float> expected_logits = map_logits(output.expected);
-
-    GGML_ASSERT(expected_logits.find(masked_token) == expected_logits.end());
-    if (add_history) {
-        if (position == penalties_position::after_filter) {
-            GGML_ASSERT(expected_logits.find(penalized_token) != expected_logits.end());
-            GGML_ASSERT(fabsf(expected_logits.at(penalized_token) - raw_logits[penalized_token]) > 1e-6f);
-        } else {
-            llama_sampler_ptr penalties(llama_sampler_init_penalties(
-                        n_vocab, penalty_last_n, penalty_repeat, penalty_freq, penalty_present));
-            accept_history(penalties.get());
-            const std::unordered_map<llama_token, float> penalized_logits =
-                map_logits(apply_cpu_sampler(raw_logits, penalties.get()));
-            GGML_ASSERT(fabsf(penalized_logits.at(penalized_token) - raw_logits[penalized_token]) > 1e-6f);
-        }
-    }
-
-    const std::string name = position == penalties_position::before_filter
-        ? "penalties " + std::string(filter_name)
-        : std::string(filter_name) + " penalties";
-    const sampler_comparison_stats stats = compare_sampler_outputs(
-            name.c_str(), expected_logits, output.actual, allow_extra_candidates);
-    const float masked_logit = find_backend_logit(output.actual, masked_token);
-    GGML_ASSERT(stats.n_masked > 0);
-    GGML_ASSERT(std::isinf(masked_logit) && masked_logit < 0.0f);
-    GGML_ASSERT(stats.n_mismatch == 0);
-}
-
-static void test_backend_penalties_sampling(const test_params & params) {
-    printf("Testing backend penalties (repeat + freq + presence)\n");
-    compare_penalties_logits(params, 64, 1.1f, 0.5f, 0.25f, "Hello Hello world");
-
-    printf("Testing backend penalties with penalty_last_n > 64\n");
-    const auto * vocab = llama_model_get_vocab(params.model.get());
-    std::vector<llama_token> tokens(8);
-    int32_t n_tok = llama_tokenize(vocab, "a", 1, tokens.data(), (int32_t) tokens.size(), false, false);
-    if (n_tok < 0) {
-        tokens.resize(-n_tok);
-        n_tok = llama_tokenize(vocab, "a", 1, tokens.data(), (int32_t) tokens.size(), false, false);
-    }
-    GGML_ASSERT(n_tok > 0);
-    const llama_token tok = tokens[0];
-
-    compare_penalties_logits(params, 80, 1.15f, 0.1f, 0.05f, "a", [tok](llama_sampler * smpl) {
-        // accept_prompt already accepted BOS + one 'a'; fill the ring to n=80
-        for (int i = 0; i < 78; ++i) {
-            llama_sampler_accept(smpl, tok);
-        }
-    });
-
-    printf("Testing backend penalties without filler entries\n");
-    compare_penalties_logits(params, 64, 1.1f, 0.5f, 0.25f, "Hello", [](llama_sampler * smpl) {
-        for (llama_token token = 0; token < 64; ++token) {
-            llama_sampler_accept(smpl, token);
-        }
-    });
-
-    printf("Testing backend top-k followed by penalties\n");
-    compare_top_k_penalties_logits(params, 8, 64, 1.1f, 0.5f, 0.25f, "Hello",
-            penalties_position::after_filter);
-
-    printf("Testing backend penalties followed by top-k\n");
-    compare_top_k_penalties_logits(params, 8, 64, 1.1f, 0.5f, 0.25f, "Hello",
-            penalties_position::before_filter);
-
-    printf("Testing backend top-p followed by penalties\n");
-    compare_masking_penalties_logits(params, "top-p", []() {
-        return llama_sampler_init_top_p(0.9f, 0);
-    }, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::after_filter, true);
-
-    printf("Testing backend top-p followed by penalties with a large history window\n");
-    compare_masking_penalties_logits(params, "top-p large-window", []() {
-        return llama_sampler_init_top_p(0.9f, 0);
-    }, 4096, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::after_filter, true);
-
-    printf("Testing backend penalties followed by top-p\n");
-    compare_masking_penalties_logits(params, "top-p", []() {
-        return llama_sampler_init_top_p(0.9f, 0);
-    }, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::before_filter, true);
-
-    printf("Testing backend min-p followed by penalties\n");
-    compare_masking_penalties_logits(params, "min-p", []() {
-        return llama_sampler_init_min_p(0.1f, 0);
-    }, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::after_filter, false);
-
-    printf("Testing backend penalties followed by min-p\n");
-    compare_masking_penalties_logits(params, "min-p", []() {
-        return llama_sampler_init_min_p(0.1f, 0);
-    }, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::before_filter, false);
-
-    printf("Testing backend top-p followed by penalties with empty history\n");
-    compare_masking_penalties_logits(params, "top-p empty", []() {
-        return llama_sampler_init_top_p(0.9f, 0);
-    }, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::after_filter, true, false);
-
-    printf("Testing backend top-p followed by individual penalties\n");
-    compare_masking_penalties_logits(params, "top-p repeat", []() {
-        return llama_sampler_init_top_p(0.9f, 0);
-    }, 64, 1.1f, 0.0f, 0.0f, "Hello", penalties_position::after_filter, true);
-    compare_masking_penalties_logits(params, "top-p frequency", []() {
-        return llama_sampler_init_top_p(0.9f, 0);
-    }, 64, 1.0f, 0.5f, 0.0f, "Hello", penalties_position::after_filter, true);
-    compare_masking_penalties_logits(params, "top-p presence", []() {
-        return llama_sampler_init_top_p(0.9f, 0);
-    }, 64, 1.0f, 0.0f, 0.25f, "Hello", penalties_position::after_filter, true);
-
-    printf("Testing backend penalty parameter values\n");
-    test_penalty_parameter_values(params);
-
-    printf("backend penalties sampling test PASSED\n");
-}
-
 // This test verifies that it is possible to have two different backend samplers,
 // one that uses the backend dist sampler, and another that uses CPU dist sampler.
 static void test_backend_mixed_sampling(const test_params & params) {
@@ -1575,7 +1014,6 @@ struct backend_test_case {
 static const backend_test_case BACKEND_TESTS[] = {
     { "greedy",          test_backend_greedy_sampling,         true  },
     { "logit_bias",      test_backend_logit_bias_sampling,     true  },
-    { "penalties",       test_backend_penalties_sampling,      true  },
     { "temp",            test_backend_temp_sampling,           true  },
     { "temp_ext",        test_backend_temp_ext_sampling,       true  },
     { "top_k",           test_backend_top_k_sampling,          true  },

tests/test-chat-peg-parser.cpp

@@ -8,7 +8,6 @@
 
 #include <iostream>
 #include <numeric>
-#include <regex>
 #include <string>
 
 #include "nlohmann/json.hpp"
@@ -22,7 +21,6 @@ static void test_example_qwen3_non_coder(testing & t);
 static void test_command7_parser_compare(testing & t);
 static void test_prefix_tool_names(testing & t);
 static void test_tagged_peg_parser(testing & t);
-static void test_permute(testing & t);
 
 int main(int argc, char * argv[]) {
     testing t(std::cout);
@@ -41,7 +39,6 @@ int main(int argc, char * argv[]) {
     t.test("comparison", test_command7_parser_compare);
     t.test("prefix tool names", test_prefix_tool_names);
     t.test("tagged peg parser", test_tagged_peg_parser);
-    t.test("permute", test_permute);
 
     return t.summary();
 }
@@ -984,103 +981,3 @@ static void test_tagged_peg_parser(testing & t) {
         t.assert_equal("fun_post should be '>'", ">", result.tags["fun_post"]);
     });
 }
-
-static void test_permute(testing & t) {
-    auto accepts = [](const common_peg_arena & parser, const std::string & input) {
-        common_peg_parse_context ctx(input);
-        return parser.parse(ctx).success();
-    };
-
-    auto gbnf_of = [](const common_peg_arena & parser) {
-        return build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); });
-    };
-
-    auto assert_gbnf_equal = [](testing & t, const std::string & expected, const std::string & actual) {
-        static const std::regex leading_ws_re = std::regex(R"((^|\n)\s+)");
-        t.assert_equal("gbnf are equal", std::regex_replace(expected, leading_ws_re, "$1"), actual);
-    };
-
-    auto count_rules = [](const std::string & gbnf, const std::string & prefix) {
-        size_t count = 0;
-        for (const auto & line : string_split<std::string>(gbnf, '\n')) {
-            if (line.rfind(prefix, 0) == 0) {
-                count++;
-            }
-        }
-        return count;
-    };
-
-    t.test("accepts every ordering", [&](testing & t) {
-        auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) {
-            return p.permute("abc", { p.literal("a"), p.literal("b"), p.literal("c") }) + p.end();
-        });
-
-        for (const std::string input : { "abc", "acb", "bac", "bca", "cab", "cba" }) {
-            t.assert_true("accepts " + input, accepts(parser, input));
-        }
-    });
-
-    t.test("single element", [&](testing & t) {
-        auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) {
-            return p.permute("a", { p.literal("a") }) + p.end();
-        });
-
-        t.assert_true("accepts a", accepts(parser, "a"));
-        t.assert_true("rejects aa", !accepts(parser, "aa"));
-    });
-
-    t.test("grammar left-factorizes shared tails", [&](testing & t) {
-        auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) {
-            return p.permute("abc", { p.literal("a"), p.literal("b"), p.literal("c") }) + p.end();
-        });
-
-        // Every rule is one remaining subset, keyed by bitmask: abc-3 is {a,b}, abc-7 is {a,b,c}.
-        // Each subset is emitted once and shared by every branch that leads into it.
-        assert_gbnf_equal(t, R"""(
-            abc-1 ::= "a"
-            abc-2 ::= "b"
-            abc-3 ::= "a" abc-2 | "b" abc-1
-            abc-4 ::= "c"
-            abc-5 ::= "a" abc-4 | "c" abc-1
-            abc-6 ::= "b" abc-4 | "c" abc-2
-            abc-7 ::= "a" abc-6 | "b" abc-5 | "c" abc-3
-            root ::= abc-7
-            space ::= | " " | "\n"{1,2} [ \t]{0,20}
-        )""", gbnf_of(parser));
-    });
-
-    t.test("grammar emits one rule per remaining subset", [&](testing & t) {
-        auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) {
-            return p.permute("abcd", { p.literal("a"), p.literal("b"), p.literal("c"), p.literal("d") }) + p.end();
-        });
-
-        // 2^4 - 1 non-empty subsets, one rule each - not the 4! = 24 orderings.
-        t.assert_equal("permute rule count", 15u, count_rules(gbnf_of(parser), "abcd-"));
-    });
-
-    t.test("grammar emits no rules for a single element", [&](testing & t) {
-        auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) {
-            return p.permute("a", { p.literal("a") }) + p.end();
-        });
-
-        assert_gbnf_equal(t, R"""(
-            root ::= "a"
-            space ::= | " " | "\n"{1,2} [ \t]{0,20}
-        )""", gbnf_of(parser));
-    });
-
-    t.test("grammar falls back to the given order when too large", [&](testing & t) {
-        auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) {
-            std::vector<common_peg_parser> parsers;
-            for (size_t i = 0; i <= COMMON_CHAT_MAX_PERMUTE; i++) {
-                parsers.push_back(p.literal(std::string(1, (char) ('a' + i))));
-            }
-            return p.permute("big", parsers) + p.end();
-        });
-
-        assert_gbnf_equal(t, R"""(
-            root ::= "a" "b" "c" "d" "e" "f" "g"
-            space ::= | " " | "\n"{1,2} [ \t]{0,20}
-        )""", gbnf_of(parser));
-    });
-}

tests/test-chat.cpp

@@ -2278,39 +2278,46 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
             .expect_content(R"({"amount": 123.45, "date": "2025-12-03"})")
             .run();
 
-        // a tool call ends the prefilled thinking block, with or without a closing </think>
+        // tool call segment in reasoning
         tst.test(
+               "Let's call a tool: <tool_call>\n"
+               "<function=python>\n"
+               "<parameter=code>\n"
+               "def hello():\n"
+               "    print(\"Not the real call!\")\n"
+               "\n"
+               "hello()\n"
+               "</parameter>\n"
+               "</function>\n"
+               "</tool_call>\n</think>\n\n"
                "<tool_call>\n"
-               "<function=run_in_terminal>\n"
-               "<parameter=command>\n"
-               "pwd\n"
+               "<function=python>\n"
+               "<parameter=code>\n"
+               "def hello():\n"
+               "    print(\"Hello, world!\")\n"
+               "\n"
+               "hello()\n"
                "</parameter>\n"
                "</function>\n"
                "</tool_call>")
             .enable_thinking(true)
             .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
-            .tools({ run_in_terminal_tool })
+            .tools({
+                python_tool
+        })
+            .expect_reasoning(
+                "Let's call a tool: <tool_call>\n"
+                "<function=python>\n"
+                "<parameter=code>\n"
+                "def hello():\n"
+                "    print(\"Not the real call!\")\n"
+                "\n"
+                "hello()\n"
+                "</parameter>\n"
+                "</function>\n"
+                "</tool_call>")
             .expect_tool_calls({
-                { "run_in_terminal", R"({"command": "pwd"})", {} },
-            })
-            .run();
-
-        // ...including after the model has thought about it
-        tst.test(
-               "Need to inspect the current directory.\n"
-               "<tool_call>\n"
-               "<function=run_in_terminal>\n"
-               "<parameter=command>\n"
-               "pwd\n"
-               "</parameter>\n"
-               "</function>\n"
-               "</tool_call>")
-            .enable_thinking(true)
-            .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
-            .tools({ run_in_terminal_tool })
-            .expect_reasoning("Need to inspect the current directory.")
-            .expect_tool_calls({
-                { "run_in_terminal", R"({"command": "pwd"})", {} },
+                { "python", "{\"code\": \"def hello():\\n    print(\\\"Hello, world!\\\")\\n\\nhello()\"}", {} },
             })
             .run();
 
@@ -2454,6 +2461,17 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
             })
             .run();
 
+        tst.test(
+               "I might call <tool_call> later, but I am still thinking.\n"
+               "</think>\n\n"
+               "Final answer without tools.")
+            .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+            .enable_thinking(true)
+            .tools({ run_in_terminal_tool })
+            .expect_reasoning("I might call <tool_call> later, but I am still thinking.")
+            .expect_content("Final answer without tools.")
+            .run();
+
         // Continuation tests
         tst.test("world!\nWhat's up?")
             .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
@@ -2758,6 +2776,49 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
             .expect_content(R"({"amount": 123.45, "date": "2025-12-03"})")
             .run();
 
+        // tool call segment in reasoning
+        tst.test(
+               "Let's call a tool: <tool_call>\n"
+               "<function=python>\n"
+               "<parameter=code>\n"
+               "def hello():\n"
+               "    print(\"Not the real call!\")\n"
+               "\n"
+               "hello()\n"
+               "</parameter>\n"
+               "</function>\n"
+               "</tool_call>\n</think>\n"
+               "<tool_call>\n"
+               "<function=python>\n"
+               "<parameter=code>\n"
+               "def hello():\n"
+               "    print(\"Hello, world!\")\n"
+               "\n"
+               "hello()\n"
+               "</parameter>\n"
+               "</function>\n"
+               "</tool_call>\n"
+            )
+            .enable_thinking(true)
+            .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+            .tools({
+                python_tool
+        })
+            .expect_reasoning("Let's call a tool: <tool_call>\n"
+               "<function=python>\n"
+               "<parameter=code>\n"
+               "def hello():\n"
+               "    print(\"Not the real call!\")\n"
+               "\n"
+               "hello()\n"
+               "</parameter>\n"
+               "</function>\n"
+               "</tool_call>\n")
+            .expect_tool_calls({
+                { "python", "{\"code\": \"def hello():\\n    print(\\\"Hello, world!\\\")\\n\\nhello()\"}", {} },
+            })
+            .run();
+
         // Continuation tests
         tst.test("world!\nWhat's up?")
             .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
@@ -3511,61 +3572,6 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
             .expect_reconstruction()
             .run();
 
-        // Some models skip the opening <tool_call> and go straight to <function=>
-        tst.test(
-               "<function=special_function>\n"
-               "<parameter=arg1>\n"
-               "1\n"
-               "</parameter>\n"
-               "</function>\n"
-               "</tool_call>")
-            .tools({ special_function_tool })
-            .expect(message_assist_call)
-            .run();
-
-        tst.test(
-               "Let me call it.\n"
-               "<function=special_function>\n"
-               "<parameter=arg1>\n"
-               "1\n"
-               "</parameter>\n"
-               "</function>\n"
-               "</tool_call>")
-            .tools({ special_function_tool })
-            .expect_content("Let me call it.\n")
-            .expect_tool_calls({
-                { "special_function", R"({"arg1": 1})", {} },
-            })
-            .run();
-
-        // Only the first call may omit it, the rest keep the </tool_call>\n<tool_call> separator
-        tst.test(
-               "<function=special_function>\n"
-               "<parameter=arg1>\n"
-               "1\n"
-               "</parameter>\n"
-               "</function>\n"
-               "</tool_call>\n"
-               "<tool_call>\n"
-               "<function=special_function_with_opt>\n"
-               "<parameter=arg1>\n"
-               "1\n"
-               "</parameter>\n"
-               "<parameter=arg2>\n"
-               "2\n"
-               "</parameter>\n"
-               "</function>\n"
-               "</tool_call>")
-            .parallel_tool_calls(true)
-            .tools({
-                special_function_tool, special_function_tool_with_optional_param
-        })
-            .expect_tool_calls({
-                { "special_function", R"({"arg1": 1})", {} },
-                { "special_function_with_opt", R"({"arg1": 1, "arg2": 2})", {} },
-            })
-            .run();
-
         tst.test(
                "<tool_call>\n"
                "<function=special_function>\n"
@@ -3674,37 +3680,6 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
             .expect_reconstruction()
             .run();
 
-        // Test flexible required argument ordering (required args still come first, in any order)
-        tst.test(
-               "<tool_call>\n"
-               "<function=edit>\n"
-               "<parameter=newString>\n#include\n</parameter>\n"
-               "<parameter=filename>\nfoo.c\n</parameter>\n"
-               "<parameter=oldString>\n#iclunde\n</parameter>\n"
-               "</function>\n"
-               "</tool_call>")
-            .tools({ edit_tool })
-            .expect_tool_calls({
-                { "edit", R"({"newString": "#include", "filename": "foo.c", "oldString": "#iclunde"})", {} },
-            })
-            .expect_reconstruction()
-            .run();
-
-        tst.test(
-               "<tool_call>\n"
-               "<function=tool_2req_4opt>\n"
-               "<parameter=req2>\n42\n</parameter>\n"
-               "<parameter=req1>\nhello\n</parameter>\n"
-               "<parameter=opt2>\n200\n</parameter>\n"
-               "</function>\n"
-               "</tool_call>")
-            .tools({ tool_2req_4opt })
-            .expect_tool_calls({
-                { "tool_2req_4opt", R"({"req2": 42, "req1": "hello", "opt2": 200})", {} },
-            })
-            .expect_reconstruction()
-            .run();
-
         // Test flexible optional argument ordering (2 required + 4 optional, reversed optional order)
         tst.test(
                "<tool_call>\n"
@@ -3987,7 +3962,6 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
             .expect_tool_calls({
                 { "special_function", R"({"arg1": 1})", {} },
             })
-            .expect_reconstruction()
             .run();
 
         // Tool call with negative number
@@ -4213,7 +4187,6 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
             .expect_tool_calls({
                 { "special_function", R"({"arg1": 1})", {} },
             })
-            .expect_reconstruction()
             .run();
 
         // Tool call with multiple params (mixed types)
@@ -4254,38 +4227,6 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
             .expect_reasoning("I'm thinking")
             .expect_content("Hello, world!\nWhat's up?")
             .run();
-
-        tst.test(
-            "Let me check the time\n\n"
-            "<｜DSML｜tool_calls>\n"
-            "<｜DSML｜invoke name=\"get_time\">\n"
-            "<｜DSML｜parameter name=\"city\" string=\"true\">Tokyo</｜DSML｜parameter>\n"
-            "</｜DSML｜invoke>\n"
-            "</｜DSML｜tool_calls>") // no </think> after the TC close because the grammar will immediately constrain it to end
-            .enable_thinking(true)
-            .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
-            .tools({ get_time_tool })
-            .expect_reasoning("Let me check the time")
-            .expect_tool_calls({ { "get_time", R"({"city": "Tokyo"})", {} } })
-            .run();
-    }
-
-    {
-        // The DSML separator belongs to the tool call block, not assistant content.
-        auto tst = peg_tester("models/templates/deepseek-ai-DeepSeek-V4-Flash-0731.jinja", detailed_debug);
-        tst.test(
-               "\n\n"
-               "<｜DSML｜tool_calls>\n"
-               "<｜DSML｜invoke name=\"special_function\">\n"
-               "<｜DSML｜parameter name=\"arg1\" string=\"false\">1</｜DSML｜parameter>\n"
-               "</｜DSML｜invoke>\n"
-               "</｜DSML｜tool_calls>")
-            .enable_thinking(false)
-            .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
-            .tools({ special_function_tool })
-            .expect(message_assist_call)
-            .expect_reconstruction()
-            .run();
     }
 
     // GLM-4.6 tests - format: <tool_call>function_name\n<arg_key>...</arg_key>\n<arg_value>...</arg_value>\n</tool_call>
@@ -6379,7 +6320,6 @@ static void test_template_generation_prompt() {
         std::vector<common_chat_msg> messages;
         bool                         add_generation_prompt  = true;
         common_chat_continuation     continue_final_message = COMMON_CHAT_CONTINUATION_NONE;
-        bool                         enable_thinking        = true;
     };
 
     auto basic = [&]() {
@@ -6411,7 +6351,6 @@ static void test_template_generation_prompt() {
         inputs.messages               = opts.messages;
         inputs.add_generation_prompt  = opts.add_generation_prompt;
         inputs.continue_final_message = opts.continue_final_message;
-        inputs.enable_thinking        = opts.enable_thinking;
 
         auto params = common_chat_templates_apply(tmpls.get(), inputs);
 
@@ -6510,156 +6449,6 @@ static void test_template_generation_prompt() {
         check(tmpls, continuation_reasoning(), "<｜Assistant｜><think>I'm");
     }
 
-    const std::string deepseek_v4_reasoning_effort_max = "Reasoning Effort: Absolute maximum";
-    const std::string deepseek_v4_flash_0731_reasoning_effort_max = "Reasoning Effort: Beyond maximum";
-
-    {
-        auto tmpls = read_templates("models/templates/deepseek-ai-DeepSeek-V4.jinja");
-        check(tmpls, basic(),                  "<｜Assistant｜><think>");
-        check(tmpls, continuation_content(),   "<｜Assistant｜><think>I'm thinking</think>Hello, ");
-        check(tmpls, continuation_reasoning(), "<｜Assistant｜><think>I'm");
-
-        auto continuation_content_no_thinking = continuation_content();
-        continuation_content_no_thinking.messages = { system_msg, message_user, simple_assist_msg("Hello, ") };
-        continuation_content_no_thinking.enable_thinking = false;
-        check(tmpls, continuation_content_no_thinking, "<｜Assistant｜></think>Hello, ");
-
-        common_chat_templates_inputs max_inputs;
-        max_inputs.messages = { system_msg, message_user };
-        max_inputs.chat_template_kwargs["reasoning_effort"] = R"("max")";
-        auto max_params = common_chat_templates_apply(tmpls.get(), max_inputs);
-        assert_contains(max_params.prompt, deepseek_v4_reasoning_effort_max);
-
-        auto high_inputs = max_inputs;
-        high_inputs.chat_template_kwargs["reasoning_effort"] = R"("high")";
-        auto high_params = common_chat_templates_apply(tmpls.get(), high_inputs);
-        assert_not_contains(high_params.prompt, deepseek_v4_reasoning_effort_max);
-
-        auto low_inputs = max_inputs;
-        low_inputs.chat_template_kwargs["reasoning_effort"] = R"("low")";
-        auto low_params = common_chat_templates_apply(tmpls.get(), low_inputs);
-        assert_not_contains(low_params.prompt, deepseek_v4_reasoning_effort_max);
-
-        common_chat_templates_inputs default_effort_inputs;
-        default_effort_inputs.messages = { system_msg, message_user };
-        auto default_effort_params = common_chat_templates_apply(tmpls.get(), default_effort_inputs);
-        assert_not_contains(default_effort_params.prompt, deepseek_v4_reasoning_effort_max);
-
-        auto non_thinking_max_inputs = max_inputs;
-        non_thinking_max_inputs.enable_thinking = false;
-        auto non_thinking_max_params = common_chat_templates_apply(tmpls.get(), non_thinking_max_inputs);
-        assert_not_contains(non_thinking_max_params.prompt, deepseek_v4_reasoning_effort_max);
-
-        common_chat_templates_inputs response_format_inputs;
-        response_format_inputs.messages = { system_msg, message_user };
-        response_format_inputs.tools = { get_time_tool };
-        response_format_inputs.json_schema =
-            R"({"type":"object","properties":{"answer":{"type":"string"}}})";
-        auto response_format_params = common_chat_templates_apply(tmpls.get(), response_format_inputs);
-        const auto tools_pos = response_format_params.prompt.find("## Tools");
-        const auto response_format_pos = response_format_params.prompt.find(
-            "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n");
-        if (tools_pos == std::string::npos || response_format_pos == std::string::npos || tools_pos > response_format_pos) {
-            LOG_ERR("Expected response format after tools\nActual: %s\n", response_format_params.prompt.c_str());
-            common_log_flush(common_log_main());
-            throw std::runtime_error("Test failed");
-        }
-        assert_contains(response_format_params.prompt, R"("answer": {"type": "string"})");
-
-        response_format_inputs.json_schema = "{}";
-        auto json_object_params = common_chat_templates_apply(tmpls.get(), response_format_inputs);
-        assert_contains(json_object_params.prompt,
-                        "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{}");
-
-        common_chat_msg assistant_history;
-        assistant_history.role              = "assistant";
-        assistant_history.content           = "Previous answer";
-        assistant_history.reasoning_content = "Previous reasoning";
-
-        common_chat_msg user_followup;
-        user_followup.role    = "user";
-        user_followup.content = "Follow up";
-
-        common_chat_templates_inputs default_history_inputs;
-        default_history_inputs.messages = { message_user, assistant_history, user_followup };
-        auto default_history_params = common_chat_templates_apply(tmpls.get(), default_history_inputs);
-        assert_contains(default_history_params.prompt, "<｜Assistant｜></think>Previous answer");
-
-        auto drop_thinking_inputs = default_history_inputs;
-        drop_thinking_inputs.chat_template_kwargs["drop_thinking"] = "false";
-        auto drop_thinking_params = common_chat_templates_apply(tmpls.get(), drop_thinking_inputs);
-        assert_contains(drop_thinking_params.prompt, "<｜Assistant｜><think>Previous reasoning</think>Previous answer");
-
-        auto preserve_reasoning_inputs = default_history_inputs;
-        preserve_reasoning_inputs.chat_template_kwargs["preserve_reasoning"] = "true";
-        auto preserve_reasoning_params = common_chat_templates_apply(tmpls.get(), preserve_reasoning_inputs);
-        assert_contains(preserve_reasoning_params.prompt, "<｜Assistant｜><think>Previous reasoning</think>Previous answer");
-        assert_equals(true, common_chat_templates_get_caps(tmpls.get()).at("supports_preserve_reasoning"));
-
-        auto no_preserve_reasoning_inputs = default_history_inputs;
-        no_preserve_reasoning_inputs.chat_template_kwargs["preserve_reasoning"] = "false";
-        auto no_preserve_reasoning_params = common_chat_templates_apply(tmpls.get(), no_preserve_reasoning_inputs);
-        assert_contains(no_preserve_reasoning_params.prompt, "<｜Assistant｜></think>Previous answer");
-
-        common_chat_msg empty_tool_call = simple_assist_msg("", "", "empty_args", "{}");
-        common_chat_templates_inputs empty_tool_inputs;
-        empty_tool_inputs.messages = { message_user, empty_tool_call };
-        empty_tool_inputs.tools    = { empty_args_tool };
-        auto empty_tool_params = common_chat_templates_apply(tmpls.get(), empty_tool_inputs);
-        assert_contains(empty_tool_params.prompt,
-                        "<｜DSML｜invoke name=\"empty_args\">\n\n</｜DSML｜invoke>");
-    }
-
-    {
-        auto tmpls = read_templates("models/templates/deepseek-ai-DeepSeek-V4-Flash-0731.jinja");
-        check(tmpls, basic(),                  "<｜Assistant｜><think>");
-        check(tmpls, continuation_content(),   "<｜Assistant｜><think>I'm thinking</think>Hello, ");
-        check(tmpls, continuation_reasoning(), "<｜Assistant｜><think>I'm");
-
-        auto continuation_content_no_thinking = continuation_content();
-        continuation_content_no_thinking.messages = { system_msg, message_user, simple_assist_msg("Hello, ") };
-        continuation_content_no_thinking.enable_thinking = false;
-        check(tmpls, continuation_content_no_thinking, "<｜Assistant｜></think>Hello, ");
-
-        common_chat_templates_inputs high_inputs;
-        high_inputs.messages = { system_msg, message_user };
-        high_inputs.chat_template_kwargs["reasoning_effort"] = R"("high")";
-        auto high_params = common_chat_templates_apply(tmpls.get(), high_inputs);
-        assert_contains(high_params.prompt, deepseek_v4_reasoning_effort_max);
-
-        auto max_inputs = high_inputs;
-        max_inputs.chat_template_kwargs["reasoning_effort"] = R"("max")";
-        auto max_params = common_chat_templates_apply(tmpls.get(), max_inputs);
-        assert_contains(max_params.prompt, deepseek_v4_flash_0731_reasoning_effort_max);
-
-        auto low_inputs = high_inputs;
-        low_inputs.chat_template_kwargs["reasoning_effort"] = R"("low")";
-        auto low_params = common_chat_templates_apply(tmpls.get(), low_inputs);
-        assert_not_contains(low_params.prompt, deepseek_v4_reasoning_effort_max);
-        assert_not_contains(low_params.prompt, deepseek_v4_flash_0731_reasoning_effort_max);
-
-        common_chat_templates_inputs default_effort_inputs;
-        default_effort_inputs.messages = { system_msg, message_user };
-        auto default_effort_params = common_chat_templates_apply(tmpls.get(), default_effort_inputs);
-        assert_not_contains(default_effort_params.prompt, deepseek_v4_reasoning_effort_max);
-        assert_not_contains(default_effort_params.prompt, deepseek_v4_flash_0731_reasoning_effort_max);
-
-        auto non_thinking_max_inputs = max_inputs;
-        non_thinking_max_inputs.enable_thinking = false;
-        auto non_thinking_max_params = common_chat_templates_apply(tmpls.get(), non_thinking_max_inputs);
-        assert_not_contains(non_thinking_max_params.prompt, deepseek_v4_flash_0731_reasoning_effort_max);
-
-        common_chat_templates_inputs response_format_inputs;
-        response_format_inputs.messages = { system_msg, message_user };
-        response_format_inputs.tools = { get_time_tool };
-        response_format_inputs.json_schema =
-            R"({"type":"object","properties":{"answer":{"type":"string"}}})";
-        auto response_format_params = common_chat_templates_apply(tmpls.get(), response_format_inputs);
-        assert_contains(response_format_params.prompt,
-                        "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n");
-        assert_contains(response_format_params.prompt, R"("answer": {"type": "string"})");
-    }
-
     {
         auto tmpls = read_templates("models/templates/openbmb-MiniCPM5-1B.jinja");
         check(tmpls, basic(),                  "<|im_start|>assistant\n<think>\n");

tests/test-llama-archs.cpp

@@ -430,7 +430,7 @@ static bool arch_supported(const llm_arch arch) {
 
     // FIXME: these hit scheduler/view-backed-output issues with WebGPU on CI.
 #ifdef GGML_USE_WEBGPU
-    if (arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA || arch == LLM_ARCH_MINIMAX_M3) {
+    if (arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA) {
         return false;
     }
 #endif // GGML_USE_WEBGPU

tests/test-model-resolution.cpp

@@ -1,506 +0,0 @@
-// tests the HF model resolution and the model handler assembly end-to-end on
-// synthetic repo listings: a local httplib server bound to the loopback
-// serves hardcoded HF API responses, so the real client, hf_cache, resolution
-// and CLI parsing run against them without external network access
-
-#include "arg.h"
-#include "common.h"
-#include "download.h"
-#include "http.h"
-#include "log.h"
-
-#include <nlohmann/json.hpp>
-
-#include <algorithm>
-#include <cstdio>
-#include <cstdlib>
-#include <filesystem>
-#include <map>
-#include <thread>
-#include <string>
-#include <vector>
-
-// the case and reordering being checked, printed with every failure
-static std::string g_context;
-
-// independent of NDEBUG, so the checks stay alive in Release builds
-#define REQUIRE(x) do {                                                         \
-    if (!(x)) {                                                                 \
-        fprintf(stderr, "%s:%d: [%s] REQUIRE(%s) failed\n",                     \
-                __FILE__, __LINE__, g_context.c_str(), #x);                     \
-        std::abort();                                                           \
-    }                                                                           \
-} while (0)
-
-#define REQUIRE_EQ(actual, expected) do {                                       \
-    if (!((actual) == (expected))) {                                            \
-        fprintf(stderr, "%s:%d: [%s] REQUIRE_EQ(%s, %s) failed\n  actual:   '%s'\n  expected: '%s'\n", \
-                __FILE__, __LINE__, g_context.c_str(), #actual, #expected,      \
-                std::string(actual).c_str(), std::string(expected).c_str());    \
-        std::abort();                                                           \
-    }                                                                           \
-} while (0)
-
-//
-// synthetic repos keyed by repo id, served over the loopback by a real
-// httplib server, so the tested code runs its own client and transport
-//
-
-static std::map<std::string, std::vector<std::string>> g_repos;
-
-static const char * COMMIT = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
-
-// the server lives in main, so its destructor runs before the static teardown
-// tears down the winsock state httplib brings in
-static void serve_repos(httplib::Server & server) {
-    server.Get(R"(/api/models/(.+)/refs)", [](const httplib::Request & req, httplib::Response & res) {
-        if (g_repos.count(req.matches[1])) {
-            res.set_content(nlohmann::json{{"branches", {{{"name", "main"}, {"targetCommit", COMMIT}}}}}.dump(),
-                            "application/json");
-        } else {
-            res.status = 404;
-        }
-    });
-    server.Get(R"(/api/models/(.+)/tree/.+)", [](const httplib::Request & req, httplib::Response & res) {
-        if (!g_repos.count(req.matches[1])) {
-            res.status = 404;
-            return;
-        }
-        auto files = nlohmann::json::array();
-        size_t i = 0;
-        for (const auto & p : g_repos[req.matches[1]]) {
-            char oid[41];
-            snprintf(oid, sizeof(oid), "%040lx", (unsigned long) ++i);
-            files.push_back({{"type", "file"}, {"path", p}, {"size", 1}, {"oid", oid}});
-        }
-        res.set_content(files.dump(), "application/json");
-    });
-}
-
-static common_params_model model_ref(const std::string & hf_repo, const std::string & hf_file = "") {
-    common_params_model m;
-    m.hf_repo = hf_repo;
-    m.hf_file = hf_file;
-    return m;
-}
-
-// the model cache is isolated under a temporary directory named after the
-// loopback port, so concurrent runs on a shared machine keep their own, and
-// the local path the handler wires for a file is snapshots/<commit>/<path>
-static std::filesystem::path cache_dir;
-
-static std::string cached(std::string repo_id, const std::string & path) {
-    string_replace_all(repo_id, "/", "--");
-    return (cache_dir / ("models--" + repo_id) / "snapshots" / COMMIT / path).string();
-}
-
-//
-// fixtures mimicking real repo layouts
-//
-
-// flat layout in the style of ggml-org/gemma-4-31B-it-GGUF
-static const std::vector<std::string> flat = {
-    "README.md",
-    "model-BF16.gguf",
-    "model-Q4_K_M.gguf",
-    "model-Q8_0.gguf",
-    "mmproj-model-BF16.gguf",
-    "mmproj-model-Q8_0.gguf",
-    "mtp-model-BF16.gguf",
-    "mtp-model-Q4_0.gguf",
-    "mtp-model-Q8_0.gguf",
-    "dflash-model-BF16.gguf",
-    "dflash-model-Q8_0.gguf",
-};
-
-// quants in subdirectories with sharded files and root sidecars,
-// in the style of stepfun-ai/Step-3.7-Flash-GGUF
-static const std::vector<std::string> subdir = {
-    "mmproj-model-f16.gguf",
-    "model-mtp-BF16.gguf",
-    "model-mtp-Q8_0.gguf",
-    "Q3_K_M/model-Q3_K_M-00001-of-00003.gguf",
-    "Q3_K_M/model-Q3_K_M-00002-of-00003.gguf",
-    "Q3_K_M/model-Q3_K_M-00003-of-00003.gguf",
-    "Q8_0/model-Q8_0-00001-of-00002.gguf",
-    "Q8_0/model-Q8_0-00002-of-00002.gguf",
-};
-
-// sidecar quants exist where the full model quant does not,
-// in the style of ggml-org/Qwen3.6-27B-GGUF
-static const std::vector<std::string> hole = {
-    "model-BF16.gguf",
-    "model-Q4_K_M.gguf",
-    "model-Q8_0.gguf",
-    "mtp-model-BF16.gguf",
-    "mtp-model-Q4_0.gguf",
-    "mtp-model-Q8_0.gguf",
-    "dflash-model-BF16.gguf",
-    "dflash-model-Q8_0.gguf",
-};
-
-// unsloth-style naming with UD quants and a suffix MTP file
-static const std::vector<std::string> unsloth = {
-    "model-UD-Q8_K_XL.gguf",
-    "mmproj-BF16.gguf",
-    "model-MTP-BF16.gguf",
-};
-
-// bartowski-style vendor prefix and mradermacher-style dot quant
-static const std::vector<std::string> vendors = {
-    "TheDrummer_Model-24B-v4.1-Q8_0.gguf",
-    "BlackSheep-24B.Q8_0.gguf",
-};
-
-// every speculative sidecar type at the same quant
-static const std::vector<std::string> quad = {
-    "model-Q8_0.gguf",
-    "mtp-model-Q8_0.gguf",
-    "dflash-model-Q8_0.gguf",
-    "eagle3-model-Q8_0.gguf",
-    "dspark-model-Q8_0.gguf",
-};
-
-static const std::vector<std::string> dflash_only = {
-    "model-Q8_0.gguf",
-    "dflash-model-Q8_0.gguf",
-};
-
-static const std::vector<std::string> eagle3_only = {
-    "model-Q8_0.gguf",
-    "eagle3-model-Q8_0.gguf",
-};
-
-// a single full quant with dspark sidecars at other quants,
-// in the style of ggml-org/DeepSeek-V4-Flash-0731-GGUF
-static const std::vector<std::string> spark = {
-    "README.md",
-    "model-MXFP4.gguf",
-    "dspark-model-BF16.gguf",
-    "dspark-model-MXFP4.gguf",
-};
-
-// dspark outranks dflash in the type auto-selection
-static const std::vector<std::string> dspark_dflash = {
-    "model-Q8_0.gguf",
-    "dflash-model-Q8_0.gguf",
-    "dspark-model-Q8_0.gguf",
-};
-
-//
-// table-driven plan resolution through the real entry point,
-// each case replayed on multiple deterministic reorderings of the listing,
-// except the cases whose pick legitimately depends on the listing order
-//
-
-struct plan_case {
-    const char * name;
-    const std::vector<std::string> & files;
-    const char * hf_repo;
-    const char * hf_file;
-    bool sidecars;        // request mmproj + mtp + dflash + eagle3 + dspark
-    bool order_dependent; // the expected pick depends on the listing order
-    const char * primary;
-    std::vector<std::string> model_files;
-    const char * mmproj;
-    const char * mtp;
-    const char * dflash;
-    const char * eagle3;
-    const char * dspark;
-};
-
-static const plan_case plan_cases[] = {
-    // exact tag picks the matching primary, sidecars follow the tag
-    {"flat exact tag", flat, "test/repo:Q8_0", "", true, false,
-     "model-Q8_0.gguf", {"model-Q8_0.gguf"},
-     "mmproj-model-Q8_0.gguf", "mtp-model-Q8_0.gguf", "dflash-model-Q8_0.gguf", "", ""},
-
-    // no tag falls back to the default quant preference
-    {"flat default", flat, "test/repo", "", false, false,
-     "model-Q4_K_M.gguf", {"model-Q4_K_M.gguf"},
-     "", "", "", "", ""},
-
-    // no tag and no default match falls back to the first model in the listing
-    {"unsloth fallback", unsloth, "test/repo", "", true, true,
-     "model-UD-Q8_K_XL.gguf", {"model-UD-Q8_K_XL.gguf"},
-     "mmproj-BF16.gguf", "", "", "", ""},
-
-    // explicit hf_file picks that exact file
-    {"flat hf_file", flat, "test/repo", "model-BF16.gguf", false, false,
-     "model-BF16.gguf", {"model-BF16.gguf"},
-     "", "", "", "", ""},
-
-    // missing hf_file resolves nothing
-    {"flat missing hf_file", flat, "test/repo", "nope.gguf", false, false,
-     "", {},
-     "", "", "", "", ""},
-
-    // a sharded primary brings all its parts, a subdir primary finds the root sidecar
-    {"subdir shards", subdir, "test/repo:Q3_K_M", "", true, false,
-     "Q3_K_M/model-Q3_K_M-00001-of-00003.gguf",
-     {"Q3_K_M/model-Q3_K_M-00001-of-00003.gguf",
-      "Q3_K_M/model-Q3_K_M-00002-of-00003.gguf",
-      "Q3_K_M/model-Q3_K_M-00003-of-00003.gguf"},
-     "mmproj-model-f16.gguf", "model-mtp-Q8_0.gguf", "", "", ""},
-
-    // a tag with no matching full model still resolves the requested sidecars
-    {"hole tag sidecar", hole, "test/repo:Q4_0", "", true, false,
-     "", {},
-     "", "mtp-model-Q4_0.gguf", "dflash-model-Q8_0.gguf", "", ""},
-
-    // the same tag without a requested sidecar resolves nothing
-    {"hole tag alone", hole, "test/repo:Q4_0", "", false, false,
-     "", {},
-     "", "", "", "", ""},
-
-    // no tag anchors the sidecars on the primary quant
-    {"hole default anchor", hole, "test/repo", "", true, false,
-     "model-Q4_K_M.gguf", {"model-Q4_K_M.gguf"},
-     "", "mtp-model-Q4_0.gguf", "dflash-model-Q8_0.gguf", "", ""},
-
-    // the mtp- keyword is case sensitive, a suffix -MTP file is not discovered
-    {"unsloth suffix mtp", unsloth, "test/repo:Q8_K_XL", "", true, false,
-     "model-UD-Q8_K_XL.gguf", {"model-UD-Q8_K_XL.gguf"},
-     "mmproj-BF16.gguf", "", "", "", ""},
-
-    // vendor prefixes and the dot quant convention both match the tag,
-    // first match wins between two files at the same quant
-    {"vendor prefix", vendors, "test/repo:Q8_0", "", false, true,
-     "TheDrummer_Model-24B-v4.1-Q8_0.gguf", {"TheDrummer_Model-24B-v4.1-Q8_0.gguf"},
-     "", "", "", "", ""},
-
-    // every sidecar type resolves at the tag
-    {"quad exact tag", quad, "test/repo:Q8_0", "", true, false,
-     "model-Q8_0.gguf", {"model-Q8_0.gguf"},
-     "", "mtp-model-Q8_0.gguf", "dflash-model-Q8_0.gguf", "eagle3-model-Q8_0.gguf", "dspark-model-Q8_0.gguf"},
-
-    // no tag anchors the dspark sidecar on the only full quant
-    {"spark default anchor", spark, "test/repo", "", true, false,
-     "model-MXFP4.gguf", {"model-MXFP4.gguf"},
-     "", "", "", "", "dspark-model-MXFP4.gguf"},
-
-    // a tag with no matching full model still resolves the exact dspark sidecar
-    {"spark tag sidecar", spark, "test/repo:BF16", "", true, false,
-     "", {},
-     "", "", "", "", "dspark-model-BF16.gguf"},
-};
-
-static void check_plan(const plan_case & c) {
-    common_download_opts opts;
-    opts.download_mmproj = c.sidecars;
-    opts.download_mtp    = c.sidecars;
-    opts.download_dflash = c.sidecars;
-    opts.download_eagle3 = c.sidecars;
-    opts.download_dspark = c.sidecars;
-
-    auto plan = common_download_get_hf_plan(model_ref(c.hf_repo, c.hf_file), opts);
-
-    REQUIRE_EQ(plan.primary.path, c.primary);
-    REQUIRE_EQ(plan.mmproj.path,  c.mmproj);
-    REQUIRE_EQ(plan.mtp.path,     c.mtp);
-    REQUIRE_EQ(plan.dflash.path,  c.dflash);
-    REQUIRE_EQ(plan.eagle3.path,  c.eagle3);
-    REQUIRE_EQ(plan.dspark.path,  c.dspark);
-
-    // exact shard set, order insensitive; the primary must be the first split
-    std::vector<std::string> actual;
-    for (const auto & f : plan.model_files) {
-        actual.push_back(f.path);
-    }
-    std::sort(actual.begin(), actual.end());
-    auto expected = c.model_files;
-    std::sort(expected.begin(), expected.end());
-    REQUIRE(actual == expected);
-    if (!expected.empty()) {
-        REQUIRE(plan.primary.path == expected.front());
-    }
-}
-
-static void test_plan_resolution() {
-    printf("test-model-resolution: plan resolution on %zu cases\n", sizeof(plan_cases) / sizeof(plan_cases[0]));
-
-    for (const auto & c : plan_cases) {
-        printf("  %s\n", c.name);
-        // invariant: the resolution is insensitive to the listing order
-        for (size_t rot = 0; rot < c.files.size(); ++rot) {
-            if (c.order_dependent && rot > 0) {
-                continue;
-            }
-            g_context = std::string(c.name) + ", reordering " + std::to_string(rot);
-            auto files = c.files;
-            std::rotate(files.begin(), files.begin() + rot, files.end());
-            if (rot % 2 == 1) {
-                std::reverse(files.begin(), files.end());
-            }
-            g_repos["test/repo"] = files;
-            check_plan(c);
-        }
-    }
-    g_repos.clear();
-}
-
-//
-// end-to-end assembly: real CLI parsing, real handler init resolving over the
-// loopback, downloads skipped by flipping offline before apply
-//
-
-static void assemble(std::vector<std::string> argv, common_params & params) {
-    std::vector<char *> cargv;
-    g_context.clear();
-    for (auto & a : argv) {
-        g_context += g_context.empty() ? a : " " + a;
-        cargv.push_back(a.data());
-    }
-    bool ok = common_params_parse((int) cargv.size(), cargv.data(), params, LLAMA_EXAMPLE_SERVER);
-    REQUIRE(ok);
-
-    auto handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER);
-
-    // skip the network execution, on_done still wires the params
-    params.offline = true;
-    common_models_handler_apply(handler, params);
-}
-
-static void test_task_assembly() {
-    printf("test-model-resolution: end-to-end assembly\n");
-
-    g_repos["test/main"]   = flat;
-    g_repos["test/hole"]   = hole;
-    g_repos["test/quad"]   = quad;
-    g_repos["test/dflash"] = dflash_only;
-    g_repos["test/eagle3"] = eagle3_only;
-    g_repos["test/spark"]  = spark;
-    g_repos["test/pair"]   = dspark_dflash;
-    g_repos["test/small"]  = {"draft-model-Q4_K_M.gguf"};
-    g_repos["test/preset"] = {"preset.ini", "model-Q8_0.gguf"};
-
-    {
-        // plain -hf wires the model and its mmproj, nothing speculative
-        common_params params;
-        assemble({"server", "-hf", "test/main:Q8_0"}, params);
-        REQUIRE_EQ(params.model.path,  cached("test/main", "model-Q8_0.gguf"));
-        REQUIRE_EQ(params.mmproj.path, cached("test/main", "mmproj-model-Q8_0.gguf"));
-        REQUIRE(params.speculative.draft.mparams.path.empty());
-    }
-    {
-        // --no-mmproj disables the mmproj discovery
-        common_params params;
-        assemble({"server", "-hf", "test/main:Q8_0", "--no-mmproj"}, params);
-        REQUIRE(params.mmproj.path.empty());
-    }
-    {
-        // an explicit --mmproj wins over the discovery
-        common_params params;
-        assemble({"server", "-hf", "test/main:Q8_0", "--mmproj", "/local/mmproj.gguf"}, params);
-        REQUIRE(params.mmproj.path == "/local/mmproj.gguf");
-    }
-    {
-        // -hf with a spec type wires the sidecar of the main repo as fallback draft
-        common_params params;
-        assemble({"server", "-hf", "test/main:Q8_0", "--spec-type", "draft-mtp"}, params);
-        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/main", "mtp-model-Q8_0.gguf"));
-    }
-    {
-        // -hfd with a spec type wires the draft repo sidecar at its tag,
-        // not its full model, and suppresses the main repo fallback
-        common_params params;
-        assemble({"server", "-hf", "test/hole:Q8_0", "-hfd", "test/hole:Q4_0", "--spec-type", "draft-mtp"}, params);
-        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/hole", "mtp-model-Q4_0.gguf"));
-    }
-    {
-        // an explicit -md file wins over the sidecar resolution
-        common_params params;
-        assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/main", "-md", "mtp-model-BF16.gguf", "--spec-type", "draft-mtp"}, params);
-        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/main", "mtp-model-BF16.gguf"));
-    }
-    {
-        // -hfd without a spec type auto-selects the type, mtp first when all ship
-        common_params params;
-        assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/quad:Q8_0"}, params);
-        REQUIRE(params.speculative.types == std::vector<enum common_speculative_type>{COMMON_SPECULATIVE_TYPE_DRAFT_MTP});
-        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/quad", "mtp-model-Q8_0.gguf"));
-    }
-    {
-        // auto-selection with only a dflash sidecar
-        common_params params;
-        assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/dflash:Q8_0"}, params);
-        REQUIRE(params.speculative.types == std::vector<enum common_speculative_type>{COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH});
-        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/dflash", "dflash-model-Q8_0.gguf"));
-    }
-    {
-        // auto-selection with only an eagle3 sidecar
-        common_params params;
-        assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/eagle3:Q8_0"}, params);
-        REQUIRE(params.speculative.types == std::vector<enum common_speculative_type>{COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3});
-        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/eagle3", "eagle3-model-Q8_0.gguf"));
-    }
-    {
-        // auto-selection prefers dspark over dflash when both ship
-        common_params params;
-        assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/pair:Q8_0"}, params);
-        REQUIRE(params.speculative.types == std::vector<enum common_speculative_type>{COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK});
-        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/pair", "dspark-model-Q8_0.gguf"));
-    }
-    {
-        // -hf with the dspark spec type wires the sidecar of the main repo,
-        // anchored on the only full quant
-        common_params params;
-        assemble({"server", "-hf", "test/spark", "--spec-type", "draft-dspark"}, params);
-        REQUIRE_EQ(params.model.path, cached("test/spark", "model-MXFP4.gguf"));
-        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/spark", "dspark-model-MXFP4.gguf"));
-    }
-    {
-        // -hfd on a repo without sidecars keeps resolving a full model as draft
-        common_params params;
-        assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/small"}, params);
-        REQUIRE(params.speculative.types == std::vector<enum common_speculative_type>{COMMON_SPECULATIVE_TYPE_NONE});
-        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/small", "draft-model-Q4_K_M.gguf"));
-    }
-    {
-        // a preset repo wires the preset and clears the model for router mode
-        common_params params;
-        assemble({"server", "-hf", "test/preset"}, params);
-        REQUIRE_EQ(params.models_preset, cached("test/preset", "preset.ini"));
-        REQUIRE(params.model.path.empty());
-        REQUIRE(params.model.hf_repo.empty());
-    }
-
-    g_repos.clear();
-}
-
-int main(void) {
-    // unbuffered, so a crash cannot swallow the reports already printed
-    setvbuf(stdout, nullptr, _IONBF, 0);
-    setvbuf(stderr, nullptr, _IONBF, 0);
-
-    // the negative cases legitimately log errors on every reordering,
-    // keep the output down to the reports
-    common_log_pause(common_log_main());
-
-    // the loopback endpoint also keeps the client init from rejecting
-    // https on the builds without TLS support
-    httplib::Server server;
-    serve_repos(server);
-    int port = server.bind_to_any_port("127.0.0.1");
-
-    // isolate the cache, its location is read once so it is set
-    // before anything else
-    cache_dir = std::filesystem::temp_directory_path() /
-                ("test-model-resolution-cache-" + std::to_string(port));
-    std::filesystem::remove_all(cache_dir);
-    common_set_env("LLAMA_CACHE", cache_dir.string());
-
-    std::thread server_thread([&server] { server.listen_after_bind(); });
-    server.wait_until_ready();
-    common_set_env("MODEL_ENDPOINT", "http://127.0.0.1:" + std::to_string(port) + "/");
-
-    test_plan_resolution();
-    test_task_assembly();
-
-    server.stop();
-    server_thread.join();
-
-    std::filesystem::remove_all(cache_dir);
-    printf("test-model-resolution: all tests OK\n");
-    return 0;
-}

tests/test-recurrent-state-rollback.cpp

@@ -83,33 +83,27 @@ int main(int argc, char ** argv) {
     if (llama_vocab_type(vocab) == LLAMA_VOCAB_TYPE_NONE) {
         tokens = { 1, 2, 3, 4, 5, 6, 7, 8, 9 };
     } else {
-        tokens = common_tokenize(ctx_src, "The quick brown fox jumps over the lazy dog", true);
+        tokens = common_tokenize(ctx_src, "The quick brown fox jumps", true);
     }
     const uint32_t n_rs_seq = llama_n_rs_seq(ctx_src);
-    constexpr uint32_t n_rollback = 3;
-    if (n_rs_seq < n_rollback) {
-        fprintf(stderr, "%s : skipping because n_rs_seq is too small\n", __func__);
-        llama_free(ctx_src);
-        llama_free(ctx_dst);
-        return 0;
+    if (tokens.size() > n_rs_seq + 1) {
+        tokens.resize(n_rs_seq + 1);
     }
-    if (tokens.empty()) {
+    if (tokens.size() < 2) {
         fprintf(stderr, "%s : not enough prompt tokens\n", __func__);
         return 1;
     }
-    tokens.resize(n_rs_seq + 1, tokens.back());
+    const uint32_t    n_tokens = tokens.size();
+    const llama_token last_tok = tokens.back();
+    const llama_pos   last_pos = (llama_pos) n_tokens - 2;
 
-    const uint32_t  n_tokens     = tokens.size();
-    const llama_pos rollback_pos = (llama_pos) n_tokens - n_rollback;
-
-    // Decode the full prompt on the source, then roll back three positions.
-    // Replaying them crosses DSV4's ratio-4 compressor boundary.
+    // Decode the full prompt on the source, then roll back the last position.
     // Rollback leaves the recurrent memory in a snapshot state (rs_idx != 0).
     if (!decode_tokens(ctx_src, tokens, n_tokens)) {
         fprintf(stderr, "%s : failed to decode prompt\n", __func__);
         return 1;
     }
-    if (!llama_memory_seq_rm(llama_get_memory(ctx_src), 0, rollback_pos, -1)) {
+    if (!llama_memory_seq_rm(llama_get_memory(ctx_src), 0, last_pos, -1)) {
         fprintf(stderr, "%s : rollback failed\n", __func__);
         return 1;
     }
@@ -119,56 +113,31 @@ int main(int argc, char ** argv) {
     ckpt.update_tgt(ctx_src, 0, 0);
     ckpt.load_tgt(ctx_dst, 0, 0);
 
+    // Replay the rolled-back token on both contexts and compare logits.
+    if (!decode_one(ctx_src, last_tok, last_pos) ||
+        !decode_one(ctx_dst, last_tok, last_pos)) {
+        fprintf(stderr, "%s : replay failed\n", __func__);
+        return 1;
+    }
+
+    const float * logits_src = llama_get_logits_ith(ctx_src, 0);
+    const float * logits_dst = llama_get_logits_ith(ctx_dst, 0);
+    if (logits_src == nullptr || logits_dst == nullptr) {
+        fprintf(stderr, "%s : missing logits\n", __func__);
+        return 1;
+    }
+
     constexpr float eps = 1e-5f;
-    std::vector<std::vector<float>> logits_src_replay(n_rollback);
-    const auto replay_and_compare = [&](const char * mode) {
-        for (uint32_t i = 0; i < n_rollback; ++i) {
-            const llama_pos pos = rollback_pos + i;
-            if (!decode_one(ctx_src, tokens[pos], pos) ||
-                !decode_one(ctx_dst, tokens[pos], pos)) {
-                fprintf(stderr, "%s : %s replay failed at position %d\n", __func__, mode, pos);
-                return false;
-            }
-
-            const float * logits_src = llama_get_logits_ith(ctx_src, 0);
-            const float * logits_dst = llama_get_logits_ith(ctx_dst, 0);
-            if (logits_src == nullptr || logits_dst == nullptr) {
-                fprintf(stderr, "%s : missing %s logits at position %d\n", __func__, mode, pos);
-                return false;
-            }
-
-            logits_src_replay[i].assign(logits_src, logits_src + n_vocab);
-            for (int token = 0; token < n_vocab; ++token) {
-                if (std::fabs(logits_src[token] - logits_dst[token]) > eps) {
-                    fprintf(stderr, "%s : %s logits mismatch at position %d, token %d (%g != %g)\n",
-                            __func__, mode, pos, token, (double) logits_src[token], (double) logits_dst[token]);
-                    return false;
-                }
-            }
+    for (int i = 0; i < n_vocab; ++i) {
+        if (std::fabs(logits_src[i] - logits_dst[i]) > eps) {
+            fprintf(stderr, "%s : logits mismatch at token %d (%g != %g)\n",
+                    __func__, i, (double) logits_src[i], (double) logits_dst[i]);
+            return 1;
         }
-        return true;
-    };
-    if (!replay_and_compare("full")) {
-        return 1;
-    }
-
-    if (!llama_memory_seq_rm(llama_get_memory(ctx_src), 0, rollback_pos, -1) ||
-        !llama_memory_seq_rm(llama_get_memory(ctx_dst), 0, rollback_pos, -1)) {
-        fprintf(stderr, "%s : partial rollback failed\n", __func__);
-        return 1;
-    }
-
-    constexpr llama_state_seq_flags partial_flags = LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY;
-    common_prompt_checkpoint ckpt_partial;
-    ckpt_partial.update_tgt(ctx_src, 0, partial_flags);
-    ckpt_partial.load_tgt(ctx_dst, 0, partial_flags);
-
-    if (!replay_and_compare("partial")) {
-        return 1;
     }
 
     // Repeat the load into a context that already has its own rollback state:
-    // groups 1..n_rs_seq hold a different prompt's history, and rs_idx[0] is
+    // groups 1..n_rs_seq hold a *different* prompt's history, and rs_idx[0] is
     // non-zero at load time. The restore must wipe that state and still match.
     llama_context * ctx_dirty = make_ctx(params, model);
     if (ctx_dirty == nullptr) {
@@ -187,32 +156,29 @@ int main(int argc, char ** argv) {
         fprintf(stderr, "%s : dirty prompt decode failed\n", __func__);
         return 1;
     }
-    if (!llama_memory_seq_rm(llama_get_memory(ctx_dirty), 0, rollback_pos, -1)) {
+    if (!llama_memory_seq_rm(llama_get_memory(ctx_dirty), 0, last_pos, -1)) {
         fprintf(stderr, "%s : dirty rollback failed\n", __func__);
         return 1;
     }
 
     ckpt.load_tgt(ctx_dirty, 0, 0);
 
-    for (uint32_t i = 0; i < n_rollback; ++i) {
-        const llama_pos pos = rollback_pos + i;
-        if (!decode_one(ctx_dirty, tokens[pos], pos)) {
-            fprintf(stderr, "%s : dirty replay failed at position %d\n", __func__, pos);
-            return 1;
-        }
+    if (!decode_one(ctx_dirty, last_tok, last_pos)) {
+        fprintf(stderr, "%s : dirty replay failed\n", __func__);
+        return 1;
+    }
 
-        const float * logits_dirty = llama_get_logits_ith(ctx_dirty, 0);
-        if (logits_dirty == nullptr) {
-            fprintf(stderr, "%s : missing dirty logits at position %d\n", __func__, pos);
-            return 1;
-        }
+    const float * logits_dirty = llama_get_logits_ith(ctx_dirty, 0);
+    if (logits_dirty == nullptr) {
+        fprintf(stderr, "%s : missing dirty logits\n", __func__);
+        return 1;
+    }
 
-        for (int token = 0; token < n_vocab; ++token) {
-            if (std::fabs(logits_src_replay[i][token] - logits_dirty[token]) > eps) {
-                fprintf(stderr, "%s : dirty-ctx logits mismatch at position %d, token %d (%g != %g)\n",
-                        __func__, pos, token, (double) logits_src_replay[i][token], (double) logits_dirty[token]);
-                return 1;
-            }
+    for (int i = 0; i < n_vocab; ++i) {
+        if (std::fabs(logits_src[i] - logits_dirty[i]) > eps) {
+            fprintf(stderr, "%s : dirty-ctx logits mismatch at token %d (%g != %g)\n",
+                    __func__, i, (double) logits_src[i], (double) logits_dirty[i]);
+            return 1;
         }
     }
 

tests/test-sampling.cpp

@@ -144,7 +144,7 @@ static void test_penalties(
 
     sampler_tester tester(probs, probs_expected);
 
-    auto * sampler = llama_sampler_init_penalties((int32_t) probs.size(), (int32_t) last_tokens.size(), repeat_penalty, alpha_frequency, alpha_presence);
+    auto * sampler = llama_sampler_init_penalties(last_tokens.size(), repeat_penalty, alpha_frequency, alpha_presence);
 
     for (size_t i = 0; i < last_tokens.size(); i++) {
         llama_sampler_accept(sampler, last_tokens[i]);