mtmd: deepseek-ocr v1 multi-tile (#24717 )

* mtmd: deepseek-ocr v1 multi-tile dynamic resolution + unified image-preprocessors for both versions (ds-ocr v1 and v2) * remove hacky API * fuse row into a long image * almost working * adapt to new preprocessor api * rm debugging printf * improve * mtmd: dsocr-tiles fixes (#25481) * ds-ocr img-preproc fuse_row tile-drop fix for multi rows and columns images * mtmd drop the duplicate redundant img_end * deepseekocr graph simplify CLS broadcast cleanup * test-deepseek-ocr: relax v1 single-view tolerance; drop trailing prompt space; make DRY opt-in and n_predict model-specific (#25486) --------- Co-authored-by: Saba Fallah <10401143+sfallah@users.noreply.github.com> Co-authored-by: Saba Fallah <sabafallah@gmail.com>
feat: pre-select models in the webui using alias (#25492 )
2026-07-10 16:07:57 +02:00 · 2026-07-10 16:05:49 +02:00 · 2026-07-10 15:04:00 +02:00 · 2026-07-10 15:03:52 +02:00 · 2026-07-10 14:42:17 +02:00 · 2026-07-10 13:11:37 +03:00
33 changed files with 1184 additions and 618 deletions
@@ -73,4 +73,3 @@ jobs:
          hf buckets rm ggml-org/${{ env.HF_BUCKET_NAME }}/index.html --yes 2>/dev/null || true
          hf buckets rm ggml-org/${{ env.HF_BUCKET_NAME }}/bundle.js --yes 2>/dev/null || true
          hf buckets rm ggml-org/${{ env.HF_BUCKET_NAME }}/bundle.css --yes 2>/dev/null || true
-          hf buckets rm ggml-org/${{ env.HF_BUCKET_NAME }}/loading.html --yes 2>/dev/null || true
@@ -3036,7 +3036,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
        {"--tools"}, "TOOL1,TOOL2,...",
        "experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n"
        "specify \"all\" to enable all tools\n"
-        "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, apply_diff, get_datetime",
+        "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime",
        [](common_params & params, const std::string & value) {
            params.server_tools = parse_csv_row(value);
        }
@@ -4,8 +4,8 @@ project("ggml" C CXX ASM)

 ### GGML Version
 set(GGML_VERSION_MAJOR 0)
-set(GGML_VERSION_MINOR 15)
-set(GGML_VERSION_PATCH 3)
+set(GGML_VERSION_MINOR 16)
+set(GGML_VERSION_PATCH 0)
 set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")

 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/")
@@ -263,13 +263,13 @@ void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
            const uint8x16_t raw16 = vcombine_u8(raw, raw);

            // First 16 elements: replicate bytes 0-3, shift, mask, subtract 1
-            uint8x16_t bytes0 = vqtbl1q_u8(raw16, idx_lo);
+            uint8x16_t bytes0 = ggml_vqtbl1q_u8(raw16, idx_lo);
            int8x16_t qv0 = vsubq_s8(
                vreinterpretq_s8_u8(vandq_u8(vshlq_u8(bytes0, shifts), mask2)),
                one);

            // Second 16 elements: replicate bytes 4-7, shift, mask, subtract 1
-            uint8x16_t bytes1 = vqtbl1q_u8(raw16, idx_hi);
+            uint8x16_t bytes1 = ggml_vqtbl1q_u8(raw16, idx_hi);
            int8x16_t qv1 = vsubq_s8(
                vreinterpretq_s8_u8(vandq_u8(vshlq_u8(bytes1, shifts), mask2)),
                one);
@@ -1 +1 @@
-eced84c86f8b012c752c016f7fe789adea168e1e
+eaa0a74fa768bb72da623a61d9da3d436053ea91
@@ -646,7 +646,7 @@ static void dsv4_set_kq_mask(
        return;
    }

-    GGML_ASSERT(dst->type == GGML_TYPE_F32);
+    GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
    GGML_ASSERT(n_stream > 0);
    GGML_ASSERT(n_tokens%n_stream == 0);
    GGML_ASSERT(dst->ne[0] == plan.n_kv);
@@ -656,13 +656,27 @@ static void dsv4_set_kq_mask(
    GGML_ASSERT((int64_t) plan.n_visible.size() == (int64_t) n_tokens);
    GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer));

-    float * data = (float *) dst->data;
+    if (dst->type == GGML_TYPE_F32) {
+        float * data = (float *) dst->data;

-    for (int64_t i = 0; i < (int64_t) n_tokens; ++i) {
-        const int32_t n_visible = plan.n_visible[i];
+        for (int64_t i = 0; i < (int64_t) n_tokens; ++i) {
+            const int32_t n_visible = plan.n_visible[i];

-        for (int64_t j = 0; j < dst->ne[0]; ++j) {
-            data[i*dst->ne[0] + j] = j < n_visible ? 0.0f : -INFINITY;
+            for (int64_t j = 0; j < dst->ne[0]; ++j) {
+                data[i*dst->ne[0] + j] = j < n_visible ? 0.0f : -INFINITY;
+            }
+        }
+    } else if (dst->type == GGML_TYPE_F16) {
+        ggml_fp16_t * data = (ggml_fp16_t *) dst->data;
+        const ggml_fp16_t fp16_ninf = llama_cast<ggml_fp16_t>(-INFINITY);
+        const ggml_fp16_t fp16_zero = llama_cast<ggml_fp16_t>(0.0f);
+
+        for (int64_t i = 0; i < (int64_t) n_tokens; ++i) {
+            const int32_t n_visible = plan.n_visible[i];
+
+            for (int64_t j = 0; j < dst->ne[0]; ++j) {
+                data[i*dst->ne[0] + j] = j < n_visible ? fp16_zero : fp16_ninf;
+            }
        }
    }
 }
@@ -679,8 +693,7 @@ static ggml_tensor * dsv4_build_raw_kq_mask(
    GGML_ASSERT(n_stream > 0);
    GGML_ASSERT(n_tokens%n_stream == 0);

-    const bool use_fattn = cparams.flash_attn && (!cparams.kv_unified || n_stream == 1);
-    const auto type = use_fattn ? GGML_TYPE_F16 : GGML_TYPE_F32;
+    const auto type = cparams.flash_attn ? GGML_TYPE_F16 : GGML_TYPE_F32;

    ggml_tensor * res = ggml_new_tensor_4d(ctx, type, n_kv, n_tokens/n_stream, 1, n_stream);
    ggml_set_input(res);
@@ -814,6 +827,7 @@ static void dsv4_build_comp_inputs(
        llm_graph_input_dsv4::comp_input & inp,
        const llama_kv_cache_dsv4_context::comp_plan & plan,
        const char * name,
+        const llama_cparams & cparams,
        int64_t n_stream) {
    inp.state_pos = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_pos.size(), std::string("dsv4_") + name + "_state_pos");
    inp.state_persist_src_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_persist_src_idxs.size(), std::string("dsv4_") + name + "_state_persist_src_idxs");
@@ -828,7 +842,7 @@ static void dsv4_build_comp_inputs(
        GGML_ASSERT(n_stream > 0);
        GGML_ASSERT(n_tokens%n_stream == 0);

-        inp.kq_mask = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, plan.n_kv, n_tokens/n_stream, 1, n_stream);
+        inp.kq_mask = ggml_new_tensor_4d(ctx, cparams.flash_attn && strcmp(name, "lid") != 0 ? GGML_TYPE_F16 : GGML_TYPE_F32, plan.n_kv, n_tokens/n_stream, 1, n_stream);
        ggml_set_input(inp.kq_mask);
        ggml_set_name(inp.kq_mask, (std::string("dsv4_") + name + "_kq_mask").c_str());
    }
@@ -3076,9 +3090,9 @@ llm_graph_input_dsv4 * llm_graph_context::build_inp_dsv4() const {
    inp_raw->self_k_rot = raw_ctx->build_input_k_rot(ctx0);
    auto inp = std::make_unique<llm_graph_input_dsv4>(cparams, std::move(inp_raw), mctx_cur);

-    dsv4_build_comp_inputs(ctx0, inp->inp_csa, mctx_cur->get_csa_plan(ubatch), "csa", n_stream);
-    dsv4_build_comp_inputs(ctx0, inp->inp_hca, mctx_cur->get_hca_plan(ubatch), "hca", n_stream);
-    dsv4_build_comp_inputs(ctx0, inp->inp_lid, mctx_cur->get_lid_plan(ubatch), "lid", n_stream);
+    dsv4_build_comp_inputs(ctx0, inp->inp_csa, mctx_cur->get_csa_plan(ubatch), "csa", cparams, n_stream);
+    dsv4_build_comp_inputs(ctx0, inp->inp_hca, mctx_cur->get_hca_plan(ubatch), "hca", cparams, n_stream);
+    dsv4_build_comp_inputs(ctx0, inp->inp_lid, mctx_cur->get_lid_plan(ubatch), "lid", cparams, n_stream);
    inp->inp_csa.k_rot = mctx_cur->get_csa()->build_input_k_rot(ctx0);
    inp->inp_hca.k_rot = mctx_cur->get_hca()->build_input_k_rot(ctx0);
    inp->inp_lid.k_rot = mctx_cur->get_lid()->build_input_k_rot(ctx0);
@@ -184,32 +184,6 @@ static ggml_tensor * dsv4_with_zero_dep(ggml_context * ctx, ggml_tensor * t, ggm
    return ggml_add(ctx, t, zero);
 }

-// Raw SWA K is stored once, but compressed K/masks can carry a stream axis.
-// Repeat raw K at graph build time before concatenating raw and compressed K.
-static ggml_tensor * dsv4_repeat_streams(ggml_context * ctx, ggml_tensor * t, int64_t n_stream) {
-    if (t->ne[3] == n_stream) {
-        return t;
-    }
-
-    GGML_ASSERT(t->ne[3] == 1);
-    return ggml_repeat_4d(ctx, t, t->ne[0], t->ne[1], t->ne[2], n_stream);
-}
-
-static ggml_tensor * dsv4_build_kq_zero_bias(
-        ggml_context * ctx,
-        const llama_cparams & cparams,
-        ggml_tensor * kq_mask,
-        int64_t n_head) {
-    if (!cparams.kv_unified || !cparams.flash_attn || kq_mask->ne[3] == 1) {
-        return nullptr;
-    }
-
-    // Keep multi-stream unified DSV4 on the explicit attention path.
-    ggml_tensor * res = ggml_new_tensor_4d(ctx, GGML_TYPE_F32,
-            kq_mask->ne[0], kq_mask->ne[1], n_head, kq_mask->ne[3]);
-    return ggml_fill(ctx, res, 0.0f);
-}
-
 static constexpr int64_t DSV4_CSA_RATIO  = 4;
 static constexpr int64_t DSV4_HCA_RATIO  = 128;

@@ -624,7 +598,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_top_k_mask(
    ggml_tensor * top_k_3d = ggml_view_4d(ctx0, top_k, top_k->ne[0], top_k->ne[1], top_k->ne[3], 1,
            top_k->nb[1], top_k->nb[2], top_k->ne[3]*top_k->nb[3], 0);

-    ggml_tensor * zeros = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, top_k_3d->ne[0], top_k_3d->ne[1], top_k_3d->ne[2]);
+    ggml_tensor * zeros = ggml_new_tensor_4d(ctx0, cparams.flash_attn ? GGML_TYPE_F16 : GGML_TYPE_F32, 1, top_k_3d->ne[0], top_k_3d->ne[1], top_k_3d->ne[2]);
    zeros = ggml_fill(ctx0, zeros, 0.0f);

    ggml_tensor * kq_mask_top_k = ggml_set_rows(ctx0, kq_mask_all, zeros, top_k_3d);
@@ -681,26 +655,16 @@ ggml_tensor * llama_model_deepseek4::graph::build_csa_lid_attention(
            csa_k->nb[1], csa_k->nb[2], csa_k->nb[3], 0);
    cb(csa_k, "csa_comp_k", il);

-    raw_k = dsv4_repeat_streams(ctx0, raw_k, csa_k->ne[3]);
-
    ggml_tensor * k_all = ggml_concat(ctx0, raw_k, csa_k, 2);
    cb(k_all, "csa_k_all", il);

    ggml_tensor * raw_mask = inp_attn->get_kq_mask();
    ggml_tensor * csa_mask = build_top_k_mask(inp_csa.kq_mask, top_k, "csa_top_k_mask", il);
-    const bool use_fattn = cparams.flash_attn && (!cparams.kv_unified || csa_mask->ne[3] == 1);
-    if (use_fattn && csa_mask->type != GGML_TYPE_F16) {
-        csa_mask = ggml_cast(ctx0, csa_mask, GGML_TYPE_F16);
-    }
-    if (raw_mask->type != csa_mask->type) {
-        raw_mask = ggml_cast(ctx0, raw_mask, csa_mask->type);
-    }

    ggml_tensor * kq_mask = ggml_concat(ctx0, raw_mask, csa_mask, 0);
    cb(kq_mask, "csa_lid_kq_mask", il);

-    ggml_tensor * kq_b = dsv4_build_kq_zero_bias(ctx0, cparams, kq_mask, q->ne[1]);
-    ggml_tensor * out = build_attn_mha(q, k_all, k_all, kq_b, kq_mask, sinks, nullptr, kq_scale, il);
+    ggml_tensor * out = build_attn_mha(q, k_all, k_all, nullptr, kq_mask, sinks, nullptr, kq_scale, il);
    if (k_rot) {
        out = llama_mul_mat_hadamard(ctx0, out, k_rot);
    }
@@ -746,26 +710,16 @@ ggml_tensor * llama_model_deepseek4::graph::build_hca_attention(
            hca_k->nb[1], hca_k->nb[2], hca_k->nb[3], 0);
    cb(hca_k, "hca_comp_k", il);

-    raw_k = dsv4_repeat_streams(ctx0, raw_k, hca_k->ne[3]);
-
    ggml_tensor * k_all = ggml_concat(ctx0, raw_k, hca_k, 2);
    cb(k_all, "hca_k_all", il);

    ggml_tensor * raw_mask = inp_attn->get_kq_mask();
    ggml_tensor * hca_mask = inp_hca.kq_mask;
-    const bool use_fattn = cparams.flash_attn && (!cparams.kv_unified || hca_mask->ne[3] == 1);
-    if (use_fattn && hca_mask->type != GGML_TYPE_F16) {
-        hca_mask = ggml_cast(ctx0, hca_mask, GGML_TYPE_F16);
-    }
-    if (raw_mask->type != hca_mask->type) {
-        raw_mask = ggml_cast(ctx0, raw_mask, hca_mask->type);
-    }

    ggml_tensor * kq_mask = ggml_concat(ctx0, raw_mask, hca_mask, 0);
    cb(kq_mask, "hca_kq_mask", il);

-    ggml_tensor * kq_b = dsv4_build_kq_zero_bias(ctx0, cparams, kq_mask, q->ne[1]);
-    ggml_tensor * out = build_attn_mha(q, k_all, k_all, kq_b, kq_mask, sinks, nullptr, kq_scale, il);
+    ggml_tensor * out = build_attn_mha(q, k_all, k_all, nullptr, kq_mask, sinks, nullptr, kq_scale, il);
    if (k_rot) {
        out = llama_mul_mat_hadamard(ctx0, out, k_rot);
    }
@@ -800,10 +754,8 @@ ggml_tensor * llama_model_deepseek4::graph::build_raw_attention(
    ggml_tensor * kq_mask = inp_attn->get_kq_mask();

    ggml_tensor * k = mctx_cur->get_k(ctx0, il);
-    k = dsv4_repeat_streams(ctx0, k, kq_mask->ne[3]);

-    ggml_tensor * kq_b = dsv4_build_kq_zero_bias(ctx0, cparams, kq_mask, q->ne[1]);
-    ggml_tensor * out = build_attn_mha(q, k, k, kq_b, kq_mask, sinks, nullptr, kq_scale, il);
+    ggml_tensor * out = build_attn_mha(q, k, k, nullptr, kq_mask, sinks, nullptr, kq_scale, il);
    if (k_rot) {
        out = llama_mul_mat_hadamard(ctx0, out, k_rot);
    }
@@ -153,9 +153,19 @@ bool cli_context::init() {

    if (use_external_server) {
        spinner.reset();
-        if (!list_and_ask_models()) {
+        try {
+            if (!list_and_ask_models()) {
+                return false;
+            }
+        } catch (const json::parse_error & e) {
+            ui::show_error(e.what());
+            ui::show_message("This might be caused by an incorrect server-base endpoint URL");
+            return false;
+        } catch (const std::exception & e) {
+            ui::show_error(e.what());
            return false;
        }
+
        // restore the spinner for the next step
        spinner.emplace("Waiting for server...");
    }
@@ -20,8 +20,8 @@ struct clip_graph {
    const clip_hparams & hparams;
    projector_type proj_type;

-    // we only support single image per batch
-    const clip_image_f32 & img;
+    const clip_image_f32 & img; // for backward compat
+    const clip_image_f32_batch * img_batch = nullptr;

    const int patch_size;
    const int n_patches_x;
@@ -63,6 +63,12 @@ struct clip_graph {
    //
    void cb(ggml_tensor * cur0, const char * name, int il) const;

+    const clip_image_f32 & get_img(size_t idx) const {
+        GGML_ASSERT(img_batch);
+        GGML_ASSERT(idx < img_batch->entries.size());
+        return img_batch->entries[idx];
+    }
+
    // siglip2 naflex
    ggml_tensor * resize_position_embeddings(uint32_t interpolation_mode = DEFAULT_INTERPOLATION_MODE);

@@ -69,6 +69,7 @@ struct clip_hparams {
    std::vector<clip_image_size> image_res_candidates;
    int32_t preproc_min_tiles = 0;
    int32_t preproc_max_tiles = 0;
+    int32_t preproc_tile_size = 0; // local tile size (deepseek-ocr)
    resize_algo image_resize_algo_rf = RESIZE_ALGO_BICUBIC;
    resize_algo image_resize_algo_ov = RESIZE_ALGO_BILINEAR;
    pad_style image_pad_rf = PAD_CEIL;  // padding style for the refined image (e.g. llava-1.6)
@@ -1024,6 +1024,8 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
            GGML_ABORT("missing cgraph builder");
    }

+    builder->img_batch = &imgs;
+
    // TODO [QWEN_VIDEO]: improve this in the future
    builder->n_batch = imgs.entries.size();

@@ -1580,7 +1582,16 @@ struct clip_model_loader {
                        get_u32(KEY_SAM_N_HEAD, hparams.sam_n_head, true);
                        get_u32(KEY_SAM_N_EMBD, hparams.sam_n_embd, true);
                        get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size, true);
+                        hparams.preproc_min_tiles = 2;
+                        if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR) {
+                            hparams.preproc_max_tiles = 9;
+                            hparams.preproc_tile_size = 640;
+                            // the CLIP/ViT body runs its layernorms at 1e-5 (the SAM stage uses 1e-6)
+                            hparams.eps = 1e-5f;
+                        }
                        if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR2) {
+                            hparams.preproc_max_tiles = 6;
+                            hparams.preproc_tile_size = 768;
                            // qwen2 encoder is GQA, requires KEY_N_HEAD_KV
                            get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);
                        }
@@ -3251,6 +3262,9 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) {
            return (img->nx() / params.patch_size) / 2;
        case PROJECTOR_TYPE_STEP3VL:
            return img->nx() / (params.patch_size * params.n_merge);
+        case PROJECTOR_TYPE_DEEPSEEKOCR:
+        case PROJECTOR_TYPE_DEEPSEEKOCR2:
+            return (img->nx() / params.patch_size) / 4;
        default:
            break;
    }
@@ -3460,10 +3474,17 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
            // E.g., 64x64 -> 16x16 patches
            n_patches /= 16;

-            // build_global_local_features adds image newlines and view separator
-            // Formula: h*(w+1) + 1 where h = w = sqrt(n_patches)
-            int h = static_cast<int>(std::sqrt(static_cast<float>(n_patches)));
-            n_patches = h * (h + 1) + 1;
+            if (img->add_viewsep) {
+                // global view: one image-newline per token-row + trailing view separator
+                const int h = static_cast<int>(std::sqrt(static_cast<float>(n_patches)));
+                n_patches = h * (h + 1) + 1;
+            } else if (img->ny() >= img->nx() && img->ny() % img->nx() == 0) {
+                // tile row: one image-newline per token-row
+                const int grid_w = img->ny() / img->nx();
+                const int tile_patches = img->nx() / (patch_size * 4); // patches per tile side (SAM divides by 4)
+                const int h = tile_patches;
+                n_patches = (tile_patches * grid_w + 1) * h;
+            }
        } break;
        case PROJECTOR_TYPE_HUNYUANVL:
            {
@@ -4103,7 +4124,10 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
        case PROJECTOR_TYPE_DEEPSEEKOCR:
        case PROJECTOR_TYPE_DEEPSEEKOCR2:
            {
-                GGML_ASSERT(pos_w == pos_h);
+                GGML_ASSERT(
+                    (pos_w == pos_h) // overview image
+                    || (pos_h >= pos_w && pos_h % pos_w == 0) // tile images
+                );

                const int window = hparams.attn_window_size;
                const int pos = pos_w;
@@ -96,6 +96,8 @@ ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) {
    const int n_heads = hparams.sam_n_head;
    const int d_heads = n_embd / n_heads;
    const int window  = hparams.attn_window_size;
+    // SAM stage runs its layernorms at 1e-6
+    const float sam_eps = 1e-6f;

    ggml_tensor * inpL;

@@ -134,7 +136,7 @@ ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) {
        ggml_tensor * shortcut = cur;

        // layernorm1
-        cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
+        cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, sam_eps, il);

        const int64_t w0 = cur->ne[1];
        const int64_t h0 = cur->ne[2];
@@ -214,7 +216,7 @@ ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) {
        ggml_tensor * inpFF = cur;

        // layernorm2
-        cur = build_norm(inpFF, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
+        cur = build_norm(inpFF, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, sam_eps, il);

        // ffn
        cur = build_ffn(cur, layer.ff_up_w, layer.ff_up_b, nullptr, nullptr, layer.ff_down_w, layer.ff_down_b,
@@ -229,12 +231,12 @@ ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) {

    cur = ggml_conv_2d(ctx0, model.neck_0_w, cur, 1, 1, 0, 0, 1, 1);
    cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 2, 0, 3));
-    cur = build_norm(cur, model.neck_1_w, model.neck_1_b, NORM_TYPE_NORMAL, hparams.eps, -1);
+    cur = build_norm(cur, model.neck_1_w, model.neck_1_b, NORM_TYPE_NORMAL, sam_eps, -1);
    cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3));

    cur = ggml_conv_2d(ctx0, model.neck_2_w, cur, 1, 1, 1, 1, 1, 1);
    cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 2, 0, 3));
-    cur = build_norm(cur, model.neck_3_w, model.neck_3_b, NORM_TYPE_NORMAL, hparams.eps, -1);
+    cur = build_norm(cur, model.neck_3_w, model.neck_3_b, NORM_TYPE_NORMAL, sam_eps, -1);
    cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3));

    cur = ggml_conv_2d(ctx0, model.net_2, cur, 2, 2, 1, 1, 1, 1);
@@ -248,8 +250,40 @@ ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) {
 ggml_cgraph * clip_graph_deepseekocr::build() {
    // patch embedding
    ggml_tensor * inp_raw = build_inp_raw();
+
+    bool is_overview = img.add_viewsep;
+    int n_tiles_per_row = 0;
+
+    // note: we expect either a batch of rows or a batch of overviews, but not a mix of both
+
+    if (!is_overview) {
+        // handle the case where we have a batch of rows
+        // sanity check
+        for (auto & entry : img_batch->entries) {
+            if (entry.add_viewsep) {
+                throw std::runtime_error("DeepSeek-OCR: mixed overview and non-overview images in batch");
+            }
+            if (entry.nx() != img.nx() || entry.ny() != img.ny()) {
+                throw std::runtime_error("DeepSeek-OCR: mixed image sizes in batch");
+            }
+        }
+
+        GGML_ASSERT(img.ny() >= img.nx());
+        GGML_ASSERT(img.ny() % img.nx() == 0);
+        n_tiles_per_row = img.ny() / img.nx();
+
+        // input shape: [tile_size, tile_size * n_tiles_per_row, 3]
+        // we want to reshape it to [tile_size, tile_size, 3, n_tiles_per_row]
+        inp_raw = ggml_reshape_4d(ctx0, inp_raw, img.nx(), img.nx(), n_tiles_per_row, 3);
+        inp_raw = ggml_cont(ctx0, ggml_permute(ctx0, inp_raw, 0, 1, 3, 2));
+    }
+
    ggml_tensor * sam_out = build_sam(inp_raw);

+    if (!is_overview) {
+        n_batch = n_tiles_per_row;
+    }
+
    const int clip_n_patches = sam_out->ne[0] * sam_out->ne[1];

    ggml_tensor * clip_out;
@@ -257,7 +291,9 @@ ggml_cgraph * clip_graph_deepseekocr::build() {
    {
        ggml_tensor * inp;

-        inp = ggml_reshape_2d(ctx0, sam_out, clip_n_patches, sam_out->ne[2]);
+        // sam_out: [patch_h, patch_w, n_embd, n_batch]
+        // -> [n_embd, clip_n_patches, n_batch]
+        inp = ggml_reshape_3d(ctx0, sam_out, clip_n_patches, sam_out->ne[2], sam_out->ne[3]);
        inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3));

        ggml_tensor * new_pos_embd = model.position_embeddings;
@@ -281,8 +317,11 @@ ggml_cgraph * clip_graph_deepseekocr::build() {
            n_pos        = tgt_size * tgt_size + 1;
        }

-        // add CLS token
-        inp = ggml_concat(ctx0, model.class_embedding, inp, 1);
+        // add CLS token per batch item
+        // inp: [n_embd, clip_n_patches, n_batch]
+        // class_embedding: [n_embd] -> [n_embd, 1, n_batch]
+        ggml_tensor * cls_embd = ggml_repeat_4d(ctx0, model.class_embedding, n_embd, 1, n_batch, 1);
+        inp = ggml_concat(ctx0, cls_embd, inp, 1);

        // for selecting learned pos embd, used by ViT
        ggml_tensor * positions        = ggml_cast(ctx0, ggml_arange(ctx0, 0, n_pos, 1), GGML_TYPE_I32);
@@ -294,25 +333,56 @@ ggml_cgraph * clip_graph_deepseekocr::build() {
        clip_out = cur;
    }

+    // sam_out: [patch_h, patch_w, n_embd, n_batch]
+    // -> [n_embd, clip_n_patches, n_batch]
    sam_out  = ggml_cont(ctx0, ggml_permute(ctx0, sam_out, 1, 2, 0, 3));
-    sam_out  = ggml_reshape_2d(ctx0, sam_out, sam_out->ne[0], clip_n_patches);
-    clip_out = ggml_view_2d(ctx0, clip_out, n_embd, clip_n_patches, clip_out->nb[1], clip_out->nb[1]);
+    sam_out  = ggml_reshape_3d(ctx0, sam_out, sam_out->ne[0], clip_n_patches, n_batch);
+
+    // clip_out: [n_embd, n_pos, n_batch] where n_pos = clip_n_patches + 1 (CLS)
+    // strip CLS token: skip first position, view only the patch tokens
+    clip_out = ggml_view_3d(ctx0, clip_out, n_embd, clip_n_patches, n_batch,
+                            clip_out->nb[1], clip_out->nb[2], clip_out->nb[1]);

    ggml_tensor * cur;
    cur = ggml_concat(ctx0, clip_out, sam_out, 0);
    cur = ggml_mul_mat(ctx0, model.mm_fc_w, cur);
    cur = ggml_add(ctx0, cur, model.mm_fc_b);

-    const auto h     = static_cast<int>(std::sqrt(static_cast<float>(cur->ne[1])));
-    const auto w     = h;
-    const auto n_dim = cur->ne[0];
+    if (is_overview) {
+        // global view: weave one newline per row + trailing view separator
+        const auto h     = static_cast<int>(std::sqrt(static_cast<float>(cur->ne[1])));
+        const auto w     = h;
+        const auto n_dim = cur->ne[0];

-    ggml_tensor * imgnl;
+        ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, h, 1);
+        cur = ggml_reshape_3d(ctx0, cur, n_dim, w, h);
+        cur = ggml_reshape_2d(ctx0, ggml_concat(ctx0, cur, imgnl, 1), n_dim, (w + 1) * h);
+        cur = ggml_concat(ctx0, cur, model.view_seperator, 1);  // (n_dim, h*(w+1) + 1)
+    } else {
+        // tile row: interleave tiles within each row, add newline per row
+        const int grid_x      = static_cast<int>(std::sqrt(static_cast<float>(clip_n_patches)));
+        const int grid_y      = grid_x;
+        const auto n_dim      = cur->ne[0];

-    imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, h, 1);
-    cur   = ggml_reshape_3d(ctx0, cur, n_dim, w, h);
-    cur   = ggml_reshape_2d(ctx0, ggml_concat(ctx0, cur, imgnl, 1), n_dim, (w + 1) * h);
-    cur   = ggml_concat(ctx0, cur, model.view_seperator, 1);  // (n_dim, h*(w+1) + 1)
+        // (n_dim, clip_n_patches, n_batch) -> (n_dim, grid_x, grid_y, n_batch)
+        cur = ggml_reshape_4d(ctx0, cur, n_dim, grid_x, grid_y, n_batch);
+
+        // tiles: re-order from A.row0 A.row1 B.row0 B.row1 ...
+        //        to A.row0 B.row0 A.row1 B.row1 ...
+        //        then add nl: A.row0 B.row0 [nl] A.row1 B.row1 [nl] ...
+        // interleave tiles: (n_dim, grid_x, grid_y, n_batch) -> (n_dim, grid_x, n_batch, grid_y)
+        cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 1, 3, 2));
+
+        // merge: (n_dim, grid_x, n_batch, grid_y) -> (n_dim, grid_x*n_batch, grid_y, 1)
+        cur = ggml_reshape_4d(ctx0, cur, n_dim, grid_x * n_batch, grid_y, 1);
+
+        // append newline per row: (n_dim, grid_x*n_batch+1, grid_y, 1)
+        ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, grid_y, 1);
+        cur = ggml_concat(ctx0, cur, imgnl, 1);
+
+        // flatten: (n_dim, (grid_x*n_batch+1)*grid_y)
+        cur = ggml_reshape_2d(ctx0, cur, n_dim, (grid_x * n_batch + 1) * grid_y);
+    }

    cb(cur, "dsocr_output", -1);

@@ -127,6 +127,7 @@ struct clip_graph_deepseekocr : clip_graph {
    clip_graph_deepseekocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
    ggml_cgraph * build() override;
    ggml_tensor * build_sam(ggml_tensor * inp); // build the SAM model
+    // bool support_batch() const override { return true; } // TODO: support batch for DeepSeek-OCR v1
 };

 struct clip_graph_deepseekocr2 : clip_graph_deepseekocr {
@@ -1107,44 +1107,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_internvl::preprocess(const clip_i
 // mtmd_image_preprocessor_deepseekocr
 //

-mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) {
-    static constexpr int native_resolutions[] = { 1024 /* base */, 1280 /* large */ };
-    // TODO: support 512 (tiny) and 640 (small) once we have eval data for them
-
-    const int64_t orig_area = static_cast<int64_t>(img.get_size().area());
-
-    size_t  mode_i   = 0;
-    int64_t min_diff = std::numeric_limits<int64_t>::max();
-    for (size_t i = 0; i < std::size(native_resolutions); i++) {
-        const int64_t r    = native_resolutions[i];
-        const int64_t diff = std::abs(orig_area - r * r);
-        if (diff < min_diff) {
-            mode_i   = i;
-            min_diff = diff;
-        }
-    }
-    const int image_size = native_resolutions[mode_i];
-
-    // Aspect-preserving fit-and-pad. Pillow bicubic + PAD_NEAREST for
-    // byte-parity with the upstream deepseek-ai/DeepSeek-OCR HF preprocessor.
-    clip_image_u8 padded;
-    img_tool::resize(img, padded, {image_size, image_size}, RESIZE_ALGO_BICUBIC_PILLOW,
-                     PAD_NEAREST, hparams.image_pad_color);
-    mtmd_image_preproc_out output;
-    output.append_overview(hparams, padded, true);
-    output.grid_x = 0;
-    output.grid_y = 0;
-    // TODO @ngxson : support slicing for DeepSeek-OCR, to do in another PR
-    return output;
-}
-
-//
-// mtmd_image_preprocessor_deepseekocr2
-//
-
-// candidate tile grids (cols, rows) with min_tiles <= cols*rows <= max_tiles
-// sorted by tile count
-std::vector<clip_image_size> mtmd_image_preprocessor_deepseekocr2::get_target_ratios() {
+std::vector<clip_image_size> mtmd_image_preprocessor_deepseekocr::get_target_ratios() const {
    std::vector<clip_image_size> ratios;
    for (int n = min_tiles; n <= max_tiles; n++) {
        for (int w = 1; w <= n; w++) {
@@ -1171,13 +1134,11 @@ std::vector<clip_image_size> mtmd_image_preprocessor_deepseekocr2::get_target_ra
    return ratios;
 }

-// pick the grid whose aspect ratio is closest to the image
-// on a tie, prefer the larger grid when the image fits
-clip_image_size mtmd_image_preprocessor_deepseekocr2::find_closest_aspect_ratio(
+clip_image_size mtmd_image_preprocessor_deepseekocr::find_closest_aspect_ratio(
    float                                aspect_ratio,
    const std::vector<clip_image_size> & target_ratios,
    int                                  width,
-    int                                  height) {
+    int                                  height) const {
    float           best_ratio_diff = std::numeric_limits<float>::max();
    clip_image_size best_ratio      = { 1, 1 };
    const float     area            = static_cast<float>(width * height);
@@ -1198,37 +1159,69 @@ clip_image_size mtmd_image_preprocessor_deepseekocr2::find_closest_aspect_ratio(
    return best_ratio;
 }

-mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr2::preprocess(const clip_image_u8 & img) {
-    // emit 768x768 local tiles when the image is larger than a tile in either
-    // dimension, then always a 1024x1024 global view. order: [tiles..., global].
-
+mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) {
    mtmd_image_preproc_out output;
+    int grid_w = 0;
+    int grid_h = 0;
    const auto img_size = img.get_size();
+
+    // global view: aspect-preserving fit-and-pad to base_size
+    clip_image_u8 padded;
+    img_tool::resize(img, padded,
+                    { base_size, base_size },
+                    RESIZE_ALGO_BICUBIC_PILLOW,
+                    PAD_NEAREST,
+                    hparams.image_pad_color);
+    output.append_overview(hparams, padded, true);
+    output.overview.add_viewsep = true;
+
+    // if this condition doesn't hold, the output is overview only, no tiles
    if (img_size.width > tile_size || img_size.height > tile_size) {
        const float           aspect_ratio  = static_cast<float>(img_size.width) / img_size.height;
        const auto            target_ratios = get_target_ratios();
-        const clip_image_size grid          = find_closest_aspect_ratio(aspect_ratio, target_ratios, img_size.width, img_size.height);
+        const clip_image_size grid =
+            find_closest_aspect_ratio(aspect_ratio, target_ratios, img_size.width, img_size.height);
+        grid_w = grid.width;
+        grid_h = grid.height;

-        // stretch onto the grid (no aspect preserve), then crop tiles row-major.
        clip_image_u8 refined;
-        img_tool::resize(img, refined, { tile_size * grid.width, tile_size * grid.height },
-                         RESIZE_ALGO_BICUBIC_PILLOW, PAD_NONE);
+        img_tool::resize(img, refined, { tile_size * grid_w, tile_size * grid_h }, RESIZE_ALGO_BICUBIC_PILLOW,
+                         PAD_NONE);

-        for (int row = 0; row < grid.height; row++) {
-            for (int col = 0; col < grid.width; col++) {
-                clip_image_u8 tile;
-                img_tool::crop(refined, tile, col * tile_size, row * tile_size, tile_size, tile_size);
-                output.append(hparams, tile, true);
+        for (int row = 0; row < grid_h; row++) {
+            if (fuse_row) {
+                // concat all tiles in this row into a single image, along the H axis
+                // output image size: w = tile_size, h = tile_size * grid_w
+                // this is to ensure the whole row is always processed together
+                clip_image_u8 row_img;
+                row_img.set_size({tile_size, tile_size * grid_w}, false);
+                for (int col = 0; col < grid_w; col++) {
+                    for (int py = 0; py < tile_size; py++) {
+                        for (int px = 0; px < tile_size; px++) {
+                            row_img.set_pixel(px, col * tile_size + py,
+                                              refined.get_pixel(col * tile_size + px, row * tile_size + py));
+                        }
+                    }
+                }
+                output.append(hparams, row_img, true);
+            } else {
+                for (int col = 0; col < grid_w; col++) {
+                    clip_image_u8 tile;
+                    img_tool::crop(refined, tile, col * tile_size, row * tile_size, tile_size, tile_size);
+                    output.append(hparams, tile, true);
+                }
            }
        }
+        if (fuse_row) {
+            grid_w = 1; // each fused row is one image; a single output column
+        }
    }

-    // global view: aspect-preserving fit-and-pad to base_size.
-    clip_image_u8 padded;
-    img_tool::resize(img, padded, { base_size, base_size }, RESIZE_ALGO_BICUBIC_PILLOW,
-                     PAD_NEAREST, hparams.image_pad_color);
-    output.append_overview(hparams, padded, true);
-    output.overview.add_viewsep = true;
+    LOG_DBG("%s: grid size: %d x %d (%d tiles) + global view\n", __func__, grid_w, grid_h, grid_w * grid_h);
+    LOG_DBG("%s: overview size: %d x %d\n", __func__, padded.get_size().width, padded.get_size().height);
+
+    output.grid_x = grid_w;
+    output.grid_y = grid_h;
    return output;
 }

@@ -160,29 +160,29 @@ struct mtmd_image_preprocessor_internvl : mtmd_image_preprocessor_llava_uhd {
    mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
 };

+// DeepSeek-OCR (v1/v2) global view + optional local tile grid
 struct mtmd_image_preprocessor_deepseekocr : mtmd_image_preprocessor {
-    mtmd_image_preprocessor_deepseekocr(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
-    mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
-};
-
-// DeepSeek-OCR-2: a 1024x1024 global view, plus InternVL-style 768x768 local
-// tiles when the image is larger than a tile in either dimension.
-struct mtmd_image_preprocessor_deepseekocr2 : mtmd_image_preprocessor {
-    static constexpr int base_size = 1024; // global view
-    static constexpr int tile_size = 768;  // local tile
-    static constexpr int min_tiles = 2;
-    static constexpr int max_tiles = 6;
-
-    mtmd_image_preprocessor_deepseekocr2(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
+    mtmd_image_preprocessor_deepseekocr(const clip_ctx * ctx)
+        : mtmd_image_preprocessor(ctx),
+            fuse_row(clip_get_projector_type(ctx) == PROJECTOR_TYPE_DEEPSEEKOCR),
+          base_size(hparams.image_size),
+          tile_size(hparams.preproc_tile_size),
+          min_tiles(hparams.preproc_min_tiles),
+          max_tiles(hparams.preproc_max_tiles) {}
    mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;

 private:
-    static std::vector<clip_image_size> get_target_ratios();
-    static clip_image_size              find_closest_aspect_ratio(
-        float                                aspect_ratio,
-        const std::vector<clip_image_size> & target_ratios,
-        int                                  width,
-        int                                  height);
+    bool fuse_row; // v1 fuses a tile-row into one image; v2 keeps tiles separate
+    int base_size; // global view
+    int tile_size; // each tile
+    int min_tiles;
+    int max_tiles;
+
+    std::vector<clip_image_size> get_target_ratios() const;
+    clip_image_size find_closest_aspect_ratio(
+            float aspect_ratio,
+            const std::vector<clip_image_size> & target_ratios,
+            int width, int height) const;
 };

 // custom image preprocessing for Step3VL
@@ -618,15 +618,10 @@ struct mtmd_context {
                    image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
                } break;
            case PROJECTOR_TYPE_DEEPSEEKOCR:
-                {
-                    img_end = "\n"; // prevent empty batch on llama-server
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_deepseekocr>(ctx_v);
-                    ov_img_first = false;
-                } break;
            case PROJECTOR_TYPE_DEEPSEEKOCR2:
                {
                    img_end = "\n"; // prevent empty batch on llama-server
-                    image_preproc = std::make_unique<mtmd_image_preprocessor_deepseekocr2>(ctx_v);
+                    image_preproc = std::make_unique<mtmd_image_preprocessor_deepseekocr>(ctx_v);
                    ov_img_first = false;
                } break;
            case PROJECTOR_TYPE_HUNYUANVL:
@@ -1132,6 +1127,7 @@ struct mtmd_tokenizer {

                // add slices (or tiles)
                if (!chunks.empty()) {
+                    LOG_DBG("%s: adding %d slices (%d rows x %d cols)\n", __func__, (int)chunks.size(), n_row, n_col);
                    GGML_ASSERT((int)chunks.size() == n_row * n_col);
                    add_text(ctx->tok_slices_start);
                    for (int y = 0; y < n_row; y++) {
@@ -1174,7 +1170,6 @@ struct mtmd_tokenizer {
                    cur.entries.emplace_back(std::move(ov_chunk));
                    add_text(ctx->tok_ov_img_end);
                }
-
            } else {

                if (preproc_out.entries.size() == 0) {
@@ -29,12 +29,15 @@ class ModelSpec:
    mmproj_arg: str
    model_default: str
    mmproj_default: str
-    prompt: str = "Free OCR. "
+    prompt: str = "Free OCR."
    n_predict: int = 512
    n_ctx: int | None = None
    # Unlimited-OCR's "document parsing" prompt emits <|det|> grounding markup that
    # the HF reference strips in result.md; drop it before scoring to match.
    strip_grounding: bool = False
+    # v2/Unlimited loop on hard tiles; DRY caps it the way HF's
+    # no_repeat_ngram_size does. v1 scores fine without it.
+    dry: bool = False


@dataclass
@@ -69,6 +72,9 @@ MODELS = {
        model_arg="--llama-model-2", mmproj_arg="--mmproj-2",
        model_default="gguf_models/deepseek-ai/deepseek-ocr-2-bf16.gguf",
        mmproj_default="gguf_models/deepseek-ai/mmproj-deepseek-ocr-2-bf16.gguf",
+        # v2 keeps generating past 512 on multi-tile; give it room to match the HF ref.
+        n_predict=2048,
+        dry=True,
    ),
    "unlimited": ModelSpec(
        key="unlimited", label="Unlimited-OCR",
@@ -83,6 +89,7 @@ MODELS = {
        n_predict=4096,
        n_ctx=16384,
        strip_grounding=True,
+        dry=True,
    ),
 }

@@ -91,7 +98,9 @@ CASES = [
        model_key="v1", label="single-view scan",
        image="tools/mtmd/test-1.jpeg",
        ground_truth="tools/mtmd/tests/test-1-ground-truth.txt",
-        hf_cer=0.3030, hf_chrf=67.52, cer_tol=0.02, chrf_tol=2.0,
+        # Fragile image: the HF ref itself swings ~0.286-0.314 across precision
+        # configs -- hence the wide tol. llama.cpp bf16 ~0.322/63.8.
+        hf_cer=0.3140, hf_chrf=67.57, cer_tol=0.04, chrf_tol=5.0,
    ),
    TestCase(
        model_key="v2", label="single-view scan",
@@ -103,6 +112,24 @@ CASES = [
        # is one pixel off and lands at ~0.69 instead.
        hf_cer=0.7761, hf_chrf=28.70, cer_tol=0.12, chrf_tol=8.0,
    ),
+    TestCase(
+        model_key="v1", label="multi-tile (dynamic resolution)",
+        image="tools/mtmd/tests/test-1-positive.png",
+        ground_truth="tools/mtmd/tests/test-1-ground-truth.txt",
+        # 429x806 -- 806 > 640 triggers the v1 "Gundam" path: (1,2) grid ->
+        # 2 local 640 tiles + 1 global 1024 view. Regression guard for the
+        # tiling preprocessor -- a broken tile path craters the score.
+        # hf_cer/hf_chrf are HF v1's measured scores -- it reads this clean crop exactly.
+        hf_cer=0.0000, hf_chrf=100.00, cer_tol=0.03, chrf_tol=3.0,
+    ),
+    TestCase(
+        model_key="v2", label="multi-tile (dynamic resolution)",
+        image="tools/mtmd/tests/test-1-positive.png",
+        ground_truth="tools/mtmd/tests/test-1-ground-truth.txt",
+        # 429x806 -- 806 > 768 triggers the v2 path: (1,2) grid ->
+        # 2 local 768 tiles + 1 global 1024 view = 545 image tokens.
+        hf_cer=0.0236, hf_chrf=97.05, cer_tol=0.03, chrf_tol=3.0,
+    ),
    TestCase(
        model_key="unlimited", label="single-view scan",
        image="tools/mtmd/test-1.jpeg",
@@ -180,14 +207,17 @@ def run_mtmd_cli(spec: "ModelSpec", model_path, mmproj_path, image_path, bin_pat
        "--flash-attn", "off",  # match the HF "eager" attention reference
        "--no-warmup",
        "-n", str(spec.n_predict),  # cap loops on hard images (KV would otherwise fill)
+    ]
+    if spec.dry:
        # HF decodes with no_repeat_ngram_size; llama.cpp's analog is DRY.
        # Default DRY breakers include "\n", so they are cleared below.
-        "--dry-multiplier", "0.8",
-        "--dry-base", "1.75",
-        "--dry-allowed-length", "2",
-        "--dry-penalty-last-n", "-1",
-        "--dry-sequence-breaker", "none",
-    ]
+        cmd += [
+            "--dry-multiplier", "0.8",
+            "--dry-base", "1.75",
+            "--dry-allowed-length", "2",
+            "--dry-penalty-last-n", "-1",
+            "--dry-sequence-breaker", "none",
+        ]
    if spec.n_ctx is not None:
        cmd += ["-c", str(spec.n_ctx)]
    logger.debug(f"  command: {' '.join(cmd)}")
@@ -175,6 +175,15 @@ bool server_http_context::init(const common_params & params) {
    // Middlewares
    //

+    // Frontend paths - all embedded UI assets
+    static const std::unordered_set<std::string> frontend_paths = []() {
+        std::unordered_set<std::string> paths { "/" };
+        for (const llama_ui_asset & a : llama_ui_get_assets()) {
+            paths.insert("/" + a.name);
+        }
+        return paths;
+    }();
+
    // Public endpoints - API routes plus all embedded UI assets
    static const std::unordered_set<std::string> get_public_endpoints = []() {
        std::unordered_set<std::string> endpoints {
@@ -182,11 +191,8 @@ bool server_http_context::init(const common_params & params) {
            "/v1/health",
            "/models",
            "/v1/models",
-            "/",
        };
-        for (const llama_ui_asset & a : llama_ui_get_assets()) {
-            endpoints.insert("/" + a.name);
-        }
+        endpoints.insert(frontend_paths.begin(), frontend_paths.end());
        return endpoints;
    }();

@@ -239,18 +245,9 @@ bool server_http_context::init(const common_params & params) {

    auto middleware_server_state = [this](const httplib::Request & req, httplib::Response & res) {
        if (!is_ready.load()) {
-#if defined(LLAMA_UI_HAS_ASSETS)
-            if (const auto tmp = string_split<std::string>(req.path, '.');
-                req.path == "/" || (!tmp.empty() && tmp.back() == "html")) {
-                if (const llama_ui_asset * a = llama_ui_find_asset("loading.html")) {
-                    res.status = 503;
-                    res.set_content(reinterpret_cast<const char*>(a->data), a->size, "text/html; charset=utf-8");
-                    return false;
-                }
+            if (frontend_paths.count(req.path)) {
+                return true; // frontend asset, allow it to load and show "loading"
            }
-#else
-            (void)req;
-#endif
            // no endpoints are allowed to be accessed when the server is not ready
            // this is to prevent any data races or inconsistent states
            res.status = 503;
@@ -9,10 +9,10 @@ struct server_tool {
    bool permission_write = false;

    virtual ~server_tool() = default;
-    virtual json get_definition() = 0;
-    virtual json invoke(json params) = 0;
+    virtual json get_definition() const = 0;
+    virtual json invoke(json params) const = 0;

-    json to_json();
+    json to_json() const;
 };

 struct server_tools {
@@ -0,0 +1,125 @@
+import os
+
+import pytest
+from utils import *
+
+server: ServerProcess
+
+# project root, used as the search directory for grep_search/file_glob_search
+PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))
+
+# marker for the grep_search test to find in this file
+GREP_MARKER = "llama_cpp_test_tools_builtin_marker_grep_search"
+
+
+@pytest.fixture(autouse=True)
+def create_server():
+    global server
+    server = ServerPreset.router()
+    server.server_tools = "all"
+
+
+def call_tool(name: str, params: dict) -> dict:
+    res = server.make_request("POST", "/tools", data={"tool": name, "params": params})
+    assert res.status_code == 200, res.body
+    assert "error" not in res.body, res.body
+    return res.body
+
+
+def call_tool_expect_error(name: str, params: dict) -> str:
+    res = server.make_request("POST", "/tools", data={"tool": name, "params": params})
+    assert res.status_code == 200, res.body
+    assert "error" in res.body, res.body
+    return res.body["error"]
+
+
+def test_tools_builtin_grep_search():
+    global server
+    server.start()
+
+    res = call_tool("grep_search", {
+        "path": PROJECT_ROOT,
+        "pattern": GREP_MARKER,
+        "include": "test_tools_builtin.py",  # bare pattern -> matches basename at any depth
+    })
+    text = res["plain_text_response"]
+    assert "test_tools_builtin.py" in text
+    assert GREP_MARKER in text
+    assert "Total matches: 1" in text
+
+
+def test_tools_builtin_read_file():
+    global server
+    server.start()
+
+    this_file = os.path.join(PROJECT_ROOT, "tools", "server", "tests", "unit", "test_tools_builtin.py")
+    res = call_tool("read_file", {"path": this_file})
+    text = res["plain_text_response"]
+    assert GREP_MARKER in text
+    assert "def test_tools_builtin_read_file" in text
+
+
+def test_tools_builtin_write_then_edit_file():
+    global server
+    server.start()
+
+    log_path = os.path.join(PROJECT_ROOT, "test.log")
+    try:
+        write_res = call_tool("write_file", {"path": log_path, "content": "line1\nline2\nline3\n"})
+        assert write_res["result"] == "file written successfully"
+
+        read_before = call_tool("read_file", {"path": log_path})
+        assert read_before["plain_text_response"] == "line1\nline2\nline3\n"
+
+        edit_res = call_tool("edit_file", {
+            "path": log_path,
+            "edits": [
+                {"old_text": "line2", "new_text": "line2-edited"},
+                {"old_text": "line3\n", "new_text": "line3\nline4\n"},
+            ],
+        })
+        assert edit_res["result"] == "file edited successfully"
+        assert edit_res["edits_applied"] == 2
+
+        read_after = call_tool("read_file", {"path": log_path})
+        assert read_after["plain_text_response"] == "line1\nline2-edited\nline3\nline4\n"
+    finally:
+        if os.path.exists(log_path):
+            os.remove(log_path)
+
+
+def test_tools_builtin_edit_file_rejects_non_unique_old_text():
+    global server
+    server.start()
+
+    log_path = os.path.join(PROJECT_ROOT, "test.log")
+    try:
+        call_tool("write_file", {"path": log_path, "content": "dup\ndup\n"})
+        err = call_tool_expect_error("edit_file", {
+            "path": log_path,
+            "edits": [{"old_text": "dup", "new_text": "changed"}],
+        })
+        assert "unique" in err
+    finally:
+        if os.path.exists(log_path):
+            os.remove(log_path)
+
+
+def test_tools_builtin_edit_file_rejects_overlapping_edits():
+    global server
+    server.start()
+
+    log_path = os.path.join(PROJECT_ROOT, "test.log")
+    try:
+        call_tool("write_file", {"path": log_path, "content": "line1\nline2\n"})
+        err = call_tool_expect_error("edit_file", {
+            "path": log_path,
+            "edits": [
+                {"old_text": "line1\nline2", "new_text": "a"},
+                {"old_text": "line2", "new_text": "b"},
+            ],
+        })
+        assert "overlap" in err
+    finally:
+        if os.path.exists(log_path):
+            os.remove(log_path)
@@ -113,6 +113,7 @@ class ServerProcess:
    ui_mcp_proxy: bool = False
    backend_sampling: bool = False
    gcp_compat: bool = False
+    server_tools: str | None = None

    # session variables
    process: subprocess.Popen | None = None
@@ -256,6 +257,8 @@ class ServerProcess:
            server_args.append("--no-cache-idle-slots")
        if self.ui_mcp_proxy:
            server_args.append("--ui-mcp-proxy")
+        if self.server_tools:
+            server_args.extend(["--tools", self.server_tools])
        if self.backend_sampling:
            server_args.append("--backend_sampling")
        if self.gcp_compat:
@@ -187,7 +187,6 @@ int main(int argc, char ** argv) {
        struct required_check { const char * label; match_fn match; bool found; };
        required_check checks[] = {
            { "index.html",           exact("index.html"),           false },
-            { "loading.html",         exact("loading.html"),         false },
            { "manifest.webmanifest", exact("manifest.webmanifest"), false },
            { "sw.js",                exact("sw.js"),                false },
            { "build.json",           exact("build.json"),           false },
@@ -1,10 +1,11 @@
 <script lang="ts">
-	import { AlertTriangle, RefreshCw } from '@lucide/svelte';
+	import { AlertTriangle, Loader2, RefreshCw } from '@lucide/svelte';
 	import { fadeInView } from '$lib/actions/fade-in-view.svelte';
 	import * as Alert from '$lib/components/ui/alert';
-	import { serverError, serverLoading, serverStore } from '$lib/stores/server.svelte';
+	import { serverError, serverLoading, serverStatus, serverStore } from '$lib/stores/server.svelte';

 	let hasError = $derived(!!serverError());
+	let isLoadingModel = $derived(serverStatus() === 503);
 </script>

 {#if hasError}
@@ -12,23 +13,31 @@
 		class="pointer-events-auto mx-auto mb-4 max-w-[48rem] px-1"
 		use:fadeInView={{ y: 10, duration: 250 }}
 	>
-		<Alert.Root variant="destructive">
-			<AlertTriangle class="h-4 w-4" />
+		<Alert.Root variant={isLoadingModel ? 'default' : 'destructive'}>
+			{#if isLoadingModel}
+				<Loader2 class="h-4 w-4 animate-spin" />
+			{:else}
+				<AlertTriangle class="h-4 w-4" />
+			{/if}

 			<Alert.Title class="flex items-center justify-between">
-				<span>Server unavailable</span>
+				<span>{isLoadingModel ? 'Loading model' : 'Server unavailable'}</span>

-				<button
-					onclick={() => serverStore.fetch()}
-					disabled={serverLoading()}
-					class="flex items-center gap-1.5 rounded-lg bg-destructive/20 px-2 py-1 text-xs font-medium hover:bg-destructive/30 disabled:opacity-50"
-				>
-					<RefreshCw class="h-3 w-3 {serverLoading() ? 'animate-spin' : ''}" />
-					{serverLoading() ? 'Retrying...' : 'Retry'}
-				</button>
+				{#if !isLoadingModel}
+					<button
+						onclick={() => serverStore.fetch()}
+						disabled={serverLoading()}
+						class="flex items-center gap-1.5 rounded-lg bg-destructive/20 px-2 py-1 text-xs font-medium hover:bg-destructive/30 disabled:opacity-50"
+					>
+						<RefreshCw class="h-3 w-3 {serverLoading() ? 'animate-spin' : ''}" />
+						{serverLoading() ? 'Retrying...' : 'Retry'}
+					</button>
+				{/if}
 			</Alert.Title>

-			<Alert.Description>{serverError()}</Alert.Description>
+			{#if !isLoadingModel}
+				<Alert.Description>{serverError()}</Alert.Description>
+			{/if}
 		</Alert.Root>
 	</div>
 {/if}
@@ -5,7 +5,7 @@
 	let {
 		ref = $bindable(null),
 		class: className,
-		sideOffset = 0,
+		sideOffset = 4,
 		side = 'top',
 		children,
 		arrowClasses,
@@ -258,12 +258,6 @@ export const GLOB_PATTERNS: string[] = [
 	'**/*.{js,css,html,ico,svg,png,webp,woff,woff2,json,webmanifest}'
 ];

-// loading.html is the model loading page served by llama-server itself.
-// The SvelteKit PWA manifest transform strips the html extension from every
-// precache entry to match clean URLs, but loading.html is a plain static asset
-// with no clean URL, so static servers answer 404 and the SW install fails.
-export const GLOB_IGNORES: string[] = ['**/loading.html'];
-
 export const SW_CONFIG = {
 	CHECK_INTERVAL_MS: 60000,
 	UPDATE_FETCH_OPTIONS: {
@@ -317,7 +311,6 @@ export const SVELTEKIT_PWA_OPTIONS: SvelteKitPWAOptions = {
 		// Uses '**/' because SvelteKit outputs files under _app/immutable/
 		// subdirectories.
 		globPatterns: GLOB_PATTERNS,
-		globIgnores: GLOB_IGNORES,
 		maximumFileSizeToCacheInBytes: CACHE_SETTINGS.MAX_FILE_SIZE_BYTES,

 		// Prevent @vite-pwa/sveltekit from auto-adding a NavigationRoute by
@@ -1115,21 +1115,18 @@ class ConversationsStore {
 	}

 	/**
-	 * Downloads a conversation as JSON file.
+	 * Downloads a single conversation as a JSONL file, serializing the full message tree.
 	 * @param convId - The conversation ID to download
 	 */
 	async downloadConversation(convId: string): Promise<void> {
-		let conversation: DatabaseConversation | null;
-		let messages: DatabaseMessage[];
+		const conversation =
+			this.activeConversation?.id === convId
+				? this.activeConversation
+				: await DatabaseService.getConversation(convId);

-		if (this.activeConversation?.id === convId) {
-			conversation = this.activeConversation;
-			messages = this.activeMessages;
-		} else {
-			conversation = await DatabaseService.getConversation(convId);
-			if (!conversation) return;
-			messages = await DatabaseService.getConversationMessages(convId);
-		}
+		if (!conversation) return;
+
+		const messages = await DatabaseService.getConversationMessages(convId);

 		this.downloadConversationFile({ conv: conversation, messages });
 	}
@@ -145,6 +145,10 @@ class ModelsStore {
 	 */

 	getModelModalities(modelId: string): ModelModalities | null {
+		if (!isRouterMode() && serverStore.props?.modalities) {
+			return this.buildModalities(serverStore.props.modalities);
+		}
+
 		const model = this.models.find((m) => m.model === modelId || m.id === modelId);
 		if (model?.modalities) {
 			return model.modalities;
@@ -629,7 +633,12 @@ class ModelsStore {
 	}

 	findModelByName(modelName: string): ModelOption | null {
-		return this.models.find((model) => model.model === modelName) ?? null;
+		return (
+			this.models.find(
+				(model) =>
+					model.model === modelName || model.id === modelName || model.aliases?.includes(modelName)
+			) ?? null
+		);
 	}

 	findModelById(modelId: string): ModelOption | null {
@@ -1,5 +1,8 @@
 import { PropsService } from '$lib/services/props.service';
 import { ServerRole } from '$lib/enums';
+import { ApiError } from '$lib/utils/api-fetch';
+
+const LOADING_RETRY_INTERVAL_MS = 1000;

 /**
 * serverStore - Server connection state, configuration, and role detection
@@ -29,8 +32,10 @@ class ServerStore {
 	props = $state<ApiLlamaCppServerProps | null>(null);
 	loading = $state(false);
 	error = $state<string | null>(null);
+	status = $state<number | null>(null);
 	role = $state<ServerRole | null>(null);
 	private fetchPromise: Promise<void> | null = null;
+	private retryTimer: ReturnType<typeof setTimeout> | null = null;

 	/**
 	 *
@@ -70,23 +75,43 @@ class ServerStore {
 	 *
 	 */

-	async fetch(): Promise<void> {
+	/**
+	 * @param background - Set by the automatic "still loading" poll. Skips the
+	 * `loading` flag flip so the UI doesn't bounce between the full loading
+	 * splash and the chat screen every retry tick.
+	 */
+	async fetch({ background = false }: { background?: boolean } = {}): Promise<void> {
 		if (this.fetchPromise) return this.fetchPromise;

-		this.loading = true;
-		this.error = null;
+		this.clearRetryTimer();
+		if (!background) {
+			this.loading = true;
+		}
+		// Don't clear an existing "still loading" error before a retry -
+		// doing so would unmount/remount the error banner every second.
+		if (this.status !== 503) {
+			this.error = null;
+		}

 		const fetchPromise = (async () => {
 			try {
 				const props = await PropsService.fetch();
 				this.props = props;
 				this.error = null;
+				this.status = null;
 				this.detectRole(props);
 			} catch (error: unknown) {
 				this.error = error instanceof Error ? error.message : String(error);
+				this.status = error instanceof ApiError ? error.status : null;
 				console.error('Error fetching server properties:', error);
+
+				if (this.status === 503) {
+					this.scheduleRetry();
+				}
 			} finally {
-				this.loading = false;
+				if (!background) {
+					this.loading = false;
+				}
 				this.fetchPromise = null;
 			}
 		})();
@@ -96,13 +121,30 @@ class ServerStore {
 	}

 	clear(): void {
+		this.clearRetryTimer();
 		this.props = null;
 		this.error = null;
+		this.status = null;
 		this.loading = false;
 		this.role = null;
 		this.fetchPromise = null;
 	}

+	private scheduleRetry(): void {
+		if (this.retryTimer) return;
+		this.retryTimer = setTimeout(() => {
+			this.retryTimer = null;
+			this.fetch({ background: true });
+		}, LOADING_RETRY_INTERVAL_MS);
+	}
+
+	private clearRetryTimer(): void {
+		if (this.retryTimer) {
+			clearTimeout(this.retryTimer);
+			this.retryTimer = null;
+		}
+	}
+
 	/**
 	 *
 	 *
@@ -125,6 +167,7 @@ export const serverStore = new ServerStore();
 export const serverProps = () => serverStore.props;
 export const serverLoading = () => serverStore.loading;
 export const serverError = () => serverStore.error;
+export const serverStatus = () => serverStore.status;
 export const serverRole = () => serverStore.role;
 export const defaultParams = () => serverStore.defaultParams;
 export const contextSize = () => serverStore.contextSize;
@@ -12,6 +12,21 @@ import { ERROR_MESSAGES, HTTP_CODE_TO_STRING } from '$lib/constants/error';
 * - Base path resolution
 */

+/**
+ * Error thrown when an API request fails, carrying the HTTP status code
+ * so callers can distinguish e.g. a 503 "still loading" response from a
+ * genuine failure.
+ */
+export class ApiError extends Error {
+	status: number;
+
+	constructor(message: string, status: number) {
+		super(message);
+		this.name = 'ApiError';
+		this.status = status;
+	}
+}
+
 export interface ApiFetchOptions extends Omit<RequestInit, 'headers'> {
 	/**
 	 * Use auth-only headers (no Content-Type).
@@ -67,7 +82,7 @@ export async function apiFetch<T>(path: string, options: ApiFetchOptions = {}):

 	if (!response.ok) {
 		const errorMessage = await parseErrorMessage(response);
-		throw new Error(errorMessage);
+		throw new ApiError(errorMessage, response.status);
 	}

 	return response.json() as Promise<T>;
@@ -119,7 +134,7 @@ export async function apiFetchWithParams<T>(

 	if (!response.ok) {
 		const errorMessage = await parseErrorMessage(response);
-		throw new Error(errorMessage);
+		throw new ApiError(errorMessage, response.status);
 	}

 	return response.json() as Promise<T>;
@@ -1,12 +0,0 @@
-<!DOCTYPE html>
-<html>
-    <head>
-        <meta http-equiv="refresh" content="5">
-    </head>
-    <body>
-        <div id="loading">
-            The model is loading. Please wait.<br/>
-            The user interface will appear soon.
-        </div>
-    </body>
-</html>
@@ -189,9 +189,5 @@ describe('PWA Build Output', () => {
 			expect(existsSync(resolve(DIST_DIR, 'pwa-192x192.png'))).toBeTruthy();
 			expect(existsSync(resolve(DIST_DIR, 'pwa-512x512.png'))).toBeTruthy();
 		});
-
-		it('has loading.html fallback page', () => {
-			expect(existsSync(resolve(DIST_DIR, 'loading.html'))).toBeTruthy();
-		});
 	});
 });
Author	SHA1	Message	Date
Xuan-Son Nguyen	3e706dd55f	mtmd: deepseek-ocr v1 multi-tile (#24717 ) * mtmd: deepseek-ocr v1 multi-tile dynamic resolution + unified image-preprocessors for both versions (ds-ocr v1 and v2) * remove hacky API * fuse row into a long image * almost working * adapt to new preprocessor api * rm debugging printf * improve * mtmd: dsocr-tiles fixes (#25481) * ds-ocr img-preproc fuse_row tile-drop fix for multi rows and columns images * mtmd drop the duplicate redundant img_end * deepseekocr graph simplify CLS broadcast cleanup * test-deepseek-ocr: relax v1 single-view tolerance; drop trailing prompt space; make DRY opt-in and n_predict model-specific (#25486) --------- Co-authored-by: Saba Fallah <10401143+sfallah@users.noreply.github.com> Co-authored-by: Saba Fallah <sabafallah@gmail.com>	2026-07-10 16:05:49 +02:00
felix	07d9378286	feat: pre-select models in the webui using alias (#25492 ) Co-authored-by: example name <example@example.org>	2026-07-10 15:04:00 +02:00
Josh Leverette	9f623c683d	ui: use server modalities in non-router mode (#24874 )	2026-07-10 15:03:52 +02:00
Xuan-Son Nguyen	a935fbffe1	server: remove loading.html (#25500 ) * server: remove loading.html * apply ui changes	2026-07-10 14:42:17 +02:00
Georgi Gerganov	0badc06ab5	sync : ggml	2026-07-10 13:11:37 +03:00
Georgi Gerganov	ac17f8ac1c	ggml : use ggml_vqtbl1q_u8 for 32-bit compat (whisper/0)	2026-07-10 13:11:37 +03:00
Xuan-Son Nguyen	c4ae9a88f8	server: improve tools, remove apply_diff (#25498 ) * server: improve tools, remove apply_diff * improve edit tool * add tools_io abstraction * add tools_io_basic * fix build * move utils to class member * add const	2026-07-10 11:52:59 +02:00
marcoStocchi	1b9691bcd5	cli: fix crash on wrong server base url (#25497 ) * llama-cli: fix crash on wrong server base url by catching exceptions and graceful exit * review: leaner catch group: json error and standard exception	2026-07-10 11:52:20 +02:00
Pascal	c7af942e8f	ui: prevent tooltip from flickering open and closed on hover (#25503 )	2026-07-10 11:49:52 +02:00
Georgi Gerganov	8f114a9b57	sync : ggml (#25517 ) * ggml : bump version to 0.16.0 (ggml/1559) * sync : ggml	2026-07-10 10:28:39 +03:00
Pascal	d46786f296	ui: export full message tree instead of active path only (#25501 ) downloadConversation serialized activeMessages, the root -> currNode path, so exporting a conversation with edited or regenerated messages dropped every alternate version and kept only the selected one. Fetch the whole message tree via getConversationMessages so the export carries all message versions, matching the multi-conversation export path which already did this. Keep the active conversation as the header source to preserve an up-to-date currNode. Forks are separate conversations, each with its own convId, and are exported on their own.	2026-07-10 09:10:45 +02:00
fairydreaming	2ed3c1abbb	llama : make all KQ masks f16 if FA is used, remove zero attention bias, remove raw_k repeats in DeepSeek V4 (#25370 ) * llama : make all KQ masks (except the lightning indexer one) f16 if FA is used and remove zero attention bias in DeepSeek V4 * llama : remove dead code that repeats unified raw_k cache for each stream in DeepSeek V4 - no longer needed as raw_k is always non-unified. --------- Co-authored-by: Stanisław Szymczyk <sszymczy@gmail.com>	2026-07-10 09:06:58 +02:00