From 6788edb4f325c1cb4210997eb79edcab2e27aeaa Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Thu, 10 Sep 2026 12:20:18 -0500 Subject: [PATCH 01/35] vulkan: small M matrix optimizations for qwen (#28457) * vulkan: optimize m=1 mul_mat by swapping A/B * vulkan: Improve small M perf Allow split_k with small M. Make small vs med tile selection (for coopmat2) depend on M, not just N. --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 27 +++++++++++++++++++-------- tests/test-backend-ops.cpp | 10 +++++++++- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 870fa1155..34400a1b7 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -5424,8 +5424,9 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { bool prefer_large = tiles_m > shader_core_count || tiles_l > shader_core_count || (tiles_l <= shader_core_count / 3 && tiles_m > shader_core_count / 2); if (n > crossover_large && prefer_large) return last; - uint32_t crossover_medium = configs[0].unaligned->wg_denoms[1]; - if (n > crossover_medium) return 1; + uint32_t crossover_medium_m = configs[0].unaligned->wg_denoms[0]; + uint32_t crossover_medium_n = configs[0].unaligned->wg_denoms[1]; + if (m > crossover_medium_m && n > crossover_medium_n) return 1; return 0; }; device->matmul_id_tile_selector = [](uint32_t /*m*/, uint32_t n, uint32_t /*k*/, uint32_t /*shader_core_count*/, @@ -9027,7 +9028,7 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m, } uint32_t split_k = 1; - if (ctx->device->shader_core_count != 0 && m >= pipeline->wg_denoms[0] && n >= pipeline->wg_denoms[1]) { + if (ctx->device->shader_core_count != 0 && n >= pipeline->wg_denoms[1]) { // If k is 'large' and the SMs will fill less than halfway, use split_k. uint32_t m_tiles = CEIL_DIV(m, pipeline->wg_denoms[0]); uint32_t n_tiles = CEIL_DIV(n, pipeline->wg_denoms[1]); @@ -9780,10 +9781,10 @@ static bool ggml_vk_should_use_mmvq(const vk_device& device, uint32_t m, uint32_ GGML_UNUSED(m); } -static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { +static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx, bool swap_inputs = false) { ggml_tensor * dst = cgraph->nodes[node_idx]; - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; + const ggml_tensor * src0 = dst->src[swap_inputs ? 1 : 0]; + const ggml_tensor * src1 = dst->src[swap_inputs ? 0 : 1]; VK_LOG_DEBUG("ggml_vk_mul_mat_vec_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; @@ -9802,8 +9803,8 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& const uint64_t ne12 = src1->ne[2]; const uint64_t ne13 = src1->ne[3]; - const uint64_t ne20 = dst->ne[0]; - const uint64_t ne21 = dst->ne[1]; + const uint64_t ne20 = dst->ne[swap_inputs ? 1 : 0]; + const uint64_t ne21 = dst->ne[swap_inputs ? 0 : 1]; // const uint64_t ne22 = dst->ne[2]; // const uint64_t ne23 = dst->ne[3]; @@ -10417,6 +10418,16 @@ static void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context& subctx, c src0->ne[1] <= ctx->device->properties.limits.maxComputeWorkGroupCount[1] && src1->ne[2] <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]) { ggml_vk_mul_mat_vec_nc_f16_f32(ctx, subctx, cgraph, node_idx); + // With one output row, B^T*A has the same flat output as A^T*B. + } else if (ctx->num_additional_fused_ops == 0 && + (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) && + (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_BF16 || ggml_is_quantized(src1->type)) && + dst->ne[0] == 1 && dst->ne[1] > mul_mat_vec_max_cols && + src0->ne[2] == 1 && src0->ne[3] == 1 && + src1->ne[2] == 1 && src1->ne[3] == 1 && + ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst) && + get_misalign_bytes(ctx, src0) == 0 && get_misalign_bytes(ctx, src1) == 0 && get_misalign_bytes(ctx, dst) == 0) { + ggml_vk_mul_mat_vec_q_f16(ctx, subctx, cgraph, node_idx, true); // mul_mat_vec supports batching ne12*ne13 when ne11==1, or treating ne11 as the batch size (up to four) // when ne12 and ne13 are one. } else if ((dst->ne[1] == 1 || (dst->ne[1] <= mul_mat_vec_max_cols && src1->ne[2] * src1->ne[3] == 1)) && diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 0a6516e61..3428629fd 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9683,9 +9683,17 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_MXFP4, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); // m == 1, with n on both sides of MMVF_MAX_BATCH_SIZE (8): mmvf below, operand swap above - for (int64_t n : {1, 7, 8, 9, 16, 128, 512}) { + for (int64_t n : {1, 7, 8, 9, 16, 127, 128, 511, 512}) { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 1, n, 2048, {1, 1}, {1, 1})); } + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 1, 512, 2048, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1, 512, 2048, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 1, 509, 2051, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1, 509, 2051, {1, 1}, {1, 1})); + + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 31, 509, 2051, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 32, 509, 2112, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 32, 509, 2112, {1, 1}, {1, 1})); #if 0 { From 50182a53fa2c26bd2a7fc31d855231effdc2f4ad Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Thu, 10 Sep 2026 12:21:29 -0500 Subject: [PATCH 02/35] vulkan: use add_alloc_dep to enable topk_moe fusion for prefill (#28422) --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 97 +++++++++++++++++----------- 1 file changed, 60 insertions(+), 37 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 34400a1b7..8e1cf3ff3 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -18328,38 +18328,30 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg bool need_disable = false; - // topk_moe often overwrites the source, but for a given row all the src values are - // loaded before anything is stored. If there's only one row, this is safe, so treat - // this as a special case. - bool is_topk_moe_single_row = ctx->fused_topk_moe_mode != TOPK_MOE_COUNT && - ggml_nrows(cgraph->nodes[i]->src[0]) == 1; - - if (!is_topk_moe_single_row) { - for (int j = 0; j < 2; ++j) { - ggml_tensor *dst = output_nodes[j]; - if (!dst) { - continue; - } - // Loop over all srcs of all nodes in the fusion. If the src overlaps - // the destination and the src is not an intermediate node that's being - // elided, then disable fusion. - for (int k = 0; k <= ctx->num_additional_fused_ops; ++k) { - for (uint32_t s = 0; s < GGML_MAX_SRC; ++s) { - ggml_tensor *src = cgraph->nodes[i + k]->src[s]; - if (!src || src->op == GGML_OP_NONE) { - continue; + for (int j = 0; j < 2; ++j) { + ggml_tensor *dst = output_nodes[j]; + if (!dst) { + continue; + } + // Loop over all srcs of all nodes in the fusion. If the src overlaps + // the destination and the src is not an intermediate node that's being + // elided, then disable fusion. + for (int k = 0; k <= ctx->num_additional_fused_ops; ++k) { + for (uint32_t s = 0; s < GGML_MAX_SRC; ++s) { + ggml_tensor *src = cgraph->nodes[i + k]->src[s]; + if (!src || src->op == GGML_OP_NONE) { + continue; + } + if (ggml_vk_tensors_overlap(src, dst, op_srcs_fused_elementwise[k])) { + bool found = false; + for (int n = 0; n < k; ++n) { + if (cgraph->nodes[i + n] == src) { + found = true; + break; + } } - if (ggml_vk_tensors_overlap(src, dst, op_srcs_fused_elementwise[k])) { - bool found = false; - for (int n = 0; n < k; ++n) { - if (cgraph->nodes[i + n] == src) { - found = true; - break; - } - } - if (!found) { - need_disable = true; - } + if (!found) { + need_disable = true; } } } @@ -18372,6 +18364,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg ctx->fused_topk_moe_scale = false; ctx->fused_topk_qsa = false; ctx->fused_rms_norm_mode = RMS_NORM_COUNT; + fusion_string = nullptr; } } @@ -18474,7 +18467,6 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg // Sort the graph for improved parallelism. static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, struct ggml_backend_graph_optimize_params * params) { - GGML_UNUSED(params); VK_LOG_DEBUG("ggml_vk_graph_optimize(" << graph->n_nodes << " nodes)"); ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; @@ -18560,19 +18552,50 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * return false; }; - if (keep_pattern(topk_moe_early_softmax_norm)) { + auto const &add_pattern_alloc_deps = [&](const std::initializer_list &pattern, int last_node) { + // Keep external inputs alive through the fused output. + std::set seen; + for (size_t j = 0; j < pattern.size(); ++j) { + ggml_tensor * node = graph->nodes[first_unused + j]; + for (uint32_t s = 0; s < GGML_MAX_SRC; ++s) { + ggml_tensor * src = node->src[s]; + if (src && seen.insert(src).second) { + params->add_alloc_dep(params->user_data, src, graph->nodes[last_node]); + } + } + seen.insert(node); + } + }; + + auto const &keep_topk_moe_pattern = [&](const std::initializer_list &pattern) -> bool { + if (!match_pattern(pattern, first_unused)) { + return false; + } + + int last_node = first_unused + (int) pattern.size() - 1; + // Some TOPK_MOE variants fuse a trailing scale. + if (last_node + 1 < graph->n_nodes && graph->nodes[last_node + 1]->op == GGML_OP_SCALE) { + last_node++; + } + + add_pattern_alloc_deps(pattern, last_node); + + return keep_pattern(pattern); + }; + + if (keep_topk_moe_pattern(topk_moe_early_softmax_norm)) { continue; } - if (keep_pattern(topk_moe_sigmoid_norm_bias)) { + if (keep_topk_moe_pattern(topk_moe_sigmoid_norm_bias)) { continue; } - if (keep_pattern(topk_moe_sqrt_softplus_norm_bias)) { + if (keep_topk_moe_pattern(topk_moe_sqrt_softplus_norm_bias)) { continue; } - if (keep_pattern(topk_moe_early_softmax)) { + if (keep_topk_moe_pattern(topk_moe_early_softmax)) { continue; } - if (keep_pattern(topk_moe_late_softmax)) { + if (keep_topk_moe_pattern(topk_moe_late_softmax)) { continue; } if (keep_pattern(snake_pattern)) { From 28ff0958291ce3465fabd7bd679d4b0edd742bd9 Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Thu, 10 Sep 2026 12:22:46 -0500 Subject: [PATCH 03/35] vulkan: use CPU writes in ggml_backend_vk_cpy_tensor_async if the context is idle (#28618) --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 8e1cf3ff3..b28fdc9bb 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -17135,6 +17135,22 @@ static bool ggml_backend_vk_cpy_tensor_async(ggml_backend_t backend_src, ggml_ba return false; } + // If the backend is idle, use a CPU copy to avoid GPU synchronization overhead. + static constexpr size_t max_cpu_copy_size = 128 * 1024; + const bool src_backend_synchronous = backend_src->iface.synchronize == nullptr; + const bool transfer_idle = !ctx->device->async_use_transfer_queue || + ctx->transfer_semaphore_last_submitted == ctx->transfer_semaphore.value; + const bool backend_idle = ctx->compute_ctx.expired() && ctx->transfer_ctx.expired() && + !ctx->submit_pending && !ctx->almost_ready_fence_pending && transfer_idle; + const bool dst_host_coherent = + (dst_buf->memory_property_flags & (vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent)) == + (vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent); + + if ((backend_src == backend_dst || src_backend_synchronous) && backend_idle && dst_host_coherent && ggml_nbytes(src) <= max_cpu_copy_size) { + ggml_vk_buffer_write(dst_buf, vk_tensor_offset(dst) + dst->view_offs, src->data, ggml_nbytes(src)); + return true; + } + vk_context cpy_ctx; if (ctx->device->async_use_transfer_queue) { cpy_ctx = ggml_vk_get_transfer_ctx(ctx); @@ -17147,7 +17163,6 @@ static bool ggml_backend_vk_cpy_tensor_async(ggml_backend_t backend_src, ggml_ba src->data, ggml_nbytes(src)); } - GGML_UNUSED(backend_src); return false; } From df03399b885831b2a1603b3abb0d8c156808e363 Mon Sep 17 00:00:00 2001 From: shaofeiqi Date: Thu, 10 Sep 2026 11:25:40 -0700 Subject: [PATCH 04/35] opencl: add A8 Q4_0 mm binary kernel support (#28268) --- ggml/src/ggml-opencl/CMakeLists.txt | 1 + ggml/src/ggml-opencl/ggml-opencl.cpp | 274 +++++++++++++++++- .../gemv_noshuffle_q4_0_f32_32b_trans.cl | 137 +++++++++ 3 files changed, 402 insertions(+), 10 deletions(-) create mode 100644 ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_32b_trans.cl diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 37e565ef4..716577bb7 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -170,6 +170,7 @@ set(GGML_OPENCL_KERNELS gemv_noshuffle_q4_0_f32 gemv_noshuffle_q4_0_f32_spec gemm_noshuffle_q4_0_f32 + gemv_noshuffle_q4_0_f32_32b_trans gemv_noshuffle_q4_1_f32 gemm_noshuffle_q4_1_f32 gemv_noshuffle_q5_0_f32 diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 3002835e8..231be2cf3 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -1160,6 +1160,8 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemm_noshuffle_q4_0_f32; cl_kernel kernel_gemv_noshuffle_q4_0_f32; cl_kernel kernel_gemv_noshuffle_q4_0_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP) + cl_kernel kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin; + cl_kernel kernel_gemv_noshuffle_q4_0_f32_32b_trans; cl_kernel kernel_gemv_noshuffle_q4_0_f32_4096_1_11008; cl_kernel kernel_gemv_noshuffle_q4_0_f32_4096_1_4096; cl_kernel kernel_gemv_noshuffle_q4_0_f32_11008_1_4096; @@ -3787,6 +3789,43 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans = nullptr; + backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin = nullptr; + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E) { + { + std::string opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q4_0_f32_32b_trans.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q4_0_f32_32b_trans.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans = + clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32_32b_trans", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + if (use_adreno_bin_kernels(backend_ctx)) { + size_t bin_size = 0; + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_0_f32_32b_trans_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } + } + } + // gemm_noshuffle_q4_1_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -6725,11 +6764,10 @@ struct ggml_tensor_extra_cl_q4_0 { CL_CHECK(clReleaseMemObject(q_img)); q_img = nullptr; } - // Currently, q_img and d_img are only initialized when SMALL_ALLOC is - // enabled. They point to the images in ggml_backend_opencl_buffer_context. - // So, there is no need to release them here. - // TODO: initialize them for non SMALL_PATH path, or remove them. - d_img = nullptr; + if (d_img != nullptr) { + CL_CHECK(clReleaseMemObject(d_img)); + d_img = nullptr; + } size_q = 0; size_d = 0; } @@ -8311,6 +8349,20 @@ inline bool enable_adreno_trans_weight_q5_K(const ggml_backend_opencl_context *b qh_img_width <= backend_ctx->image_max_buffer_size; } +inline bool use_q4_0_ila_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (!backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans || + !backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin) { + return false; + } + return (tensor->ne[0] % 32 == 0) && (tensor->ne[1] % 64 == 0); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + // The flat-GEMV large-m escape is OPT-IN (GGML_OPENCL_FLAT_LARGE_M=1) because it // is SLOWER than the route it replaces, not because it is unsafe. It was first // parked on the theory that it out-of-bounds-writes at vocab-scale shapes; that @@ -9573,10 +9625,34 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(K % 32 == 0); - // Transpose q as ushort - transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); - // Transpose d as ushort - transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); + if (use_q4_0_ila_kernels(backend_ctx, tensor)) { + cl_int err; + cl_image_format wimg_fmt; + cl_image_desc wimg_desc; + + // transpose quants as 32-bit words (M-first) + GGML_ASSERT(M % 64 == 0); + transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K / 8, M); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K / 32, M); + + wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = (size_t)M * K / 8; + wimg_desc.buffer = extra->q; + CL_CHECK((extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + + wimg_fmt = { CL_R, CL_HALF_FLOAT }; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = (size_t)M * K / 32; + wimg_desc.buffer = extra->d; + CL_CHECK((extra->d_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + } else { + // Transpose q and d as ushort + transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); + } } #endif // GGML_OPENCL_USE_ADRENO_KERNELS return; @@ -11104,7 +11180,11 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, buf_trans_d.allocate(backend_ctx->context, size_d); buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); - transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); + if (use_q4_0_ila_kernels(backend_ctx, tensor)) { + transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K / 8); + } else { + transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K / 4); + } transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); cl_uchar mask_0F = 0x0F; @@ -18347,6 +18427,166 @@ static void ggml_cl_mul_mat_q1_0_f32_adreno(ggml_backend_t backend, const ggml_t #endif } +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_mul_mat_q4_0_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + + const int ne1 = dst->ne[1]; + + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); + + cl_context context = backend_ctx->context; + cl_kernel kernel; + + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; + + int M = ne01; + int N = ne1; + int K = ne00; + + if (ne1 == 1) { + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; + + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_RGBA, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &K)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &M)); + + size_t wavesize = backend_ctx->adreno_wave_size; + size_t local_work_size[3] = { wavesize, 4, 1 }; + size_t global_work_size[3] = { (size_t)CEIL_DIV(M, 64) * 64, 4, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + } else { + const int gemm_tile_n = 64; + int N_pad = (N + gemm_tile_n - 1) & ~(gemm_tile_n - 1); + + cl_mem a_img = extra0_q4_0->q_img; + cl_mem s_img = extra0_q4_0->d_img; + GGML_ASSERT(a_img && s_img && "ILA Q4_0 weight images missing; set_tensor should have built them"); + + // Pad B through a zero-filled scratch buffer when N needs + // padding, since the GEMM kernel always reads a full N-tile. + const bool need_pad = N_pad > N; + cl_mem b_sub_buf = nullptr; + cl_mem b_padded = nullptr; + if (need_pad) { + CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, + (size_t)K * N_pad * sizeof(float), NULL, &err), err)); + const float zero = 0.0f; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), + 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, + offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); + } else { + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + } + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = need_pad ? (size_t)K * N_pad : (size_t)K * N; + img_desc.buffer = need_pad ? b_padded : b_sub_buf; + cl_mem b_img; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + cl_mem d_sub_buf; + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub_buf; + cl_mem d_img; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + int line_stride_matrix_A_in_bytes = M * 4; + int line_stride_matrix_S_in_bytes = M * 2; + int line_stride_matrix_B_in_bytes = K * 4; + int line_stride_matrix_C_in_bytes = M * 4; + + int c_offset_for_kernel = 0; + int b_offset_for_kernel = 0; + + kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin; + + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &a_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &s_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &b_offset_for_kernel)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &c_offset_for_kernel)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_A_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_S_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_B_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_C_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); + + size_t local_work_size[3] = { 64, 2, 2 }; + size_t m_tiles = (size_t)CEIL_DIV(M, 64); + size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_img)); + if (b_sub_buf) { + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } + if (b_padded) { + CL_CHECK(clReleaseMemObject(b_padded)); + } + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); + } +} +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); @@ -18399,6 +18639,20 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t static const bool q40_mc3 = (getenv("GGML_OPENCL_Q40_MC3") != nullptr); const bool use_q40_mc3 = q40_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); + const bool use_ila = use_q4_0_ila_kernels(backend_ctx, src0); + + if (use_ila) { + if (use_q40_mc3) { + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q40_MC3 is bypassed by Q4_0 binary kernels\n"); + warned = true; + } + } + ggml_cl_mul_mat_q4_0_f32_adreno_ila(backend, src0, src1, dst); + return; + } + if (ne1 == 1 || use_q40_mc3) { cl_mem q_img = nullptr; cl_mem b_sub_buf = nullptr; diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_32b_trans.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_32b_trans.cl new file mode 100644 index 000000000..565285f4b --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_32b_trans.cl @@ -0,0 +1,137 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#endif + +#define QK4_0 32 +#define N_SIMDGROUP 4 + +#define dequantizeBlockAccum_ila_1row_hi(total_sum, bits4, scale, y) \ + float shared_y; \ + shared_y = sub_group_broadcast(y.s0, 0); \ + total_sum += ((bits4.s0 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 0); \ + total_sum += (((bits4.s0 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 0); \ + total_sum += (((bits4.s0 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 0); \ + total_sum += (((bits4.s0 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 0); \ + total_sum += ((bits4.s1 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 0); \ + total_sum += (((bits4.s1 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 0); \ + total_sum += (((bits4.s1 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 0); \ + total_sum += (((bits4.s1 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 1); \ + total_sum += ((bits4.s2 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 1); \ + total_sum += (((bits4.s2 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 1); \ + total_sum += (((bits4.s2 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 1); \ + total_sum += (((bits4.s2 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 1); \ + total_sum += ((bits4.s3 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 1); \ + total_sum += (((bits4.s3 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 1); \ + total_sum += (((bits4.s3 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 1); \ + total_sum += (((bits4.s3 & 0xF000) >> 12) - 8) * scale * shared_y; + +#define dequantizeBlockAccum_ila_1row_lo(total_sum, bits4, scale, y) \ + shared_y = sub_group_broadcast(y.s0, 2); \ + total_sum += ((bits4.s4 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 2); \ + total_sum += (((bits4.s4 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 2); \ + total_sum += (((bits4.s4 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 2); \ + total_sum += (((bits4.s4 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 2); \ + total_sum += ((bits4.s5 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 2); \ + total_sum += (((bits4.s5 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 2); \ + total_sum += (((bits4.s5 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 2); \ + total_sum += (((bits4.s5 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 3); \ + total_sum += ((bits4.s6 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 3); \ + total_sum += (((bits4.s6 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 3); \ + total_sum += (((bits4.s6 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 3); \ + total_sum += (((bits4.s6 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 3); \ + total_sum += ((bits4.s7 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 3); \ + total_sum += (((bits4.s7 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 3); \ + total_sum += (((bits4.s7 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 3); \ + total_sum += (((bits4.s7 & 0xF000) >> 12) - 8) * scale * shared_y; + + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +__kernel void kernel_gemv_noshuffle_q4_0_f32_32b_trans( + __read_only image1d_buffer_t src0_q, + global half * src0_d, + __read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + uint K = ne00; + uint M = ne01; + + __private uint4 regA; + __private half regS; + __private float8 regB; + __private float totalSum = 0.0f; + + for (uint k = groupId; k < (K / QK4_0); k += N_SIMDGROUP) { + regS = src0_d[k * M + gid]; + if (slid < 4) { + regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); + regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); + } + regA.s0 = read_imageui(src0_q, ((k * 4 + 0) * M + gid)).x; + regA.s1 = read_imageui(src0_q, ((k * 4 + 1) * M + gid)).x; + regA.s2 = read_imageui(src0_q, ((k * 4 + 2) * M + gid)).x; + regA.s3 = read_imageui(src0_q, ((k * 4 + 3) * M + gid)).x; + + dequantizeBlockAccum_ila_1row_hi(totalSum, as_ushort8(regA), regS, regB); + dequantizeBlockAccum_ila_1row_lo(totalSum, as_ushort8(regA), regS, regB); + } + + __local float reduceLM[SIMDGROUP_WIDTH * 3]; + if (groupId == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = totalSum; + if (groupId == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = totalSum; + if (groupId == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = totalSum; + barrier(CLK_LOCAL_MEM_FENCE); + if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 0 + slid]; + if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 1 + slid]; + if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 2 + slid]; + + if (groupId == 0) { + dst = (global float*)((global char*)dst + offsetd); + if (gid < M) { + dst[gid] = totalSum; + } + } +} From 481c65f091f74c5e7089dd0a3a1cc6b50cced31e Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Fri, 11 Sep 2026 00:44:13 -0500 Subject: [PATCH 05/35] vulkan: fix data race and OOB access in argsort(large) (#28705) argsort had a data race in the inner loop, which VVL caught. But I don't think this was causing failures in practice. argsort_large has OOB accesses which might explain the failures in CI, but I couldn't reproduce it locally and I don't think it's a convincing explanation of the failures. --- .../ggml-vulkan/vulkan-shaders/argsort.comp | 28 +++++++++++-------- .../vulkan-shaders/argsort_large.comp | 5 +++- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp b/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp index 0fc2b9b72..4ba63f7ae 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp @@ -33,7 +33,11 @@ void argsort(bool needs_bounds_check, const uint row) { const uint row_offset = row * p.ncols; // initialize indices - dst_row[col] = ivec2(col, floatBitsToInt(data_a[row_offset + col])); + ivec2 value = ivec2(col, 0); + if (!needs_bounds_check || col < p.ncols) { + value.y = floatBitsToInt(data_a[row_offset + col]); + } + dst_row[col] = value; barrier(); uint num_outer_loop_iters = NCOLS_PADDED_LOG2; @@ -42,18 +46,20 @@ void argsort(bool needs_bounds_check, const uint row) { [[unroll]] for (uint j = k / 2, inner_idx = 0; inner_idx < num_inner_loop_iters; j /= 2, inner_idx++) { const int ixj = int(col ^ j); - int idx_0 = (col & k) == 0 ? col : ixj; - int idx_1 = (col & k) == 0 ? ixj : col; + if (ixj > col) { + int idx_0 = (col & k) == 0 ? col : ixj; + int idx_1 = (col & k) == 0 ? ixj : col; - ivec2 sh_idx_0 = dst_row[idx_0]; - ivec2 sh_idx_1 = dst_row[idx_1]; - bool idx_0_oob = needs_bounds_check ? sh_idx_0.x >= p.ncols : false; - bool idx_1_oob = needs_bounds_check ? sh_idx_1.x >= p.ncols : false; + ivec2 sh_idx_0 = dst_row[idx_0]; + ivec2 sh_idx_1 = dst_row[idx_1]; + bool idx_0_oob = needs_bounds_check ? sh_idx_0.x >= p.ncols : false; + bool idx_1_oob = needs_bounds_check ? sh_idx_1.x >= p.ncols : false; - if ((idx_0_oob || - (!idx_1_oob && intBitsToFloat(sh_idx_0.y) > intBitsToFloat(sh_idx_1.y))) && (ixj > col)) { - dst_row[idx_0] = sh_idx_1; - dst_row[idx_1] = sh_idx_0; + if (idx_0_oob || + (!idx_1_oob && intBitsToFloat(sh_idx_0.y) > intBitsToFloat(sh_idx_1.y))) { + dst_row[idx_0] = sh_idx_1; + dst_row[idx_1] = sh_idx_0; + } } barrier(); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp b/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp index 920bac6bb..b2df44137 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp @@ -42,7 +42,10 @@ void argsort(bool needs_bounds_check, const uint row) { [[unroll]] for (int u = 0; u < WG_UNROLL_FACTOR; ++u) { uint c = u*BLOCK_SIZE + col; if (c < p.ncols_padded) { - ivec2 v = ivec2(c, floatBitsToInt(data_a[row_offset + c])); + ivec2 v = ivec2(c, 0); + if (!needs_bounds_check || c < p.ncols) { + v.y = floatBitsToInt(data_a[row_offset + c]); + } tmp_idx[idx_offset + c] = v; } } From 451b89bae0c4b1dd612eb503ceace906c01ddcc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Fri, 11 Sep 2026 07:56:18 +0200 Subject: [PATCH 06/35] ci : key cache to sanitizer matrix (#28708) --- .github/workflows/server-sanitize.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/server-sanitize.yml b/.github/workflows/server-sanitize.yml index 52e175f83..11237cf51 100644 --- a/.github/workflows/server-sanitize.yml +++ b/.github/workflows/server-sanitize.yml @@ -75,7 +75,7 @@ jobs: - name: ccache-buckets-restore uses: ./.github/actions/ccache-buckets with: - key: server-sanitize + key: server-sanitize-${{ matrix.sanitizer }} folder: llama.cpp hf_bucket: ggml-org/cache @@ -99,7 +99,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: - key: server-sanitize + key: server-sanitize-${{ matrix.sanitizer }} folder: llama.cpp evict-old-files: 1d hf_bucket: ggml-org/cache From 16378d93f94012d4228c8c7683adce3f286aee5d Mon Sep 17 00:00:00 2001 From: "Piotr Wilkin (ilintar)" Date: Fri, 11 Sep 2026 09:58:20 +0200 Subject: [PATCH 07/35] CUDA/HIP: Flash Attention tuning (gfx1201) (#28102) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * HIP: enable mma FA for head size 256 on RDNA4, tune configs Assisted-by: Claude Assisted-by: Codex * HIP: prefer whole-tile FA grids over stream-k on AMD WMMA Assisted-by: Claude Assisted-by: Codex * revise stream_k logic * revise kernel selection logic --------- Co-authored-by: Johannes Gäßler --- ggml/src/ggml-cuda/fattn-common.cuh | 19 ++++++++++++++----- ggml/src/ggml-cuda/fattn-mma-f16.cuh | 6 +++--- ggml/src/ggml-cuda/fattn.cu | 23 +++++++++++++++++++++-- tests/test-backend-ops.cpp | 16 ++++++++++++++++ 4 files changed, 54 insertions(+), 10 deletions(-) diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh index 7442bc22a..48b631e60 100644 --- a/ggml/src/ggml-cuda/fattn-common.cuh +++ b/ggml/src/ggml-cuda/fattn-common.cuh @@ -1133,12 +1133,21 @@ void launch_fattn( dim3 blocks_num; if (stream_k) { - // For short contexts it can be faster to have the SMs work on whole tiles because this lets us skip the fixup. - const int max_blocks = max_blocks_per_sm*nsm; - const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks; - const int tiles_efficiency_percent = 100 * ntiles_dst / (max_blocks*tiles_nwaves); + auto should_use_stream_k = [](const int cc, const int ntiles_dst, const int max_blocks, const int DKQ) { + const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks; + const int tiles_efficiency_percent = 100 * ntiles_dst / (max_blocks*tiles_nwaves); - const bool use_stream_k = cc >= GGML_CUDA_CC_ADA_LOVELACE || amd_wmma_available(cc) || tiles_efficiency_percent < 75; + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_ADA_LOVELACE) { + return true; + } + if (amd_wmma_available(cc) && DKQ == 64) { + return true; // TODO better configuration + } + return tiles_efficiency_percent < 75; + }; + + const int max_blocks = max_blocks_per_sm*nsm; + const bool use_stream_k = should_use_stream_k(cc, ntiles_dst, max_blocks, Q->ne[0]); blocks_num.x = ntiles_dst; blocks_num.y = 1; diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuh index bc5060e81..578f6cf79 100644 --- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh @@ -158,8 +158,8 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 8, 64, 2, 32, 128, 128, 128, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 64, 2, 32, 128, 128, 128, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2, 64, 128, 128, 64, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2, 64, 128, 128, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 256, 2, 64, 128, 128, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 256, 2, 64, 128, 128, 64, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 32, 128, 2, 32, 160, 128, 128, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 64, 128, 2, 32, 160, 128, 128, 1, true); @@ -1826,7 +1826,7 @@ static __global__ void flash_attn_ext_f16( #endif // __CUDA_ARCH__ == GGML_CUDA_CC_TURING #if defined(AMD_WMMA_AVAILABLE) - if (ncols1*ncols2 < 16 || ncols2 == 1 || DKQ > 128) { + if (ncols1*ncols2 < 16 || ncols2 == 1 || DKQ > 256) { NO_DEVICE_CODE; return; } diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index d11a964d5..ceb472793 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -221,6 +221,24 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2(ggml_backend_cuda_con } } + // On RDNA it is preferable to minimize wasted compute vs. duplicate I/O for the mask. + if (amd_wmma_available(cc)) { + if (use_gqa_opt && gqa_ratio % 8 == 0) { + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); + return; + } + + if (use_gqa_opt && gqa_ratio % 4 == 0) { + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); + return; + } + + if (use_gqa_opt && gqa_ratio % 2 == 0) { + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); + return; + } + } + if (use_gqa_opt && gqa_ratio > 4) { ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); return; @@ -646,8 +664,9 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const } } - // AMD WMMA is always faster than the tile kernel if the full tile width of 16 can be utilized. - if ((amd_wmma_available(cc) && gqa_opt_applies && Q->ne[0] <= 128) && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[1] * gqa_ratio_eff > 8) { + // AMD WMMA is faster than the tile kernel if the wide tiles with high arithmetic intensity can be utilized. + if ((amd_wmma_available(cc) && gqa_opt_applies && Q->ne[0] <= 256) && Q->ne[0] != 40 && Q->ne[0] != 72 && + Q->ne[1] * gqa_ratio_eff > (Q->ne[0] <= 128 ? 8 : 16)) { return BEST_FATTN_KERNEL_MMA_F16; } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 3428629fd..ef4fc30ce 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10574,6 +10574,13 @@ static std::vector> make_test_cases_eval() { GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); } + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 512, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 4096, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 4096, 16, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {2, 1}, 4096, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {4, 1}, 4096, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {12, 1}, 4096, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + // dense-allocated (non-view) quant K/V at batch >= 64, in cache and native layouts test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {4, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); @@ -11031,6 +11038,15 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 10000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 20000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 4096, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 4096, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 16384, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 16384, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 65536, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 65536, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 131072, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 131072, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + for (int kv : { 4096, 8192, 16384,32768, 65536, }) { for (int hs : { 64, 128, 256, 576, }) { const int hsv = hs == 576 ? 512 : hs; From b0dcb8192b201e402ec3eff524e55450f8070e3e Mon Sep 17 00:00:00 2001 From: Jesus Gulfo Date: Fri, 11 Sep 2026 03:33:26 -0500 Subject: [PATCH 08/35] server: fix speculation after an image (#28715) * server: fix speculation after an image Pass the actual position to the drafter after an image, instead of the token count. Affects every drafter, not just DFlash. * rename draft n_past to pos0 n_past is used to denote number of tokens and this parameter is meant to be a position --- common/speculative.cpp | 20 +++++++++---------- common/speculative.h | 2 +- .../speculative-simple/speculative-simple.cpp | 2 +- tools/server/server-context.cpp | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/common/speculative.cpp b/common/speculative.cpp index b7811b853..7c8a06365 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -296,7 +296,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { drafting[seq_id] = true; common_sampler_reset(smpls[seq_id].get()); - common_batch_add(batch, dp.id_last, dp.n_past, { seq_id }, true); + common_batch_add(batch, dp.id_last, dp.pos0, { seq_id }, true); } int ret = llama_decode(ctx_dft, batch); @@ -355,7 +355,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { continue; } - common_batch_add(batch, id, dp.n_past + i + 1, { seq_id }, true); + common_batch_add(batch, id, dp.pos0 + i + 1, { seq_id }, true); } if (batch.n_tokens == 0) { @@ -1197,7 +1197,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { common_sampler_reset(smpls[seq_id].get()); - const int32_t n = (int32_t) dp.n_past; + const int32_t n = (int32_t) dp.pos0; const int32_t n_draft = params.n_max; @@ -1621,7 +1621,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { drafting[seq_id] = true; common_sampler_reset(smpls[seq_id].get()); - common_batch_add(batch, dp.id_last, dp.n_past, { seq_id }, true); + common_batch_add(batch, dp.id_last, dp.pos0, { seq_id }, true); std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, pending_h[seq_id].data(), row_bytes); i_last[seq_id] = batch.n_tokens - 1; @@ -1635,16 +1635,16 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { while (n_drafting > 0) { // each step decodes under a different head, i.e. a different decoder layer, and - // KV is per layer. process() filled this layer's KV only for positions < n_past + // KV is per layer. process() filled this layer's KV only for positions < pos0 // (prompt + accepted prefix) — nothing in the draft region yet. so reset the - // draft region (the seq_rm lower bound is n_past, leaving the prompt KV intact) + // draft region (the seq_rm lower bound is pos0, leaving the prompt KV intact) // and select head i so it rebuilds its own layer's KV there; decoding just the // latest token would leave its attention reading cells only another head wrote. if (chain_heads) { auto * mem_dft = llama_get_memory(ctx_dft); for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { if (drafting[seq_id]) { - llama_memory_seq_rm(mem_dft, seq_id, dparams[seq_id].n_past, -1); + llama_memory_seq_rm(mem_dft, seq_id, dparams[seq_id].pos0, -1); } } llama_set_nextn_layer_offset(ctx_dft, i); @@ -1710,17 +1710,17 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { const int n_rows = (int) result.size() + 1; // id_last + tokens drafted so far for (int t = 0; t < n_rows; ++t) { const llama_token tok = (t == 0) ? dp.id_last : result[t - 1]; - common_batch_add(batch, tok, dp.n_past + t, { seq_id }, t == n_rows - 1); + common_batch_add(batch, tok, dp.pos0 + t, { seq_id }, t == n_rows - 1); std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, chain_h[seq_id].data() + (size_t) t * n_embd, row_bytes); } } else if (is_mem_shared) { // note: with shared memory (e.g. Gemma4 assistants) we use the same position for all draft tokens // ref: https://github.com/huggingface/transformers/blob/effde20942e3f82a1b97449f60b3a48c5ff96145/docs/source/en/model_doc/gemma4_assistant.md?plain=1#L36-L37 - common_batch_add(batch, id, dp.n_past, { seq_id }, true); + common_batch_add(batch, id, dp.pos0, { seq_id }, true); std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, h_row, row_bytes); } else { - common_batch_add(batch, id, dp.n_past + i + 1, { seq_id }, true); + common_batch_add(batch, id, dp.pos0 + i + 1, { seq_id }, true); std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, h_row, row_bytes); } diff --git a/common/speculative.h b/common/speculative.h index 22505891f..c968750e2 100644 --- a/common/speculative.h +++ b/common/speculative.h @@ -61,7 +61,7 @@ struct common_speculative_draft_params { // can be used to constraint the max draft based on the remaining context size int32_t n_max = -1; - llama_pos n_past; + llama_pos pos0; llama_token id_last; // TODO: remove in the future by keeping track of the prompt from the _begin() call and the consecutive accept calls diff --git a/examples/speculative-simple/speculative-simple.cpp b/examples/speculative-simple/speculative-simple.cpp index 487ae03ab..863af5a2c 100644 --- a/examples/speculative-simple/speculative-simple.cpp +++ b/examples/speculative-simple/speculative-simple.cpp @@ -188,7 +188,7 @@ int main(int argc, char ** argv) { common_speculative_get_draft_params(spec, seq_id) = { /* .drafting = */ true, /* .n_max = */ n_draft_max, - /* .n_past = */ n_past, + /* .pos0 = */ n_past, /* .id_last = */ id_last, /* .prompt = */ &prompt_tgt, /* .result = */ &draft, // output diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index fe068d3e9..b6835e434 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3028,7 +3028,7 @@ private: common_speculative_get_draft_params(spec.get(), slot.id) = { /* .drafting = */ true, /* .n_max = */ n_draft_max, - /* .n_past = */ slot.prompt.n_tokens(), + /* .pos0 = */ slot.prompt.tokens.pos_next(), /* .id_last = */ slot.sampled, /* .prompt = */ &slot.spec_prompt, /* .result = */ &slot.spec_draft, From 5cdd3d1dad5cbb7107b3e9f6d23239ba88ac0123 Mon Sep 17 00:00:00 2001 From: Logan Chu Date: Fri, 11 Sep 2026 02:02:31 -0700 Subject: [PATCH 09/35] =?UTF-8?q?model=20:=20fix=20MTP=20context=20kv=20ca?= =?UTF-8?q?che=20allocation=20for=20deepseek2,=20glm4moe,=20=E2=80=A6=20(#?= =?UTF-8?q?28630)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * model : fix MTP context kv cache allocation for deepseek2, glm4moe, cohere2moe architectures (#28626) * model: add inverse architecture gating and comprehensive architecture testing for mtp layer filtering * model : slim NextN filter comment, drop test-llama-archs changes --- src/llama-model.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index d10b60afd..f9e9a8bcb 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2644,9 +2644,9 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, filter = [&](uint32_t il) { return il >= hparams.n_layer(); }; } - if ((arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_HY_V3 || arch == LLM_ARCH_GLM_DSA || - arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_DEEPSEEK32) && - hparams.n_layer_nextn > 0) { + // don't filter when n_layer_nextn is repurposed for a router layer the trunk attends + // or when a model is entirely n_layer_nextn layers and has no trunk + if (hparams.n_layer_nextn > 0 && hparams.n_layer() > 0 && hparams.router_layer < 0) { if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) { filter = [&](uint32_t il) { return il >= hparams.n_layer(); }; } else { From aac810230f9ef0cf73a47c56e46e87d0988be348 Mon Sep 17 00:00:00 2001 From: Foad Abo Dahood <32059146+masterFoad@users.noreply.github.com> Date: Fri, 11 Sep 2026 12:30:20 +0300 Subject: [PATCH 10/35] metal : fix idle threads in the remaining iq mul_mv kernels for ne00 < 1024 (#28692) * metal : fix idle threads in the remaining iq mul_mv kernels for ne00 < 1024 Generalize the row split from #28086 to the six other kernels that use the same lane-to-block mapping: iq1_s, iq1_m, iq2_xxs, iq2_xs, iq2_s and iq3_s. Each of them assigns one 32-element chunk per thread, so when a row has fewer than 32 chunks the rest of the simdgroup is idle. When nb32 < 32 and nb32 divides 32, 32/nb32 threads now share each chunk and each takes a slice of the rows, reusing the FC_mul_mv_split function constant and the dispatch wrapper introduced for iq3_xxs. The plain path is untouched: wide matrices keep one thread per chunk and N_R0_ = 4. Only the split path uses N_R0__SPLIT = 8. The K-quants have the same idle-thread issue but a different lane mapping, so they are left for a separate change. * metal : offset the src0 row pointer once in the iq mul_mv kernels q2, dh, sc, qh and signs are all derived from xr, so the row slice offset only has to be applied to xr. * metal : fold iq mul_mv row split into offset0 Compute row0 and row1 before initializing the source pointers and apply the row slice directly to offset0. This keeps x and its derived pointers on the existing path while applying the split row offset once. --- ggml/src/ggml-metal/ggml-metal-device.cpp | 72 +++++ ggml/src/ggml-metal/ggml-metal-impl.h | 6 + ggml/src/ggml-metal/kernels/mul_mv.metal | 308 ++++++++++++++++------ 3 files changed, 304 insertions(+), 82 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index 1137c5f6d..bf3d07e78 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -932,12 +932,24 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta nsg = N_SG_IQ2_XXS; nr0 = N_R0_IQ2_XXS; smem = 256*8+128; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_XXS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ2_XS: { nsg = N_SG_IQ2_XS; nr0 = N_R0_IQ2_XS; smem = 512*8+128; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_XS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ3_XXS: { @@ -957,21 +969,45 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta nsg = N_SG_IQ3_S; nr0 = N_R0_IQ3_S; smem = 512*4; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ3_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ2_S: { nsg = N_SG_IQ2_S; nr0 = N_R0_IQ2_S; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ1_S: { nsg = N_SG_IQ1_S; nr0 = N_R0_IQ1_S; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ1_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ1_M: { nsg = N_SG_IQ1_M; nr0 = N_R0_IQ1_M; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ1_M_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ4_NL: { @@ -1177,12 +1213,24 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m nsg = N_SG_IQ2_XXS; nr0 = N_R0_IQ2_XXS; smem = 256*8+128; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_XXS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ2_XS: { nsg = N_SG_IQ2_XS; nr0 = N_R0_IQ2_XS; smem = 512*8+128; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_XS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ3_XXS: { @@ -1202,21 +1250,45 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m nsg = N_SG_IQ3_S; nr0 = N_R0_IQ3_S; smem = 512*4; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ3_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ2_S: { nsg = N_SG_IQ2_S; nr0 = N_R0_IQ2_S; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ1_S: { nsg = N_SG_IQ1_S; nr0 = N_R0_IQ1_S; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ1_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ1_M: { nsg = N_SG_IQ1_M; nr0 = N_R0_IQ1_M; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ1_M_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ4_NL: { diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index 1fe947633..28a9ba101 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -62,18 +62,23 @@ #define N_R0_IQ1_S 4 #define N_SG_IQ1_S 2 +#define N_R0_IQ1_S_SPLIT 8 #define N_R0_IQ1_M 4 #define N_SG_IQ1_M 2 +#define N_R0_IQ1_M_SPLIT 8 #define N_R0_IQ2_XXS 4 #define N_SG_IQ2_XXS 2 +#define N_R0_IQ2_XXS_SPLIT 8 #define N_R0_IQ2_XS 4 #define N_SG_IQ2_XS 2 +#define N_R0_IQ2_XS_SPLIT 8 #define N_R0_IQ2_S 4 #define N_SG_IQ2_S 2 +#define N_R0_IQ2_S_SPLIT 8 #define N_R0_IQ3_XXS 4 #define N_SG_IQ3_XXS 2 @@ -81,6 +86,7 @@ #define N_R0_IQ3_S 4 #define N_SG_IQ3_S 2 +#define N_R0_IQ3_S_SPLIT 8 #define N_R0_IQ4_NL 2 #define N_SG_IQ4_NL 2 diff --git a/ggml/src/ggml-metal/kernels/mul_mv.metal b/ggml/src/ggml-metal/kernels/mul_mv.metal index fbe8398ea..8e2df2765 100644 --- a/ggml/src/ggml-metal/kernels/mul_mv.metal +++ b/ggml/src/ggml-metal/kernels/mul_mv.metal @@ -1889,8 +1889,19 @@ void kernel_mul_mv_iq2_xxs_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq2_xxs * x = (device const block_iq2_xxs *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -1898,8 +1909,6 @@ void kernel_mul_mv_iq2_xxs_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem); threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 256); { @@ -1912,11 +1921,9 @@ void kernel_mul_mv_iq2_xxs_f32_impl( threadgroup_barrier(mem_flags::mem_threadgroup); } - const int ix = tiisg; - device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { for (short i = 0; i < 32; ++i) { yl[i] = y4[i]; } @@ -1928,7 +1935,7 @@ void kernel_mul_mv_iq2_xxs_f32_impl( device const uint16_t * q2 = xr->qs + 4 * ib; device const half * dh = &xr->d; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { const float db = dh[0]; device const uint8_t * aux8 = (device const uint8_t *)q2; const uint32_t aux32 = q2[2] | (q2[3] << 16); @@ -1948,7 +1955,7 @@ void kernel_mul_mv_iq2_xxs_f32_impl( q2 += args.nb01/2; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -1961,6 +1968,23 @@ void kernel_mul_mv_iq2_xxs_f32_impl( } } +template +void kernel_mul_mv_iq2_xxs_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq2_xxs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq2_xxs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq2_xxs_f32")]] kernel void kernel_mul_mv_iq2_xxs_f32( constant ggml_metal_kargs_mul_mv & args, @@ -1971,7 +1995,7 @@ kernel void kernel_mul_mv_iq2_xxs_f32( uint3 tgpig[[threadgroup_position_in_grid]], ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq2_xxs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + kernel_mul_mv_iq2_xxs_f32_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); } template @@ -1997,8 +2021,19 @@ void kernel_mul_mv_iq2_xs_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq2_xs * x = (device const block_iq2_xs *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2006,8 +2041,6 @@ void kernel_mul_mv_iq2_xs_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem); threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 512); { @@ -2020,11 +2053,9 @@ void kernel_mul_mv_iq2_xs_f32_impl( threadgroup_barrier(mem_flags::mem_threadgroup); } - const int ix = tiisg; - device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { for (short i = 0; i < 32; ++i) { yl[i] = y4[i]; } @@ -2037,7 +2068,7 @@ void kernel_mul_mv_iq2_xs_f32_impl( device const uint8_t * sc = xr->scales + ib; device const half * dh = &xr->d; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { const float db = dh[0]; const uint8_t ls1 = sc[0] & 0xf; const uint8_t ls2 = sc[0] >> 4; @@ -2066,7 +2097,7 @@ void kernel_mul_mv_iq2_xs_f32_impl( sc += args.nb01; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -2079,6 +2110,23 @@ void kernel_mul_mv_iq2_xs_f32_impl( } } +template +void kernel_mul_mv_iq2_xs_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq2_xs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq2_xs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq2_xs_f32")]] kernel void kernel_mul_mv_iq2_xs_f32( constant ggml_metal_kargs_mul_mv & args, @@ -2090,7 +2138,7 @@ kernel void kernel_mul_mv_iq2_xs_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq2_xs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + kernel_mul_mv_iq2_xs_f32_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); } // FC_mul_mv_split: for nb32 < 32 (nb32 divides 32), 32/nb32 threads share each chunk and each takes a slice of the rows @@ -2117,8 +2165,19 @@ void kernel_mul_mv_iq3_xxs_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq3_xxs * x = (device const block_iq3_xxs *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2126,8 +2185,6 @@ void kernel_mul_mv_iq3_xxs_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - threadgroup uint32_t * svalues = (threadgroup uint32_t *)(shmem); threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 256); { @@ -2140,15 +2197,6 @@ void kernel_mul_mv_iq3_xxs_f32_impl( threadgroup_barrier(mem_flags::mem_threadgroup); } - const short ntx = FC_mul_mv_split ? nb32 : 32; - const short nrep = 32 / ntx; - - const short ix = tiisg % ntx; - const short irep = tiisg / ntx; - - const short row0 = (nr0 * irep ) / nrep; - const short row1 = (nr0 * (irep + 1)) / nrep; - device const float * y4 = y + 32 * ix; for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { @@ -2160,9 +2208,9 @@ void kernel_mul_mv_iq3_xxs_f32_impl( const int ib = ib32 % (QK_K / 32); device const block_iq3_xxs * xr = x + ibl; - device const uint8_t * q3 = xr->qs + 8 * ib + (uint64_t) row0*args.nb01; - device const uint16_t * gas = (device const uint16_t *)(xr->qs + QK_K/4) + 2 * ib + (uint64_t) row0*args.nb01/2; - device const half * dh = &xr->d + (uint64_t) row0*args.nb01/2; + device const uint8_t * q3 = xr->qs + 8 * ib; + device const uint16_t * gas = (device const uint16_t *)(xr->qs + QK_K/4) + 2 * ib; + device const half * dh = &xr->d; for (short row = row0; row < row1; row++) { const float db = dh[0]; @@ -2253,8 +2301,19 @@ void kernel_mul_mv_iq3_s_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq3_s * x = (device const block_iq3_s *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2262,8 +2321,6 @@ void kernel_mul_mv_iq3_s_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - threadgroup uint32_t * svalues = (threadgroup uint32_t *) shmem; { int nval = 8; @@ -2272,11 +2329,9 @@ void kernel_mul_mv_iq3_s_f32_impl( threadgroup_barrier(mem_flags::mem_threadgroup); } - const int ix = tiisg; - device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { for (short i = 0; i < 32; ++i) { yl[i] = y4[i]; } @@ -2291,7 +2346,7 @@ void kernel_mul_mv_iq3_s_f32_impl( device const uint8_t * signs = xr->signs + 4 * ib; device const half * dh = &xr->d; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { const float db = dh[0]; const float d = db * (1 + 2*((sc[0] >> 4*(ib%2)) & 0xf)); @@ -2315,7 +2370,7 @@ void kernel_mul_mv_iq3_s_f32_impl( signs += args.nb01; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -2328,6 +2383,23 @@ void kernel_mul_mv_iq3_s_f32_impl( } } +template +void kernel_mul_mv_iq3_s_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq3_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq3_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq3_s_f32")]] kernel void kernel_mul_mv_iq3_s_f32( constant ggml_metal_kargs_mul_mv & args, @@ -2339,7 +2411,7 @@ kernel void kernel_mul_mv_iq3_s_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq3_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + kernel_mul_mv_iq3_s_f32_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); } template @@ -2365,8 +2437,19 @@ void kernel_mul_mv_iq2_s_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq2_s * x = (device const block_iq2_s *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2374,8 +2457,6 @@ void kernel_mul_mv_iq2_s_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - //threadgroup uint64_t * svalues = (threadgroup uint64_t *) shmem; //{ // int nval = 32; @@ -2384,11 +2465,9 @@ void kernel_mul_mv_iq2_s_f32_impl( // threadgroup_barrier(mem_flags::mem_threadgroup); //} - const short ix = tiisg; - device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { for (short i = 0; i < 32; ++i) { yl[i] = y4[i]; } @@ -2403,7 +2482,7 @@ void kernel_mul_mv_iq2_s_f32_impl( device const uint8_t * signs = qs + QK_K/8; device const half * dh = &xr->d; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { const float db = dh[0]; const float d1 = db * (0.5f + (sc[0] & 0xf)); const float d2 = db * (0.5f + (sc[0] >> 4)); @@ -2428,7 +2507,7 @@ void kernel_mul_mv_iq2_s_f32_impl( signs += args.nb01; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -2441,6 +2520,23 @@ void kernel_mul_mv_iq2_s_f32_impl( } } +template +void kernel_mul_mv_iq2_s_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq2_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq2_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq2_s_f32")]] kernel void kernel_mul_mv_iq2_s_f32( constant ggml_metal_kargs_mul_mv & args, @@ -2452,7 +2548,7 @@ kernel void kernel_mul_mv_iq2_s_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq2_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + kernel_mul_mv_iq2_s_f32_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); } template @@ -2478,8 +2574,19 @@ void kernel_mul_mv_iq1_s_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq1_s * x = (device const block_iq1_s *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2487,13 +2594,9 @@ void kernel_mul_mv_iq1_s_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - - const short ix = tiisg; - device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { float sumy = 0; for (short i = 0; i < 32; ++i) { yl[i] = y4[i]; @@ -2508,7 +2611,7 @@ void kernel_mul_mv_iq1_s_f32_impl( device const uint16_t * qh = xr->qh + ib; device const half * dh = &xr->d; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700))); constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 5) & 0x700))); constant uint8_t * grid3 = (constant uint8_t *)(iq1s_grid_gpu + (qs[2] | ((qh[0] << 2) & 0x700))); @@ -2528,7 +2631,7 @@ void kernel_mul_mv_iq1_s_f32_impl( qh += args.nb01/2; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -2541,6 +2644,23 @@ void kernel_mul_mv_iq1_s_f32_impl( } } +template +void kernel_mul_mv_iq1_s_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq1_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq1_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq1_s_f32")]] kernel void kernel_mul_mv_iq1_s_f32( constant ggml_metal_kargs_mul_mv & args, @@ -2551,7 +2671,7 @@ kernel void kernel_mul_mv_iq1_s_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq1_s_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); + kernel_mul_mv_iq1_s_f32_disp(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); } template @@ -2577,8 +2697,19 @@ void kernel_mul_mv_iq1_m_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq1_m * x = (device const block_iq1_m *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2586,15 +2717,11 @@ void kernel_mul_mv_iq1_m_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - - const short ix = tiisg; - device const float * y4 = y + 32 * ix; iq1m_scale_t scale; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { float4 sumy = {0.f}; for (short i = 0; i < 8; ++i) { yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0]; @@ -2611,7 +2738,7 @@ void kernel_mul_mv_iq1_m_f32_impl( device const uint8_t * qh = xr->qh + 2 * ib; device const uint16_t * sc = (device const uint16_t *)xr->scales; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700))); @@ -2637,7 +2764,7 @@ void kernel_mul_mv_iq1_m_f32_impl( qh += args.nb01; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -2650,6 +2777,23 @@ void kernel_mul_mv_iq1_m_f32_impl( } } +template +void kernel_mul_mv_iq1_m_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq1_m_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq1_m_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq1_m_f32")]] kernel void kernel_mul_mv_iq1_m_f32( constant ggml_metal_kargs_mul_mv & args, @@ -2660,7 +2804,7 @@ kernel void kernel_mul_mv_iq1_m_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq1_m_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); + kernel_mul_mv_iq1_m_f32_disp(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); } template @@ -3239,13 +3383,13 @@ template [[host_name("kernel_mul_mv_id_q3_K_f32")]] kernel kernel_mul_mv_id_t template [[host_name("kernel_mul_mv_id_q4_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_q5_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_q6_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq1_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq1_m_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq2_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq2_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq1_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq1_m_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq2_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq2_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_iq3_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq3_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq2_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq3_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq2_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_iq4_nl_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_iq4_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_tq2_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; From a2878d30df0130dde503a7d9ba30d3d21bd71b9f Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 11 Sep 2026 12:41:54 +0300 Subject: [PATCH 11/35] metal : single-source fusion table + fusion debug rework (#28164) * metal : rework fusion patterns into a single table All fusable op patterns for the Metal backend are now declared once in a fusion table (ggml-metal-fuse.cpp) and consumed by both the graph optimizer (ggml_metal_fuse_max, packing) and the op encoders (ggml_metal_fuse_next, compute). The two phases share the same pattern table plus ggml_can_fuse_subgraph_ext for the structural checks, and differ only in the mode used for the pattern check (STRUCTURAL at optimize time, since tensors are not allocated yet, and FULL at compute time, including Metal buffer placement). This also protects the snake activation (MUL + SIN + SQR + MUL + ADD) from being reordered during graph optimization, which was previously unprotected. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * metal : fix absolute output indices in fusion patterns ggml_can_fuse_subgraph_ext expects the outputs array to contain absolute graph node indices (it indexes cgraph->nodes[outputs[i]]), but the fusion table query was passing a relative index (n_ops - 1). As a result the last node of every pattern was not recognized as an output and was subjected to the elidable use-count check, which failed for essentially all fusions. This silently disabled the norm/MUL fusion and caused a ~5% token-generation regression. Pass the absolute graph index of the last node instead. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * metal : fuse gated_delta_net with cache cpy Add GGML_METAL_FUSE_GDN_CACHE to the fusion table: when the gated_delta_net kernel is followed by a cpy that scatters its recurrent state snapshots into the KV cache, the kernel writes the snapshots straight into the cache buffer and the trailing cpy is elided. The gdn output has other consumers (the attn scores view), so unlike the elision-chain patterns this is not a simple chain: a 'raw' flag on the fusion pattern skips the generic chain/shape and ggml_can_fuse_subgraph_ext checks, making the pattern-specific check callback the sole validator. Packing (ggml_metal_fuse_max) now matches on the same view-transparent node sequence that the compute phase uses, so the gdn + cache cpy group is packed along with any intermediate views and stays adjacent through the reorder. The fused cpy is a view consumer of the gdn (it writes the cache directly), so its mem-range is skipped in the encoder; the skip is restricted to CPY nodes consuming the previous fused node through a view so other fusions are unaffected. Add test_gated_delta_net_cache_fusion and register 5 cases. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * metal : drop is_view_consumer mem-range skip The is_view_consumer skip was carried over from the upstream gated_delta_net cache-fusion draft, but it is not needed: keeping the elided cpy's mem-range in the concurrency tracker only ever adds a (conservative) memory barrier at the fusion point. It can never remove a barrier, so it cannot introduce a race. The worst case is one spurious barrier per gdn+cache-cpy fusion, which is within run-to-run noise on Qwen3.5-0.8B Q8_0. Dropping the check keeps the mem-range loop uniform for all fused groups. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * metal : rename gated_delta_net fused state output args Rename the fused cache-write kernel argument to match the rest of the kargs: state_out_stride -> nb_out (and widen it to uint64_t), and the local buffer id bid_state_out -> bid_out. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * metal : rename raw fusion flag to unsafe raw did not convey that the flag opts a fusion pattern out of the generic elision-chain safety net (ggml_can_fuse_subgraph_ext + chain/shape checks). rename it to 'unsafe' to make explicit that the pattern's check callback is the sole validator and must re-establish the safety guarantees itself. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * metal : tidy fusion pattern checks and table - const-correct ggml_metal_fuse_outputs buffer - annotate unused check-callback parameters - drop a redundant size_t cast - align the ops/table initializers and add blank-line separation Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * metal : add generic fusion stats via ad-hoc proc-address API Add a device-owned fusion context that lets a test tool count how many times each fusion pattern fires and toggle fusion. It is exposed through the ad-hoc ggml_backend_reg_get_proc_address mechanism with generic names so the testing tool is backend-agnostic: - ggml_backend_fusion_stats_init: start collecting fusion stats; when a context is created afterwards it registers the labels/counters and encodes single-threaded (n_cb == 0) so the counters are race-free - ggml_backend_fusion_stats_reset / _get_stats / _set_enabled The context lives on the metal device (not on the last backend context), so counters accumulate across contexts and reads are always consistent. The enable/disable toggle is initialized from GGML_METAL_FUSION_DISABLE and can be overridden by the test through set_enabled. Labels are synthesized from the fuse table via ggml_metal_fuse_label (e.g. "GATED_DELTA_NET+CPY"). Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : add fusion count regression test with per-backend baseline test-fusion runs every dummy model generated by test-llama-archs on a single backend (single-threaded encoding, n_cb == 0) with fusion enabled and disabled, and for each mode (prefill / decode) reports the per-fusion counters and the NMSE between the fused and unfused logits, plus the NMSE against a CPU reference. A fusion pattern that silently stops matching (or fires when it should not) is caught as a regression by comparing the counters against a committed per-backend TSV baseline: - --record writes the golden baseline, --check (default) validates it - the unfused run doubles as a control: its counters must be all-zero - NMSE is skipped when it is NaN or the arch is already broken on the device (e.g. plamo2 on Metal), so the count check is the hard gate - baseline counts depend only on graph structure, not weights (verified stable across weight seeds) - the fusion stats API is resolved through the ad-hoc get_proc_address mechanism with generic names; a backend that does not export it makes the test fail with an error The committed MTL0.tsv baseline covers 110 dummy archs (298 rows). Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : rename fusion api helpers to match stats_init signature Align the test with the ad-hoc fusion stats API: fusion_stats_init no longer takes an enable bool (stats are turned on by calling it), so the proc-address wrappers and typedefs are renamed to the api_* convention. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : rename backend to device in fusion test CLI The fusion test operates on a compute device (e.g. MTL0), not a backend, so rename the --backend argument to --device and the backend_name variable to device_name. Keep "backend" where it refers to the ggml backend interface (the ad-hoc proc-address mechanism). Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : add --model and --help to fusion test --model FILE runs the fusion regression test over a single model file instead of enumerating a --models DIR. --models and --model are mutually exclusive. Also add a --help/-h option that prints the usage. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : use backend base name for fusion baseline output The fusion test is invoked with a specific device name (e.g. MTL0), but its output - the recorded baseline and the header it writes - should be named after the backend base name (e.g. MTL, via ggml_backend_reg_name), since the counters depend on the backend, not on the specific device index. Rename the committed baseline to MTL.tsv. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : run fusion test from ci instead of ctest The fusion test needs Metal and generates a lot of dummy models, so it does not belong in the generic ctest suite. Move it to ci/run.sh as gg_run_test_fusion, gated on GG_BUILD_METAL like gg_run_test_llama_archs_tensor_split: it generates the dummy models with test-llama-archs -o and then validates the fusion counts against the committed baseline. test-fusion.cpp is still built (llama_build) but no longer registered as a ctest. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : align fusion baseline TSV columns Pad the TSV fields to fixed widths so the columns line up regardless of the variable arch and fusion-label lengths, and trim each field on parse so the padded file is still accepted. Regenerate the committed MTL.tsv baseline in the padded format (data unchanged, verified identical modulo padding). Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : widen label column and align fusion TSV header Give the label column more room (28 chars) and fix the column header widths so they match the data rows (moe/mode/label), keeping the header aligned with the values. Regenerate the MTL.tsv baseline in the new format (data unchanged). Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : switch fusion baseline from TSV to CSV Use comma-separated values like the rest of the project, keeping the padded, aligned columns. Split on ',' and trim on parse. Rename the committed baseline to MTL.csv (data unchanged, verified identical modulo padding/separator). Update the ci/run.sh check path accordingly. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * cont : rebase + update MTL stats * tests : avoid graph reallocations for some archs * metal : tidy fusion debugging context and op init - simplify the shared fusion debugging context comments - shorten the ggml_metal_fusion struct comment - align the ggml_metal_fuse struct fields and comments - move the fusion parameter of ggml_metal_op_init right after dev Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : dedup fusion baseline into any mode prefill and decode always produce the same per-graph fusion count, so store a single row per label with mode = "any" and the per-graph count instead of two rows. this halves the baseline size and keeps the check stable. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * ci : move fusion model generation to a separate step the dummy models generated by test-llama-archs are reused by other tests, so generate them once in their own step instead of inside test_fusion. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : bump nmse thold * models : fix plamo2 graph * tests : remove "skip" logic from test-fusion * tests : set qwen3tts dummy vocab to codec head size the dummy qwen3tts model used a vocab of 4096 while the codec head is 3072, so the graph padded the output with -inf which made the NMSE in test-fusion produce NaN. use the exact codec head size instead so the padding is not generated at all. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : regen fusion baseline reflect the plamo2 graph fix, which changed its fusion pattern split (RMS_NORM+MUL 11->10, RMS_NORM+MUL+ADD 3->4; same total). Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * ci : skip dummy model generation on OpenVINO test-llama-archs does not build on the OpenVINO platform, so do not try to generate the dummy models there. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * cont : minor * tests : enable test-llama-archs on windows * cont : disable on windows + workaround * metal : naming nits * test-fusion : add instructions to update baseline * context : fix Kimi-K3 graph reserve * fusion : update MTL * cont : fix naming * metal : rework fusion info storage Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp * metal : align fusion info API Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp * metal : use opaque fusion handle in ad-hoc API Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp * ci : move fusion test to dedicated workflow Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp * cont : run only on ggml changes * cont : simplify * fusion : remove multi-output stuff for now * ci : fix typo --- .github/workflows/fusion.yml | 67 +++ ci/run.sh | 30 + ggml/src/ggml-metal/CMakeLists.txt | 1 + ggml/src/ggml-metal/ggml-metal-common.cpp | 47 +- ggml/src/ggml-metal/ggml-metal-context.h | 1 + ggml/src/ggml-metal/ggml-metal-context.m | 55 +- ggml/src/ggml-metal/ggml-metal-device.h | 5 + ggml/src/ggml-metal/ggml-metal-device.m | 17 + ggml/src/ggml-metal/ggml-metal-fusion.cpp | 502 ++++++++++++++++ ggml/src/ggml-metal/ggml-metal-fusion.h | 104 ++++ ggml/src/ggml-metal/ggml-metal-impl.h | 1 + ggml/src/ggml-metal/ggml-metal-ops.cpp | 274 ++++----- ggml/src/ggml-metal/ggml-metal-ops.h | 7 +- ggml/src/ggml-metal/ggml-metal.cpp | 42 ++ .../ggml-metal/kernels/gated_delta_net.metal | 20 +- src/llama-context.cpp | 2 + src/models/minimax-01.cpp | 1 + src/models/plamo2.cpp | 10 +- src/models/qwen3vl.cpp | 1 + tests/.gitignore | 1 + tests/CMakeLists.txt | 4 +- tests/fusion/MTL.csv | 154 +++++ tests/test-backend-ops.cpp | 123 ++++ tests/test-fusion.cpp | 565 ++++++++++++++++++ tests/test-llama-archs.cpp | 3 +- tests/test-save-load-state.cpp | 4 +- 26 files changed, 1800 insertions(+), 241 deletions(-) create mode 100644 .github/workflows/fusion.yml create mode 100644 ggml/src/ggml-metal/ggml-metal-fusion.cpp create mode 100644 ggml/src/ggml-metal/ggml-metal-fusion.h create mode 100644 tests/fusion/MTL.csv create mode 100644 tests/test-fusion.cpp diff --git a/.github/workflows/fusion.yml b/.github/workflows/fusion.yml new file mode 100644 index 000000000..ad7d5ab60 --- /dev/null +++ b/.github/workflows/fusion.yml @@ -0,0 +1,67 @@ +name: Fusion + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/fusion.yml', + 'ggml/**', + 'tests/fusion/**', + 'tests/test-fusion.cpp' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/fusion.yml', + 'ggml/**', + 'tests/fusion/**', + 'tests/test-fusion.cpp' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + # TODO: add jobs for other backends as they adopt the fusion debug API + metal: + runs-on: [self-hosted, macOS, ARM64] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Build + id: cmake_build + run: | + cmake -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLAMA_FATAL_WARNINGS=ON \ + -DLLAMA_OPENSSL=OFF \ + -DGGML_SCHED_NO_REALLOC=ON \ + -DGGML_BLAS=OFF \ + -DGGML_METAL=ON + time cmake --build build --config Release --target test-llama-archs -j $(sysctl -n hw.logicalcpu) + time cmake --build build --config Release --target test-fusion -j $(sysctl -n hw.logicalcpu) + + - name: Generate models + id: generate_models + run: | + rm -rf build-ci-models && mkdir -p build-ci-models + ./build/bin/test-llama-archs -o build-ci-models + + - name: Test fusion + id: test_fusion + run: | + ./build/bin/test-fusion --models build-ci-models --device MTL0 --check tests/fusion/MTL.csv diff --git a/ci/run.sh b/ci/run.sh index 546359727..294cbe57b 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -334,6 +334,35 @@ function gg_sum_test_llama_archs_tensor_split { gg_printf '```\n' } +# test_llama_archs_models + +function gg_run_test_llama_archs_models { + cd ${SRC} + + set -e + + # TODO: fix and re-enable `test-llama-archs` on OpenVINO + # TODO: the `test-llama-archs` currently does not build on Windows, so we check if the binary exists + if [ -z ${GG_BUILD_OPENVINO} ] && [ -f ./build-ci-release/bin/test-llama-archs ]; then + rm -rf build-ci-models && mkdir -p build-ci-models + + # generate the dummy models used by the model-dependent tests + ./build-ci-release/bin/test-llama-archs -o build-ci-models 2>&1 + fi + + set +e +} + +function gg_sum_test_llama_archs_models { + gg_printf '### %s\n\n' "${ci}" + + gg_printf 'Generates the dummy models used by the model-dependent tests\n' + gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" + gg_printf '```\n' + gg_printf '%s\n' "$(cat $OUT/${ci}.log)" + gg_printf '```\n' +} + # test_scripts function gg_run_test_scripts { @@ -790,6 +819,7 @@ ret=0 test $ret -eq 0 && gg_run ctest_debug test $ret -eq 0 && gg_run ctest_release +test $ret -eq 0 && gg_run test_llama_archs_models test $ret -eq 0 && gg_run test_llama_archs_tensor_split if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then diff --git a/ggml/src/ggml-metal/CMakeLists.txt b/ggml/src/ggml-metal/CMakeLists.txt index a661e710a..e7afdb695 100644 --- a/ggml/src/ggml-metal/CMakeLists.txt +++ b/ggml/src/ggml-metal/CMakeLists.txt @@ -10,6 +10,7 @@ ggml_add_backend_library(ggml-metal ggml-metal-device.cpp ggml-metal-common.cpp ggml-metal-context.m + ggml-metal-fusion.cpp ggml-metal-ops.cpp ggml-metal-tuning.cpp ) diff --git a/ggml/src/ggml-metal/ggml-metal-common.cpp b/ggml/src/ggml-metal/ggml-metal-common.cpp index 6f1638a11..05755eb3b 100644 --- a/ggml/src/ggml-metal/ggml-metal-common.cpp +++ b/ggml/src/ggml-metal/ggml-metal-common.cpp @@ -1,4 +1,5 @@ #include "ggml-metal-common.h" +#include "ggml-metal-fusion.h" #include "ggml.h" #include "ggml-impl.h" @@ -390,59 +391,31 @@ static std::vector ggml_metal_graph_optimize_reorder(const std::vectorn_nodes; - enum ggml_op ops[MAX_FUSE]; - std::vector nodes; nodes.reserve(gf->n_nodes); // fuse nodes: // we don't want to make reorders that break fusing, so we first pack all fusable tensors // and perform the reorder over the fused nodes. after the reorder is done, we unfuse + // + // the fusable sequences are declared in the fusion table (ggml-metal-fuse.cpp), so the + // packing here is driven by the same patterns that the op encoders will later use for (int i = 0; i < n; i++) { node_info node = { /*.node =*/ gf->nodes[i], /*.fused =*/ {}, }; - // fuse only ops that start with these operations - // can be expanded when needed - if (node.op() == GGML_OP_ADD || - node.op() == GGML_OP_NORM || - node.op() == GGML_OP_RMS_NORM) { - ops[0] = node.op(); + const int f = ggml_metal_fusion_max(gf, i); - int f = i + 1; - while (f < n && f < i + MAX_FUSE) { - // conservatively allow fusing only these ops - // can be expanded when needed - if (gf->nodes[f]->op != GGML_OP_ADD && - gf->nodes[f]->op != GGML_OP_MUL && - gf->nodes[f]->op != GGML_OP_NORM && - gf->nodes[f]->op != GGML_OP_RMS_NORM) { - break; - } - ops[f - i] = gf->nodes[f]->op; - f++; - } + // add the fused tensors into the node info so we can unfuse them later + for (int k = 1; k < f; k++) { + ++i; - f -= i; - for (; f > 1; f--) { - if (ggml_can_fuse(gf, i, ops, f)) { - break; - } - } - - // add the fused tensors into the node info so we can unfuse them later - for (int k = 1; k < f; k++) { - ++i; - - // the .dst() becomes the last fused tensor - node.add_fused(gf->nodes[i]); - } + // the .dst() becomes the last fused tensor + node.add_fused(gf->nodes[i]); } nodes.push_back(std::move(node)); diff --git a/ggml/src/ggml-metal/ggml-metal-context.h b/ggml/src/ggml-metal/ggml-metal-context.h index abf4b06ed..b538b1ad2 100644 --- a/ggml/src/ggml-metal/ggml-metal-context.h +++ b/ggml/src/ggml-metal/ggml-metal-context.h @@ -33,6 +33,7 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx); void ggml_metal_set_n_cb (ggml_metal_t ctx, int n_cb); void ggml_metal_set_abort_callback (ggml_metal_t ctx, ggml_abort_callback abort_callback, void * user_data); + bool ggml_metal_supports_family (ggml_metal_t ctx, int family); void ggml_metal_capture_next_compute(ggml_metal_t ctx); diff --git a/ggml/src/ggml-metal/ggml-metal-context.m b/ggml/src/ggml-metal/ggml-metal-context.m index 6cdc4006b..bf4fe2dcd 100644 --- a/ggml/src/ggml-metal/ggml-metal-context.m +++ b/ggml/src/ggml-metal/ggml-metal-context.m @@ -6,6 +6,7 @@ #import "ggml-metal-impl.h" #import "ggml-metal-common.h" #import "ggml-metal-ops.h" +#import "ggml-metal-fusion.h" #import @@ -36,15 +37,12 @@ struct ggml_metal { // additional, inference-time compiled pipelines ggml_metal_pipelines_t pipelines_ext; - bool use_fusion; bool use_concurrency; bool use_graph_optimize; int debug_graph; - int debug_fusion; - // how many times a given op was fused - uint64_t fuse_cnt[GGML_OP_COUNT]; + struct ggml_metal_fusion_info * finfo; // capture state int capture_compute; @@ -139,7 +137,6 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) { res->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT); - res->use_fusion = getenv("GGML_METAL_FUSION_DISABLE") == nil; res->use_concurrency = getenv("GGML_METAL_CONCURRENCY_DISABLE") == nil; { @@ -147,20 +144,19 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) { res->debug_graph = val ? atoi(val) : 0; } - { - const char * val = getenv("GGML_METAL_FUSION_DEBUG"); - res->debug_fusion = val ? atoi(val) : 0; - } - res->use_graph_optimize = true; if (getenv("GGML_METAL_GRAPH_OPTIMIZE_DISABLE") != NULL) { res->use_graph_optimize = false; } - memset(res->fuse_cnt, 0, sizeof(res->fuse_cnt)); + res->finfo = ggml_metal_device_get_fusion_info(dev); + if (ggml_metal_fusion_info_stats(res->finfo)) { + ggml_metal_fusion_info_labels_init(res->finfo); + res->n_cb = 0; + } - GGML_LOG_INFO("%s: use fusion = %s\n", __func__, res->use_fusion ? "true" : "false"); + GGML_LOG_INFO("%s: use fusion = %s\n", __func__, ggml_metal_fusion_info_enabled(res->finfo) ? "true" : "false"); GGML_LOG_INFO("%s: use concurrency = %s\n", __func__, res->use_concurrency ? "true" : "false"); GGML_LOG_INFO("%s: use graph optimize = %s\n", __func__, res->use_graph_optimize ? "true" : "false"); @@ -222,15 +218,18 @@ void ggml_metal_free(ggml_metal_t ctx) { ctx->pipelines_ext = nil; } - if (ctx->debug_fusion > 0) { + if (ggml_metal_fusion_info_debug(ctx->finfo) > 0) { GGML_LOG_DEBUG("%s: fusion stats:\n", __func__); - for (int i = 0; i < GGML_OP_COUNT; i++) { - if (ctx->fuse_cnt[i] == 0) { + + const int n_fusions = ggml_metal_fusion_info_n_fusions(ctx->finfo); + for (int i = 0; i < n_fusions; i++) { + const uint64_t count = ggml_metal_fusion_info_count(ctx->finfo, i); + if (count == 0) { continue; } // note: cannot use ggml_log here - GGML_LOG_DEBUG("%s: - %s: %" PRIu64 "\n", __func__, ggml_op_name((enum ggml_op) i), ctx->fuse_cnt[i]); + GGML_LOG_DEBUG("%s: - %s: %" PRIu64 "\n", __func__, ggml_metal_fusion_info_label(ctx->finfo, i), count); } } @@ -481,10 +480,17 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * @autoreleasepool { ctx->gf = gf; - ctx->n_nodes_0 = MIN(n_main, gf->n_nodes); - ctx->n_nodes_1 = gf->n_nodes - ctx->n_nodes_0; + if (ctx->n_cb == 0) { + // single-threaded encoding: the whole graph is encoded by one command buffer + ctx->n_nodes_0 = gf->n_nodes; + ctx->n_nodes_1 = 0; + ctx->n_nodes_per_cb = 0; + } else { + ctx->n_nodes_0 = MIN(n_main, gf->n_nodes); + ctx->n_nodes_1 = gf->n_nodes - ctx->n_nodes_0; - ctx->n_nodes_per_cb = (ctx->n_nodes_1 + ctx->n_cb - 1) / ctx->n_cb; + ctx->n_nodes_per_cb = (ctx->n_nodes_1 + ctx->n_cb - 1) / ctx->n_cb; + } if (ctx->capture_compute >= 0) { ctx->capture_compute--; @@ -682,6 +688,12 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx) { } void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { + // when fusion stats are collected the graph must be encoded by a single thread so the + // counters are race-free; override whatever the caller requested + if (ggml_metal_fusion_info_stats(ctx->finfo)) { + n_cb = 0; + } + if (ctx->n_cb != n_cb) { ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_COMMAND_BUFFERS); @@ -717,13 +729,12 @@ void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { ctx->dev, cmd_buf, ctx->gf, + ctx->finfo, idx_start, idx_end, - ctx->use_fusion, ctx->use_concurrency, ctx->capture_compute, - ctx->debug_graph, - ctx->debug_fusion); + ctx->debug_graph); for (int idx = 0; idx < ggml_metal_op_n_nodes(ctx_op); ++idx) { const int res = ggml_metal_op_encode(ctx_op, idx); diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index 31fc07d44..ced33aadf 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -325,6 +325,11 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te const struct ggml_metal_device_props * ggml_metal_device_get_props(ggml_metal_device_t dev); +struct ggml_metal_fusion_info; + +// the device-owned fusion debugging context (NULL unless fusion debugging is enabled) +struct ggml_metal_fusion_info * ggml_metal_device_get_fusion_info(ggml_metal_device_t dev); + // // device buffers // diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index afd6f5210..5654c5004 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1,4 +1,5 @@ #import "ggml-metal-device.h" +#import "ggml-metal-fusion.h" #import "ggml-impl.h" #import "ggml-backend-impl.h" @@ -896,6 +897,9 @@ struct ggml_metal_device { struct ggml_metal_device_props props; + // shared fusion debugging context + struct ggml_metal_fusion_info * finfo; + // virtual address for GPU memory allocations atomic_uintptr_t addr_virt; }; @@ -1274,6 +1278,13 @@ ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) { dev->props.max_working_set_size = dev->mtl_device.maxBufferLength; } + { + const char * val = getenv("GGML_METAL_FUSION_DEBUG"); + dev->finfo = ggml_metal_fusion_info_init( + getenv("GGML_METAL_FUSION_DISABLE") == nil, + val ? atoi(val) : 0); + } + snprintf(dev->props.name, sizeof(dev->props.name), "%s%d", "MTL", device); const char * gpu_name = [[dev->mtl_device name] UTF8String]; if (n_devices > 1) { @@ -1348,6 +1359,8 @@ void ggml_metal_device_free(ggml_metal_device_t dev) { assert(dev != NULL); @autoreleasepool { + ggml_metal_fusion_info_free(dev->finfo); + ggml_metal_rsets_free(dev->rsets); ggml_metal_library_free(dev->library); @@ -1935,6 +1948,10 @@ static void ggml_metal_device_disable_tensor(ggml_metal_device_t dev) { dev->props.has_tensor = false; } +struct ggml_metal_fusion_info * ggml_metal_device_get_fusion_info(ggml_metal_device_t dev) { + return dev->finfo; +} + // // device buffers // diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.cpp b/ggml/src/ggml-metal/ggml-metal-fusion.cpp new file mode 100644 index 000000000..ac3ac0414 --- /dev/null +++ b/ggml/src/ggml-metal/ggml-metal-fusion.cpp @@ -0,0 +1,502 @@ +#include "ggml-metal-fusion.h" + +#include "ggml-backend-impl.h" +#include "ggml-metal-device.h" + +#include +#include +#include + +// ---- helpers ------------------------------------------------------------- + +// true if two tensors live in the same Metal buffer +static bool ggml_metal_fusion_same_buffer(const ggml_tensor * a, const ggml_tensor * b) { + if (!a || !b) { + return false; + } + + ggml_backend_buffer_t ba = a->view_src ? a->view_src->buffer : a->buffer; + ggml_backend_buffer_t bb = b->view_src ? b->view_src->buffer : b->buffer; + + ggml_metal_buffer_t ca = (ggml_metal_buffer_t) ba->context; + ggml_metal_buffer_t cb = (ggml_metal_buffer_t) bb->context; + + return ggml_metal_buffer_get_id(ca, a).metal == ggml_metal_buffer_get_id(cb, b).metal; +} + +// ---- pattern checks ------------------------------------------------------ + +// NORM/RMS_NORM + MUL + ADD: the weight/bias of each fused step must match the norm input +// width, be contiguous rows, and the fused outputs must stay F32 +static bool ggml_metal_fusion_check_norm( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(mode); + + GGML_ASSERT(fusion->n_ops >= 2); + + for (int j = 1; j < fusion->n_ops; j++) { + // the fused MUL/ADD must read the previous node as src0 + if (nodes[j]->src[0] != nodes[j - 1]) { + return false; + } + + // the weight/bias must have the same row width as the norm input + if (nodes[j]->src[1]->ne[0] != nodes[0]->ne[0]) { + return false; + } + + if (!ggml_is_contiguous_rows(nodes[j]->src[1])) { + return false; + } + + if (nodes[j]->type != GGML_TYPE_F32) { + return false; + } + } + + return true; +} + +// ADD x N: each ADD reads the previous ADD as src0, and all addends must share layout +// (and, in FULL mode, live in the same Metal buffer) +static bool ggml_metal_fusion_check_add_chain( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + ggml_metal_fusion_mode mode) { + GGML_ASSERT(fusion->n_ops >= 2); + + for (int j = 1; j < fusion->n_ops; j++) { + if (nodes[j]->src[0] != nodes[j - 1]) { + return false; + } + + if (!ggml_are_same_layout(nodes[j]->src[1], nodes[j - 1]->src[1])) { + return false; + } + + if (mode == GGML_METAL_FUSION_FULL) { + if (!ggml_metal_fusion_same_buffer(nodes[j]->src[1], nodes[0]->src[1])) { + return false; + } + } + } + + return true; +} + +// GATED_DELTA_NET + CPY: the trailing cpy scatters the gdn state snapshots into the recurrent +// cache, so the gdn kernel writes them straight to the cache and the cpy is elided. +// mirrors ggml_metal_op_can_fuse_gdn_cache (PR #25788). the gdn output has other consumers (the +// attn scores view), so unlike the other patterns this is not an elision chain: the structural +// checks live entirely in this callback (unsafe = true). +static bool ggml_metal_fusion_check_gdn_cache( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(fusion); + + const ggml_tensor * gdn = nodes[0]; + const ggml_tensor * cpy = nodes[1]; + + // the kernel skips the snapshot tail, so the gdn output must not be a graph output + if (gdn->type != GGML_TYPE_F32 || (gdn->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return false; + } + + if (cpy->op != GGML_OP_CPY || (cpy->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return false; + } + + const int64_t S_v = gdn->src[2]->ne[0]; + const int64_t H = gdn->src[2]->ne[1]; + const int64_t n_tokens = gdn->src[2]->ne[2]; + const int64_t n_seqs = gdn->src[2]->ne[3]; + const int64_t K = ggml_get_op_params_i32(gdn, 0); + const size_t tail_off = ggml_row_size(GGML_TYPE_F32, S_v * H * n_tokens * n_seqs); + + const int64_t D = S_v * S_v * H; + const int64_t n_written = std::min(n_tokens, K); + + const ggml_tensor * src = cpy->src[0]; // gdn snapshot tail view + const ggml_tensor * dst = cpy->src[1]; // cache view + + // src must be this gdn's snapshot tail (contiguous, at the tail offset) + if (src->op != GGML_OP_VIEW || src->view_src != gdn || + src->view_offs != tail_off || !ggml_is_contiguous(src)) { + return false; + } + + const int64_t expected_ne[GGML_MAX_DIMS] = { D, n_seqs, n_written, 1 }; + if (dst->type != GGML_TYPE_F32 || + !std::equal(expected_ne, expected_ne + GGML_MAX_DIMS, dst->ne) || + dst->nb[0] != ggml_type_size(GGML_TYPE_F32) || + dst->nb[1] != ggml_row_size(GGML_TYPE_F32, D)) { + return false; + } + + if (mode == GGML_METAL_FUSION_FULL) { + // the cache must be allocated so the kernel can write straight to its buffer + if (dst->data == nullptr) { + return false; + } + } + + return true; +} + +// MUL + SIN + SQR + MUL + ADD (snake activation) +static bool ggml_metal_fusion_check_snake( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(fusion); + GGML_UNUSED(mode); + + const ggml_tensor * mul0 = nodes[0]; + const ggml_tensor * sin_node = nodes[1]; + const ggml_tensor * sqr = nodes[2]; + const ggml_tensor * mul1 = nodes[3]; + const ggml_tensor * add = nodes[4]; + + // x carries the full activation shape, a is the broadcast operand + const ggml_tensor * x = ggml_are_same_shape(mul0, mul0->src[0]) ? mul0->src[0] : mul0->src[1]; + const ggml_tensor * a = (x == mul0->src[0]) ? mul0->src[1] : mul0->src[0]; + + // mul1 reads sqr and inv_b in either operand order + const ggml_tensor * inv_b = (mul1->src[0] == sqr) ? mul1->src[1] : mul1->src[0]; + + // closure check: the trailing add reads the same x as the leading mul + const ggml_tensor * x_in_add = (add->src[0] == mul1) ? add->src[1] : add->src[0]; + + // x is in the supported whitelist and every chain intermediate shares x's type. + // a and inv_b bind as device const float * in the kernel, so they stay F32. + const bool types_ok = + (x->type == GGML_TYPE_F32 || x->type == GGML_TYPE_F16 || x->type == GGML_TYPE_BF16) && + (a->type == GGML_TYPE_F32) && (inv_b->type == GGML_TYPE_F32) && + (mul0->type == x->type) && (sin_node->type == x->type) && + (sqr->type == x->type) && (mul1->type == x->type) && + (add->type == x->type); + + // a / inv_b collapse to [1, C, 1, 1], x and add stay 2D + const bool shape_ok = ggml_are_same_shape(a, inv_b) && a->ne[0] == 1 && a->ne[1] == x->ne[1]; + const bool dim_ok = + (x->ne[2] == 1) && (x->ne[3] == 1) && + (add->ne[2] == 1) && (add->ne[3] == 1) && + (a->ne[2] == 1) && (a->ne[3] == 1) && + (inv_b->ne[2] == 1) && (inv_b->ne[3] == 1); + + // kernel reads x[idx] and a[c] / inv_b[c] linearly, so every operand is contiguous + const bool contig_ok = + ggml_is_contiguous(x) && ggml_is_contiguous(add) && + ggml_is_contiguous(a) && ggml_is_contiguous(inv_b); + + return types_ok && shape_ok && dim_ok && contig_ok && x_in_add == x; +} + +// ---- patterns ------------------------------------------------------------ + +static const ggml_op ops_norm_mul[] = { GGML_OP_NORM, GGML_OP_MUL }; +static const ggml_op ops_norm_mul_add[] = { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD }; +static const ggml_op ops_rms_norm_mul[] = { GGML_OP_RMS_NORM, GGML_OP_MUL }; +static const ggml_op ops_rms_norm_mul_add[] = { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD }; + +static const ggml_op ops_add_2[] = { GGML_OP_ADD, GGML_OP_ADD }; +static const ggml_op ops_add_3[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const ggml_op ops_add_4[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const ggml_op ops_add_5[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const ggml_op ops_add_6[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const ggml_op ops_add_7[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const ggml_op ops_snake[] = { GGML_OP_MUL, GGML_OP_SIN, GGML_OP_SQR, GGML_OP_MUL, GGML_OP_ADD }; + +static const ggml_op ops_gdn_cache[] = { GGML_OP_GATED_DELTA_NET, GGML_OP_CPY }; + +static const ggml_metal_fusion ggml_metal_fusions[] = { + { GGML_METAL_FUSION_NORM_MUL, ops_norm_mul, 2, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_MUL_ADD, ops_norm_mul_add, 3, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_MUL, ops_rms_norm_mul, 2, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_MUL_ADD, ops_rms_norm_mul_add, 3, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_2, 2, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_3, 3, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_4, 4, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_5, 5, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_6, 6, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_7, 7, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_SNAKE, ops_snake, 5, false, ggml_metal_fusion_check_snake }, + { GGML_METAL_FUSION_GDN_CACHE, ops_gdn_cache, 2, true, ggml_metal_fusion_check_gdn_cache }, +}; + +const ggml_metal_fusion * ggml_metal_fusion_all(int * n) { + *n = (int) sizeof(ggml_metal_fusions) / sizeof(ggml_metal_fusions[0]); + + return ggml_metal_fusions; +} + +// ---- shared fusion info --------------------------------------------------- + +static std::string ggml_metal_fusion_label(const ggml_metal_fusion * fusion) { + GGML_ASSERT(fusion != nullptr); + + std::string label; + for (int j = 0; j < fusion->n_ops; j++) { + if (j > 0) { + label += '+'; + } + label += ggml_op_name(fusion->ops[j]); + } + return label; +} + +struct ggml_metal_fusion_info { + std::vector labels; + std::vector counts; + bool enabled; + bool stats; + bool labels_set; + int debug; +}; + +struct ggml_metal_fusion_info * ggml_metal_fusion_info_init(bool enabled, int debug) { + ggml_metal_fusion_info * finfo = new ggml_metal_fusion_info; + finfo->enabled = enabled; + finfo->stats = debug > 0; + finfo->labels_set = false; + finfo->debug = debug; + + if (finfo->stats) { + ggml_metal_fusion_info_labels_init(finfo); + } + + return finfo; +} + +void ggml_metal_fusion_info_free(struct ggml_metal_fusion_info * finfo) { + delete finfo; +} + +bool ggml_metal_fusion_info_enabled(const struct ggml_metal_fusion_info * finfo) { + return finfo->enabled; +} + +bool ggml_metal_fusion_info_stats(const struct ggml_metal_fusion_info * finfo) { + return finfo->stats; +} + +int ggml_metal_fusion_info_debug(const struct ggml_metal_fusion_info * finfo) { + return finfo->debug; +} + +int ggml_metal_fusion_info_n_fusions(const struct ggml_metal_fusion_info * finfo) { + return (int) finfo->labels.size(); +} + +const char * ggml_metal_fusion_info_label(const struct ggml_metal_fusion_info * finfo, int idx) { + GGML_ASSERT(idx >= 0 && idx < (int) finfo->labels.size()); + return finfo->labels[idx].c_str(); +} + +uint64_t ggml_metal_fusion_info_count(const struct ggml_metal_fusion_info * finfo, int idx) { + GGML_ASSERT(idx >= 0 && idx < (int) finfo->counts.size()); + return finfo->counts[idx]; +} + +void ggml_metal_fusion_info_count_fusion(struct ggml_metal_fusion_info * finfo, const struct ggml_metal_fusion * fusion) { + if (!finfo->stats || fusion == nullptr) { + return; + } + + int n = 0; + const ggml_metal_fusion * all = ggml_metal_fusion_all(&n); + + int idx = -1; + for (int i = 0; i < n; i++) { + if (&all[i] == fusion) { + idx = i; + break; + } + } + + if (idx >= 0 && idx < (int) finfo->counts.size()) { + finfo->counts[idx]++; + } +} + +void ggml_metal_fusion_info_set_enabled(struct ggml_metal_fusion_info * finfo, bool enabled) { + finfo->enabled = enabled; +} + +void ggml_metal_fusion_info_labels_init(struct ggml_metal_fusion_info * finfo) { + if (finfo->labels_set) { + return; + } + + int n = 0; + const ggml_metal_fusion * all = ggml_metal_fusion_all(&n); + + finfo->labels.clear(); + finfo->counts.assign(n, 0); + finfo->labels.reserve(n); + + for (int i = 0; i < n; i++) { + finfo->labels.emplace_back(ggml_metal_fusion_label(&all[i])); + } + + finfo->labels_set = true; +} + +void ggml_metal_fusion_info_stats_init(struct ggml_metal_fusion_info * finfo) { + finfo->stats = true; + ggml_metal_fusion_info_labels_init(finfo); +} + +void ggml_metal_fusion_info_stats_reset(struct ggml_metal_fusion_info * finfo) { + std::fill(finfo->counts.begin(), finfo->counts.end(), 0); +} + +int ggml_metal_fusion_info_stats_get(const struct ggml_metal_fusion_info * finfo, const char ** labels, uint64_t * counts, int n) { + const int n_fusions = (int) finfo->labels.size(); + + if (labels == nullptr) { + return n_fusions; + } + + const int n_fill = std::min(n, n_fusions); + for (int i = 0; i < n_fill; i++) { + labels[i] = finfo->labels[i].c_str(); + if (counts != nullptr) { + counts[i] = finfo->counts[i]; + } + } + + return n_fill; +} + +// ---- queries ------------------------------------------------------------- + +// find the longest pattern matching the node sequence starting at idx +// (idx is a position in node_idxs, which maps to graph node indices) +const ggml_metal_fusion * ggml_metal_fusion_next( + const ggml_cgraph * gf, + const int * node_idxs, + int n_idxs, + int idx, + ggml_metal_fusion_mode mode, + int * n_out) { + int n = 0; + const ggml_metal_fusion * all = ggml_metal_fusion_all(&n); + + const ggml_metal_fusion * res = nullptr; + int best = 1; + + for (int i = 0; i < n; i++) { + const ggml_metal_fusion * fusion = &all[i]; + + // only look for a longer match than the current best + if (fusion->n_ops <= best) { + continue; + } + if (idx + fusion->n_ops > n_idxs) { + continue; + } + + const ggml_tensor * nodes[GGML_METAL_FUSION_MAX]; + + // the op sequence must match exactly + bool ok = true; + for (int j = 0; j < fusion->n_ops; j++) { + nodes[j] = gf->nodes[node_idxs[idx + j]]; + if (nodes[j]->op != fusion->ops[j]) { + ok = false; + break; + } + } + if (!ok) { + continue; + } + + if (!fusion->unsafe) { + // common element-wise chain constraints: each node reads the previous one, + // and all nodes have the same shape + for (int j = 1; j < fusion->n_ops && ok; j++) { + if (nodes[j]->src[0] != nodes[j - 1] && nodes[j]->src[1] != nodes[j - 1]) { + ok = false; + break; + } + if (!ggml_are_same_shape(nodes[j], nodes[j - 1])) { + ok = false; + break; + } + } + if (!ok) { + continue; + } + + // all current fusions are single-output elision chains, so the last node is the only output + // TODO: multi-output fusions: store pattern-relative offsets in the table and translate them here + int outputs_buf[1]; + outputs_buf[0] = node_idxs[idx + fusion->n_ops - 1]; + + // structural subgraph checks (op sequence, elidable uses, view containment) + if (!ggml_can_fuse_subgraph_ext(gf, node_idxs + idx, fusion->n_ops, fusion->ops, outputs_buf, 1)) { + continue; + } + } + + // pattern-specific checks (the sole validator for unsafe patterns) + if (fusion->check && !fusion->check(fusion, nodes, mode)) { + continue; + } + + best = fusion->n_ops; + res = fusion; + } + + *n_out = best; + + return res; +} + +// optimize phase: maximum number of nodes starting at idx (a raw sequential graph index) that +// could be fused, chaining patterns back-to-back. matching runs on the same filtered (view +// transparent) node sequence that the compute phase uses, so the returned count is the raw index +// span from idx to the last matched node (intermediate views are packed along). +int ggml_metal_fusion_max(const ggml_cgraph * gf, int idx) { + // an empty/view node cannot start a pattern - pack it alone + if (ggml_op_is_empty(gf->nodes[idx]->op) || ggml_is_empty(gf->nodes[idx])) { + return 1; + } + + // collect the non-empty node indices starting at idx + int idxs[GGML_METAL_FUSION_MAX]; + int n_idxs = 0; + for (int i = idx; i < gf->n_nodes && n_idxs < GGML_METAL_FUSION_MAX; i++) { + if (!ggml_op_is_empty(gf->nodes[i]->op) && !ggml_is_empty(gf->nodes[i])) { + idxs[n_idxs++] = i; + } + } + if (n_idxs == 0) { + return 1; + } + + int total = 0; + int i_f = 0; + + while (i_f < n_idxs && total < GGML_METAL_FUSION_MAX) { + int len = 1; + const ggml_metal_fusion * fusion = ggml_metal_fusion_next(gf, idxs, n_idxs, i_f, GGML_METAL_FUSION_STRUCTURAL, &len); + if (!fusion || total + len > GGML_METAL_FUSION_MAX) { + break; + } + + total += len; + i_f += len; + } + + if (i_f == 0) { + return 1; + } + + // map the matched non-empty nodes back to the raw index span (views are included) + return std::min(GGML_METAL_FUSION_MAX, idxs[i_f - 1] - idx + 1); +} diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.h b/ggml/src/ggml-metal/ggml-metal-fusion.h new file mode 100644 index 000000000..e8515bdec --- /dev/null +++ b/ggml/src/ggml-metal/ggml-metal-fusion.h @@ -0,0 +1,104 @@ +// single source of truth for the fusions supported by the Metal backend +// +// every fusable subgraph is declared exactly once as a ggml_metal_fusion entry in +// the table in ggml-metal-fusion.cpp. both the graph optimizer (ggml_metal_fusion_max) +// and the op encoders (ggml_metal_fusion_next) consult this same table, so the two +// phases can never disagree about what can be fused. + +#pragma once + +#include "ggml-impl.h" + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// the maximum number of nodes that can be fused in a single kernel +// (also the maximum length of a packed fusion group during graph optimization) +#define GGML_METAL_FUSION_MAX 16 + +typedef enum ggml_metal_fusion_mode { + // structural checks only; used by the graph optimizer, at which point the graph + // tensors are not allocated yet, so buffer placement cannot be verified + GGML_METAL_FUSION_STRUCTURAL = 0, + // full checks, including buffer placement; used by the op encoders + GGML_METAL_FUSION_FULL, +} ggml_metal_fusion_mode; + +// identifier of each fusion pattern so the op encoders know which kernel to use +typedef enum ggml_metal_fusion_id { + GGML_METAL_FUSION_NONE = 0, + GGML_METAL_FUSION_NORM_MUL, // NORM/RMS_NORM + MUL + GGML_METAL_FUSION_NORM_MUL_ADD, // NORM/RMS_NORM + MUL + ADD + GGML_METAL_FUSION_ADD_CHAIN, // ADD x N (N in [2, 7]) + GGML_METAL_FUSION_SNAKE, // MUL + SIN + SQR + MUL + ADD + GGML_METAL_FUSION_GDN_CACHE, // GATED_DELTA_NET + CPY (write snapshots into the recurrent cache) +} ggml_metal_fusion_id; + +struct ggml_metal_fusion { + ggml_metal_fusion_id id; + + const enum ggml_op * ops; // op sequence (fixed length) + int n_ops; // number of ops + + // if unsafe: the generic chain/shape + ggml_can_fuse_subgraph checks are skipped and the + // check callback below is the sole validator (used for patterns that are not elision chains, + // e.g. the gdn + cache-cpy write-through fusion) + bool unsafe; + + // extra backend constraints on top of ggml_can_fuse_subgraph + // nodes[j] is the j-th node of the pattern + bool (*check)(const struct ggml_metal_fusion * fusion, + const struct ggml_tensor * const * nodes, + ggml_metal_fusion_mode mode); +}; + +typedef struct ggml_metal_fusion ggml_metal_fusion; + +// the single table of all fusions supported by the Metal backend +const ggml_metal_fusion * ggml_metal_fusion_all(int * n); + +// ---- shared fusion info --------------------------------------------------- + +// shared fusion debugging context, owned by the device; newly created backend contexts for that +// device register with it so the fusion counters are race-free and accumulate across contexts. +struct ggml_metal_fusion_info; // defined in ggml-metal-fusion.cpp + +struct ggml_metal_fusion_info * ggml_metal_fusion_info_init(bool enabled, int debug); +void ggml_metal_fusion_info_free(struct ggml_metal_fusion_info * finfo); + +bool ggml_metal_fusion_info_enabled(const struct ggml_metal_fusion_info * finfo); +bool ggml_metal_fusion_info_stats (const struct ggml_metal_fusion_info * finfo); +int ggml_metal_fusion_info_debug (const struct ggml_metal_fusion_info * finfo); + +int ggml_metal_fusion_info_n_fusions(const struct ggml_metal_fusion_info * finfo); +const char * ggml_metal_fusion_info_label (const struct ggml_metal_fusion_info * finfo, int idx); +uint64_t ggml_metal_fusion_info_count (const struct ggml_metal_fusion_info * finfo, int idx); + +void ggml_metal_fusion_info_count_fusion(struct ggml_metal_fusion_info * finfo, const struct ggml_metal_fusion * fusion); +void ggml_metal_fusion_info_set_enabled (struct ggml_metal_fusion_info * finfo, bool enabled); + +void ggml_metal_fusion_info_stats_init ( struct ggml_metal_fusion_info * finfo); +void ggml_metal_fusion_info_stats_reset( struct ggml_metal_fusion_info * finfo); +int ggml_metal_fusion_info_stats_get (const struct ggml_metal_fusion_info * finfo, const char ** labels, uint64_t * counts, int n); +void ggml_metal_fusion_info_labels_init( struct ggml_metal_fusion_info * finfo); + +// compute phase: longest fusion starting at idx (a position in node_idxs) that matches in `mode`. +// returns the matching pattern (nullptr if no fusion) and sets *n_out to the number of nodes consumed. +const ggml_metal_fusion * ggml_metal_fusion_next( + const struct ggml_cgraph * gf, + const int * node_idxs, + int n_idxs, + int idx, + ggml_metal_fusion_mode mode, + int * n_out); + +// optimize phase: maximum number of nodes starting at idx (a raw sequential graph index) that +// could be fused, chaining patterns back-to-back. returns at least 1. +int ggml_metal_fusion_max(const struct ggml_cgraph * gf, int idx); + +#ifdef __cplusplus +} +#endif diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index 28a9ba101..7ad21341e 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -985,6 +985,7 @@ typedef struct { uint64_t nb1; uint64_t nb2; uint64_t nb3; + uint64_t nb_out; // 0 => snapshots are appended after the attn scores (unfused) } ggml_metal_kargs_gated_delta_net; typedef struct { diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index 3db8bca43..b4e87cb2c 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -7,6 +7,7 @@ #include "ggml-metal-impl.h" #include "ggml-metal-common.h" #include "ggml-metal-device.h" +#include "ggml-metal-fusion.h" #include "ggml-metal-tuning.h" #include @@ -31,24 +32,22 @@ struct ggml_metal_op { ggml_metal_device_t dev, ggml_metal_cmd_buf_t cmd_buf, ggml_cgraph * gf, + ggml_metal_fusion_info * finfo, int idx_start, int idx_end, - bool use_fusion, bool use_concurrency, bool use_capture, - int debug_graph, - int debug_fusion) { + int debug_graph) { this->dev = dev; this->lib = ggml_metal_device_get_library(dev); this->enc = ggml_metal_encoder_init(cmd_buf, use_concurrency); this->mem_ranges = ggml_mem_ranges_init(debug_graph); + this->finfo = finfo; this->idx_start = idx_start; this->idx_end = idx_end; - this->use_fusion = use_fusion; this->use_concurrency = use_concurrency; this->use_capture = use_capture; this->debug_graph = debug_graph; - this->debug_fusion = debug_fusion; this->gf = gf; idxs.reserve(gf->n_nodes); @@ -78,15 +77,24 @@ struct ggml_metal_op { return ggml_graph_node(gf, idxs[i]); } - bool can_fuse(int i0, const ggml_op * ops, int n_ops) const { - assert(use_fusion); + // consult the fusion table for the longest pattern starting at i0 + // returns the matching pattern (nullptr if no fusion) and sets *n_out to the number of nodes + const ggml_metal_fusion * can_fuse(int i0, enum ggml_metal_fusion_mode mode, int * n_out) const { + assert(use_fusion()); assert(i0 >= 0 && i0 < n_nodes()); - if (i0 + n_ops > n_nodes()) { - return false; - } + return ggml_metal_fusion_next(gf, idxs.data(), (int) idxs.size(), i0, mode, n_out); + } - return ggml_can_fuse_ext(gf, idxs.data() + i0, ops, n_ops); + // whether to attempt fusion; the toggle lives in the shared fusion debugging context owned + // by the device (initialized from GGML_METAL_FUSION_DISABLE, overridable by the test) + bool use_fusion() const { + return ggml_metal_fusion_info_enabled(finfo); + } + + // record that a fusion fired, indexed by the matching table entry + void count_fusions(const ggml_metal_fusion * fusion) const { + ggml_metal_fusion_info_count_fusion(finfo, fusion); } ggml_metal_device_t dev; @@ -94,12 +102,13 @@ struct ggml_metal_op { ggml_metal_encoder_t enc; ggml_mem_ranges_t mem_ranges; - bool use_fusion; + // shared fusion debugging context + ggml_metal_fusion_info * finfo; + bool use_concurrency; bool use_capture; int debug_graph; - int debug_fusion; private: ggml_cgraph * gf; @@ -115,24 +124,22 @@ ggml_metal_op_t ggml_metal_op_init( ggml_metal_device_t dev, ggml_metal_cmd_buf_t cmd_buf, ggml_cgraph * gf, + ggml_metal_fusion_info * finfo, int idx_start, int idx_end, - bool use_fusion, bool use_concurrency, bool use_capture, - int debug_graph, - int debug_fusion) { + int debug_graph) { ggml_metal_op_t res = new ggml_metal_op( dev, cmd_buf, gf, + finfo, idx_start, idx_end, - use_fusion, use_concurrency, use_capture, - debug_graph, - debug_fusion); + debug_graph); return res; } @@ -1868,6 +1875,8 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; + const bool use_fusion = ctx->use_fusion(); + const int debug_fusion = ggml_metal_fusion_info_debug(ctx->finfo); GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -1880,6 +1889,31 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) { auto pipeline = ggml_metal_library_get_pipeline_gated_delta_net(lib, op); + // when fused with the trailing cache cpy, the snapshots are written straight into the + // recurrent cache and the cpy is skipped (see GGML_METAL_FUSION_GDN_CACHE) + ggml_metal_buffer_id bid_out = ggml_metal_get_buffer_id(op); + uint64_t nb_out = 0; + int n_fuse = 1; + + if (use_fusion) { + int n = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); + + if (fusion && fusion->id == GGML_METAL_FUSION_GDN_CACHE) { + const ggml_tensor * dst_cache = ctx->node(idx + 1)->src[1]; // cache view + + bid_out = ggml_metal_get_buffer_id(dst_cache); + nb_out = dst_cache->nb[2]/sizeof(float); + n_fuse = 2; + + ctx->count_fusions(fusion); + + if (debug_fusion > 1) { + GGML_LOG_DEBUG("%s: fuse: GATED_DELTA_NET + CPY\n", __func__); + } + } + } + int ida = 0; ggml_metal_kargs_gated_delta_net args = { @@ -1918,23 +1952,25 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) { /*.nb1 =*/ nb1, /*.nb2 =*/ nb2, /*.nb3 =*/ nb3, + /*.nb_out =*/ nb_out, }; ggml_metal_encoder_set_pipeline(enc, pipeline); - ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), ida++); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), ida++); // args ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), ida++); // q ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), ida++); // k ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), ida++); // v ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[3]), ida++); // gate ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[4]), ida++); // beta ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[5]), ida++); // state - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++); // dst + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++); // dst (attn) + ggml_metal_encoder_set_buffer (enc, bid_out, ida++); // state_out const int nsg = pipeline.nsg; ggml_metal_encoder_dispatch_threadgroups(enc, op->src[2]->ne[0]/nsg, op->src[2]->ne[1], op->src[2]->ne[3], 32, nsg, 1); - return 1; + return n_fuse; } int ggml_metal_op_solve_tri(ggml_metal_op_t ctx, int idx) { @@ -3718,56 +3754,20 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { return 1; } -// Snake activation autofuse: mul -> sin -> sqr -> mul -> add -static bool ggml_metal_op_can_fuse_snake(ggml_metal_op_t ctx, int idx) { - static constexpr ggml_op snake_ops[5] = { GGML_OP_MUL, GGML_OP_SIN, GGML_OP_SQR, GGML_OP_MUL, GGML_OP_ADD }; - - if (ctx->node(idx)->op != GGML_OP_MUL || !ctx->can_fuse(idx, snake_ops, 5)) { - return false; - } - - const ggml_tensor * mul0 = ctx->node(idx + 0); - const ggml_tensor * sin_node = ctx->node(idx + 1); - const ggml_tensor * sqr = ctx->node(idx + 2); - const ggml_tensor * mul1 = ctx->node(idx + 3); - const ggml_tensor * add = ctx->node(idx + 4); - - // x carries the full activation shape, a is the broadcast operand - const ggml_tensor * x = ggml_are_same_shape(mul0, mul0->src[0]) ? mul0->src[0] : mul0->src[1]; - const ggml_tensor * a = (x == mul0->src[0]) ? mul0->src[1] : mul0->src[0]; - - // mul1 reads sqr and inv_b in either operand order - const ggml_tensor * inv_b = (mul1->src[0] == sqr) ? mul1->src[1] : mul1->src[0]; - - // closure check: the trailing add reads the same x as the leading mul - const ggml_tensor * x_in_add = (add->src[0] == mul1) ? add->src[1] : add->src[0]; - - // x is in the supported whitelist and every chain intermediate shares x's type. - // a and inv_b bind as device const float * in the kernel, so they stay F32. - const bool types_ok = - (x->type == GGML_TYPE_F32 || x->type == GGML_TYPE_F16 || x->type == GGML_TYPE_BF16) && - (a->type == GGML_TYPE_F32) && (inv_b->type == GGML_TYPE_F32) && - (mul0->type == x->type) && (sin_node->type == x->type) && - (sqr->type == x->type) && (mul1->type == x->type) && - (add->type == x->type); - // a / inv_b collapse to [1, C, 1, 1], x and add stay 2D - const bool shape_ok = ggml_are_same_shape(a, inv_b) && a->ne[0] == 1 && a->ne[1] == x->ne[1]; - const bool dim_ok = - (x->ne[2] == 1) && (x->ne[3] == 1) && - (add->ne[2] == 1) && (add->ne[3] == 1) && - (a->ne[2] == 1) && (a->ne[3] == 1) && - (inv_b->ne[2] == 1) && (inv_b->ne[3] == 1); - // kernel reads x[idx] and a[c] / inv_b[c] linearly, so every operand is contiguous - const bool contig_ok = - ggml_is_contiguous(x) && ggml_is_contiguous(add) && - ggml_is_contiguous(a) && ggml_is_contiguous(inv_b); - - return types_ok && shape_ok && dim_ok && contig_ok && x_in_add == x; -} - int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { - if (ctx->use_fusion && ggml_metal_op_can_fuse_snake(ctx, idx)) { - return ggml_metal_op_snake_fused(ctx, idx); + int n_fuse = 1; + const ggml_metal_fusion * fusion = nullptr; + + if (ctx->use_fusion()) { + int n = 1; + fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); + n_fuse = n; + + // snake activation autofuse: mul -> sin -> sqr -> mul -> add + if (fusion && fusion->id == GGML_METAL_FUSION_SNAKE) { + ctx->count_fusions(fusion); + return ggml_metal_op_snake_fused(ctx, idx); + } } ggml_tensor * op = ctx->node(idx); @@ -3775,9 +3775,9 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; - const bool use_fusion = ctx->use_fusion; + const bool use_fusion = ctx->use_fusion(); - const int debug_fusion = ctx->debug_fusion; + const int debug_fusion = ggml_metal_fusion_info_debug(ctx->finfo); GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -3822,57 +3822,19 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { /*.o1 =*/ { bid_src1.offs }, }; - ggml_op fops[8]; - - int n_fuse = 1; - // c[0] = add(a, b[0]) // c[1] = add(c[0], b[1]) // c[2] = add(c[1], b[2]) // ... - if (use_fusion) { - fops[0] = GGML_OP_ADD; - fops[1] = GGML_OP_ADD; - fops[2] = GGML_OP_ADD; - fops[3] = GGML_OP_ADD; - fops[4] = GGML_OP_ADD; - fops[5] = GGML_OP_ADD; - fops[6] = GGML_OP_ADD; - fops[7] = GGML_OP_ADD; - - // note: in metal, we sometimes encode the graph in parallel so we have to avoid fusing ops - // across splits. idx_end indicates the last node in the current split - for (n_fuse = 0; n_fuse <= 6; ++n_fuse) { - if (!ctx->can_fuse(idx + n_fuse, fops + n_fuse, 2)) { - break; - } - - ggml_tensor * f0 = ctx->node(idx + n_fuse); - ggml_tensor * f1 = ctx->node(idx + n_fuse + 1); - - if (f0 != f1->src[0]) { - break; - } - - // b[0] === b[1] === ... - if (!ggml_are_same_layout(f0->src[1], f1->src[1])) { - break; - } - - // only fuse ops if src1 is in the same Metal buffer - ggml_metal_buffer_id bid_fuse = ggml_metal_get_buffer_id(f1->src[1]); - if (bid_fuse.metal != bid_src1.metal) { - break; - } - - //ctx->fuse_cnt[ops[n_fuse + 1]->op]++; - - args.o1[n_fuse + 1] = bid_fuse.offs; + if (use_fusion && fusion && fusion->id == GGML_METAL_FUSION_ADD_CHAIN) { + // the offsets of the fused addends are relative to the start of the src1 buffer + for (int i = 1; i < n_fuse; i++) { + args.o1[i] = ggml_metal_get_buffer_id(ctx->node(idx + i)->src[1]).offs; } - ++n_fuse; + ctx->count_fusions(fusion); - if (debug_fusion > 1 && n_fuse > 1) { + if (debug_fusion > 1) { GGML_LOG_DEBUG("%s: fuse: ADD x %d\n", __func__, n_fuse); } } @@ -4080,9 +4042,9 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; - const bool use_fusion = ctx->use_fusion; + const bool use_fusion = ctx->use_fusion(); - const int debug_fusion = ctx->debug_fusion; + const int debug_fusion = ggml_metal_fusion_info_debug(ctx->finfo); GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -4110,8 +4072,6 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) { /*.nbf3 =*/ { nb03 }, }; - ggml_op fops[8]; - int n_fuse = 1; ggml_metal_buffer_id bid_fuse[2] = { bid_src0, bid_src0 }; @@ -4120,55 +4080,35 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) { // d[1] = mul(d[0], b) // d[2] = add(d[1], c) if (use_fusion) { - fops[0] = op->op; - fops[1] = GGML_OP_MUL; - fops[2] = GGML_OP_ADD; + int n = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); - for (n_fuse = 0; n_fuse <= 1; ++n_fuse) { - if (!ctx->can_fuse(idx + n_fuse, fops + n_fuse, 2)) { - break; + if (fusion && (fusion->id == GGML_METAL_FUSION_NORM_MUL || fusion->id == GGML_METAL_FUSION_NORM_MUL_ADD)) { + n_fuse = n; + + ctx->count_fusions(fusion); + + for (int i = 1; i < n_fuse; i++) { + const ggml_tensor * fn = ctx->node(idx + i); + + bid_fuse[i - 1] = ggml_metal_get_buffer_id(fn->src[1]); + + args.nef1[i] = fn->src[1]->ne[1]; + args.nef2[i] = fn->src[1]->ne[2]; + args.nef3[i] = fn->src[1]->ne[3]; + + args.nbf1[i] = fn->src[1]->nb[1]; + args.nbf2[i] = fn->src[1]->nb[2]; + args.nbf3[i] = fn->src[1]->nb[3]; } - ggml_tensor * f0 = ctx->node(idx + n_fuse); - ggml_tensor * f1 = ctx->node(idx + n_fuse + 1); - - if (f0 != f1->src[0]) { - break; - } - - if (f1->src[1]->ne[0] != op->ne[0]) { - break; - } - - if (!ggml_is_contiguous_rows(f1->src[1])) { - break; - } - - if (f1->type != GGML_TYPE_F32) { - break; - } - - //ctx->fuse_cnt[f1->op]++; - - bid_fuse[n_fuse] = ggml_metal_get_buffer_id(f1->src[1]); - - args.nef1[n_fuse + 1] = f1->src[1]->ne[1]; - args.nef2[n_fuse + 1] = f1->src[1]->ne[2]; - args.nef3[n_fuse + 1] = f1->src[1]->ne[3]; - - args.nbf1[n_fuse + 1] = f1->src[1]->nb[1]; - args.nbf2[n_fuse + 1] = f1->src[1]->nb[2]; - args.nbf3[n_fuse + 1] = f1->src[1]->nb[3]; - } - - ++n_fuse; - - if (debug_fusion > 1 && n_fuse > 1) { - if (n_fuse == 2) { - GGML_LOG_DEBUG("%s: fuse: %s + MUL\n", __func__, ggml_op_name(op->op)); - } - if (n_fuse == 3) { - GGML_LOG_DEBUG("%s: fuse: %s + MUL + ADD\n", __func__, ggml_op_name(op->op)); + if (debug_fusion > 1) { + if (n_fuse == 2) { + GGML_LOG_DEBUG("%s: fuse: %s + MUL\n", __func__, ggml_op_name(op->op)); + } + if (n_fuse == 3) { + GGML_LOG_DEBUG("%s: fuse: %s + MUL + ADD\n", __func__, ggml_op_name(op->op)); + } } } } diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.h index f8fe50b46..4dd8ce7af 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.h +++ b/ggml/src/ggml-metal/ggml-metal-ops.h @@ -8,17 +8,18 @@ extern "C" { typedef struct ggml_metal_op * ggml_metal_op_t; +struct ggml_metal_fusion; // forward decl (ggml-metal-device.h) + ggml_metal_op_t ggml_metal_op_init( ggml_metal_device_t dev, ggml_metal_cmd_buf_t cmd_buf, struct ggml_cgraph * gf, + struct ggml_metal_fusion_info * finfo, int idx_start, int idx_end, - bool use_fusion, bool use_concurrency, bool use_capture, - int debug_graph, - int debug_fusion); + int debug_graph); void ggml_metal_op_free(ggml_metal_op_t ctx); diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp index 3bd6abd06..4cbec8645 100644 --- a/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp @@ -4,6 +4,7 @@ #include "ggml-backend-impl.h" #include "ggml-metal-device.h" +#include "ggml-metal-fusion.h" #include "ggml-metal-context.h" #include "ggml-metal-ops.h" #include "ggml-metal-tuning.h" @@ -906,6 +907,30 @@ static const char * ggml_backend_metal_tuning_device_token(ggml_backend_dev_t de return ggml_metal_device_id_token(ggml_metal_device_get_props(ctx_dev)->device_id); } +// generic fusion debugging API (ad-hoc proc-address mechanism): the test resolves the device +// fusion context once and passes that opaque handle to the rest of the functions +typedef void * ggml_backend_fusion_t; + +static ggml_backend_fusion_t ggml_backend_metal_fusion_get(ggml_backend_dev_t dev) { + return ggml_metal_device_get_fusion_info((ggml_metal_device_t)dev->context); +} + +static void ggml_backend_metal_fusion_stats_init(ggml_backend_fusion_t finfo) { + ggml_metal_fusion_info_stats_init((struct ggml_metal_fusion_info *) finfo); +} + +static void ggml_backend_metal_fusion_stats_reset(ggml_backend_fusion_t finfo) { + ggml_metal_fusion_info_stats_reset((struct ggml_metal_fusion_info *) finfo); +} + +static int ggml_backend_metal_fusion_stats_get(ggml_backend_fusion_t finfo, const char ** labels, uint64_t * counts, int n) { + return ggml_metal_fusion_info_stats_get((struct ggml_metal_fusion_info *) finfo, labels, counts, n); +} + +static void ggml_backend_metal_fusion_set_enabled(ggml_backend_fusion_t finfo, bool enabled) { + ggml_metal_fusion_info_set_enabled((struct ggml_metal_fusion_info *) finfo, enabled); +} + static void * ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, const char * name) { if (strcmp(name, "ggml_backend_get_features") == 0) { return (void *)ggml_backend_metal_get_features; @@ -928,6 +953,23 @@ static void * ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, const if (strcmp(name, "ggml_backend_metal_tuning_device_token") == 0) { return (void *)ggml_backend_metal_tuning_device_token; } + // generic fusion debugging API (ad-hoc proc-address mechanism, not part of the official + // ggml backend interface yet; a backend that adopts it exports these exact names) + if (strcmp(name, "ggml_backend_fusion_get") == 0) { + return (void *)ggml_backend_metal_fusion_get; + } + if (strcmp(name, "ggml_backend_fusion_stats_init") == 0) { + return (void *)ggml_backend_metal_fusion_stats_init; + } + if (strcmp(name, "ggml_backend_fusion_stats_reset") == 0) { + return (void *)ggml_backend_metal_fusion_stats_reset; + } + if (strcmp(name, "ggml_backend_fusion_stats_get") == 0) { + return (void *)ggml_backend_metal_fusion_stats_get; + } + if (strcmp(name, "ggml_backend_fusion_set_enabled") == 0) { + return (void *)ggml_backend_metal_fusion_set_enabled; + } return NULL; diff --git a/ggml/src/ggml-metal/kernels/gated_delta_net.metal b/ggml/src/ggml-metal/kernels/gated_delta_net.metal index 8422d8e29..5e4861ece 100644 --- a/ggml/src/ggml-metal/kernels/gated_delta_net.metal +++ b/ggml/src/ggml-metal/kernels/gated_delta_net.metal @@ -15,6 +15,7 @@ kernel void kernel_gated_delta_net_impl( device const char * b, device const char * s, device char * dst, + device char * dst_fuse, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tpitg[[thread_position_in_threadgroup]], uint3 ntg[[threads_per_threadgroup]]) { @@ -65,6 +66,12 @@ kernel void kernel_gated_delta_net_impl( // per-(seq,head) offset within a slot const uint state_out_base = (i23*args.ne21 + i21)*S_v*S_v + i20*S_v; + // when fused with the cache cpy, write the snapshots straight into the cache buffer using + // the slot stride; otherwise append them after the attn scores (nb_out == 0) + const bool fused = args.nb_out > 0; + const device float * state_out = fused ? (device float *)dst_fuse : (device float *)dst + attn_size; + const uint slot_stride = fused ? (uint)args.nb_out : state_size_per_snap; + for (short t = 0; t < args.ne22; t++) { float s_k = 0.0f; @@ -116,7 +123,7 @@ kernel void kernel_gated_delta_net_impl( if (K > 1) { const int target_slot = (int)args.ne22 - 1 - (int)t; if (target_slot >= 0 && target_slot < (int)K) { - device float * dst_state = (device float *) (dst) + attn_size + (uint)target_slot * state_size_per_snap + state_out_base; + device float * dst_state = (device float *)state_out + (uint)target_slot * slot_stride + state_out_base; FOR_UNROLL (short j = 0; j < NSG; j++) { const short is = tx*NSG + j; dst_state[is] = ls[j]; @@ -126,7 +133,7 @@ kernel void kernel_gated_delta_net_impl( } if (K == 1) { - device float * dst_state = (device float *) (dst) + attn_size + state_out_base; + device float * dst_state = (device float *)state_out + state_out_base; FOR_UNROLL (short j = 0; j < NSG; j++) { const short is = tx*NSG + j; dst_state[is] = ls[j]; @@ -158,6 +165,7 @@ kernel void kernel_gated_delta_net_impl( device const char * b, device const char * s, device char * dst, + device char * dst_fuse, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tpitg[[thread_position_in_threadgroup]], uint3 ntg[[threads_per_threadgroup]]) { @@ -230,7 +238,13 @@ kernel void kernel_gated_delta_net_impl( dst_attn += args.ne21*S_v; } - device float * dst_state = (device float *) (dst) + args.ne23*args.ne22*args.ne21*S_v + (i23*args.ne21 + i21)*S_v*S_v + i20; + // when fused with the cache cpy, write the snapshots straight into the cache buffer using + // the slot stride; otherwise append them after the attn scores (nb_out == 0) + const bool fused = args.nb_out > 0; + const device float * state_out = fused ? (device float *)dst_fuse : (device float *)dst + args.ne23*args.ne22*args.ne21*S_v; + const uint slot_stride = fused ? (uint)args.nb_out : S_v*S_v; + + device float * dst_state = (device float *)state_out + (i23*args.ne21 + i21)*slot_stride + i20; device T * dstt_state = (device T *) (dst_state); FOR_UNROLL (short j = 0; j < NSG; j++) { diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 21501574a..6334f3cca 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -666,7 +666,9 @@ void llama_context::sched_reserve() { // need to implement a more robust mechanism that tries a few different inputs and analyzes the results ggml_cgraph * gf = nullptr; switch (model.arch) { + case LLM_ARCH_KIMI_LINEAR: case LLM_ARCH_MINIMAX_01: + // [TAG_RESERVE_DIAG_DECAY] // the `inp_diag_decay` tensor size scales with `n_seq_tokens^2` which // makes `n_seqs == 1` use more memory for the compute graph compared to `n_seqs > 1` gf = graph_reserve(n_tokens, 1, n_outputs_pp, mctx.get(), model.hparams.no_alloc); diff --git a/src/models/minimax-01.cpp b/src/models/minimax-01.cpp index 361114acc..9fa2e8fc0 100644 --- a/src/models/minimax-01.cpp +++ b/src/models/minimax-01.cpp @@ -229,6 +229,7 @@ llama_model_minimax_01::graph::graph(const llama_model & model, const llm_graph_ ggml_set_input(inp->inp_k_decay); cb(inp->inp_k_decay, "k_decay_exp", -1); + // [TAG_RESERVE_DIAG_DECAY] inp->inp_diag_decay = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_seq_tokens, n_seq_tokens, n_head, n_seqs); ggml_set_input(inp->inp_diag_decay); cb(inp->inp_diag_decay, "diag_decay_exp", -1); diff --git a/src/models/plamo2.cpp b/src/models/plamo2.cpp index d946b3cff..ba1cea146 100644 --- a/src/models/plamo2.cpp +++ b/src/models/plamo2.cpp @@ -142,6 +142,11 @@ llama_model_plamo2::graph::graph(const llama_model & model, const llm_graph_para cur = build_plamo2_attn_layer(inp_hybrid->get_attn(), inp_pos, cur, model, il); } + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + residual = ggml_get_rows(ctx0, residual, inp_out_ids); + } + // post_mixer_norm cur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il); cb(cur, "attn_post_norm", il); @@ -167,11 +172,6 @@ llama_model_plamo2::graph::graph(const llama_model & model, const llm_graph_para cur = build_norm(cur, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, il); cb(cur, "ffn_post_norm", il); - if (il == n_layer - 1 && inp_out_ids) { - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - residual = ggml_get_rows(ctx0, residual, inp_out_ids); - } - // residual connection cur = ggml_add(ctx0, cur, residual); cb(cur, "ffn_residual", il); diff --git a/src/models/qwen3vl.cpp b/src/models/qwen3vl.cpp index 5596620f0..30c08ed35 100644 --- a/src/models/qwen3vl.cpp +++ b/src/models/qwen3vl.cpp @@ -18,6 +18,7 @@ void llama_model_qwen3vl::load_arch_tensors(llama_model_loader &) { int64_t n_vocab_out = n_vocab; if (arch == LLM_ARCH_QWEN3TTS) { + // [TAG_LLAMA_N_VOCAB_OUT] n_vocab_out = 3072; } diff --git a/tests/.gitignore b/tests/.gitignore index 52b292b1f..04095c9dd 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,6 +1,7 @@ * !*.* !snapshots/ +!fusion/ *.o ggml-common.h **/*.swp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5531c4ce3..920c58c73 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -196,7 +196,7 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) # llama_build_and_test(test-double-float.cpp) # SLOW - llama_build_and_test(test-llama-archs.cpp) + llama_build(test-llama-archs.cpp) set(MODEL_DIR "${CMAKE_CURRENT_BINARY_DIR}/test-models/") file(MAKE_DIRECTORY "${MODEL_DIR}") @@ -255,6 +255,8 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) ARGS --models "${MODEL_DIR}" ) set_tests_properties(test-save-load-state PROPERTIES FIXTURES_REQUIRED generate-models) + + llama_build(test-fusion.cpp) endif() llama_build_and_test(test-chat-peg-parser.cpp peg-parser/simple-tokenize.cpp) diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv new file mode 100644 index 000000000..067316abf --- /dev/null +++ b/tests/fusion/MTL.csv @@ -0,0 +1,154 @@ +# test-fusion baseline for device MTL +# arch ,moe ,mode ,label , count +arcee ,0 ,any ,RMS_NORM+MUL , 5 +arctic ,0 ,any ,RMS_NORM+MUL , 7 +baichuan ,0 ,any ,RMS_NORM+MUL , 5 +bailingmoe ,1 ,any ,ADD+ADD , 2 +bailingmoe ,1 ,any ,RMS_NORM+MUL , 5 +bailingmoe2 ,1 ,any ,ADD+ADD , 1 +bailingmoe2 ,1 ,any ,RMS_NORM+MUL , 9 +bailingmoe3 ,1 ,any ,ADD+ADD , 1 +bailingmoe3 ,1 ,any ,GATED_DELTA_NET+CPY , 1 +bailingmoe3 ,1 ,any ,RMS_NORM+MUL , 8 +bloom ,0 ,any ,NORM+MUL+ADD , 6 +chatglm ,0 ,any ,RMS_NORM+MUL , 5 +codeshell ,0 ,any ,NORM+MUL+ADD , 5 +cogvlm ,0 ,any ,RMS_NORM+MUL , 5 +command-r ,0 ,any ,NORM+MUL , 3 +dbrx ,0 ,any ,NORM+MUL , 5 +deci ,0 ,any ,RMS_NORM+MUL , 5 +deepseek ,0 ,any ,ADD+ADD , 1 +deepseek ,0 ,any ,RMS_NORM+MUL , 5 +deepseek2 ,0 ,any ,ADD+ADD , 1 +deepseek2 ,0 ,any ,RMS_NORM+MUL , 9 +deepseek32 ,0 ,any ,ADD+ADD , 1 +deepseek32 ,0 ,any ,NORM+MUL+ADD , 2 +deepseek32 ,0 ,any ,RMS_NORM+MUL , 9 +deepseek4 ,0 ,any ,RMS_NORM+MUL , 20 +dots1 ,0 ,any ,ADD+ADD , 1 +dots1 ,0 ,any ,RMS_NORM+MUL , 9 +dream ,0 ,any ,RMS_NORM+MUL , 5 +ernie4_5-moe ,1 ,any ,ADD+ADD , 1 +ernie4_5-moe ,1 ,any ,RMS_NORM+MUL , 5 +ernie4_5 ,0 ,any ,RMS_NORM+MUL , 5 +exaone ,0 ,any ,RMS_NORM+MUL , 5 +exaone4 ,0 ,any ,RMS_NORM+MUL , 5 +exaone4 ,0 ,any ,RMS_NORM+MUL+ADD , 4 +falcon ,0 ,any ,ADD+ADD , 2 +falcon ,0 ,any ,NORM+MUL+ADD , 5 +falcon-h1 ,0 ,any ,ADD+ADD , 2 +falcon-h1 ,0 ,any ,RMS_NORM+MUL , 9 +gemma ,0 ,any ,RMS_NORM+MUL , 5 +gemma2 ,0 ,any ,RMS_NORM+MUL , 5 +gemma2 ,0 ,any ,RMS_NORM+MUL+ADD , 4 +glm-dsa ,0 ,any ,ADD+ADD , 1 +glm-dsa ,0 ,any ,NORM+MUL+ADD , 2 +glm-dsa ,0 ,any ,RMS_NORM+MUL , 9 +glm4 ,0 ,any ,RMS_NORM+MUL , 5 +glm4 ,0 ,any ,RMS_NORM+MUL+ADD , 4 +glm4moe ,1 ,any ,ADD+ADD , 1 +glm4moe ,1 ,any ,RMS_NORM+MUL , 9 +gpt-oss ,0 ,any ,RMS_NORM+MUL , 5 +gpt2 ,0 ,any ,NORM+MUL+ADD , 5 +gptneox ,0 ,any ,NORM+MUL+ADD , 5 +granite ,0 ,any ,RMS_NORM+MUL , 5 +granite ,0 ,any ,RMS_NORM+MUL , 5 +granitehybrid ,0 ,any ,RMS_NORM+MUL , 6 +granitemoe ,1 ,any ,RMS_NORM+MUL , 5 +granitemoe ,1 ,any ,RMS_NORM+MUL , 5 +grok ,0 ,any ,RMS_NORM+MUL , 5 +grok ,0 ,any ,RMS_NORM+MUL+ADD , 4 +grovemoe ,1 ,any ,ADD+ADD , 2 +grovemoe ,1 ,any ,RMS_NORM+MUL , 9 +hunyuan-dense ,0 ,any ,RMS_NORM+MUL , 9 +hunyuan-moe ,1 ,any ,ADD+ADD , 2 +hunyuan-moe ,1 ,any ,RMS_NORM+MUL , 9 +hunyuan_vl ,0 ,any ,RMS_NORM+MUL , 9 +hy_v3 ,0 ,any ,ADD+ADD , 2 +hy_v3 ,0 ,any ,RMS_NORM+MUL , 9 +hy_v4 ,0 ,any ,NORM+MUL+ADD , 1 +hy_v4 ,0 ,any ,RMS_NORM+MUL , 9 +internlm2 ,0 ,any ,RMS_NORM+MUL , 5 +jais ,0 ,any ,NORM+MUL+ADD , 5 +jais2 ,0 ,any ,NORM+MUL+ADD , 5 +jamba ,0 ,any ,RMS_NORM+MUL , 8 +kimi-k3 ,0 ,any ,GATED_DELTA_NET+CPY , 1 +kimi-k3 ,0 ,any ,RMS_NORM+MUL , 17 +kimi-linear ,0 ,any ,ADD+ADD , 1 +kimi-linear ,0 ,any ,GATED_DELTA_NET+CPY , 1 +kimi-linear ,0 ,any ,RMS_NORM+MUL , 7 +lfm2 ,0 ,any ,RMS_NORM+MUL , 7 +lfm2moe ,1 ,any ,RMS_NORM+MUL , 7 +llada ,0 ,any ,RMS_NORM+MUL , 5 +llada-moe ,1 ,any ,RMS_NORM+MUL , 9 +llama ,0 ,any ,RMS_NORM+MUL , 5 +llama ,0 ,any ,RMS_NORM+MUL , 5 +llama4 ,0 ,any ,ADD+ADD , 2 +llama4 ,0 ,any ,RMS_NORM+MUL , 9 +maincoder ,0 ,any ,RMS_NORM+MUL , 9 +mamba ,0 ,any ,RMS_NORM+MUL , 3 +mamba2 ,0 ,any ,RMS_NORM+MUL , 5 +minicpm ,0 ,any ,RMS_NORM+MUL , 5 +minicpm ,0 ,any ,RMS_NORM+MUL , 5 +minicpm3 ,0 ,any ,RMS_NORM+MUL , 9 +minimax-01 ,0 ,any ,RMS_NORM+MUL , 6 +minimax-m2 ,0 ,any ,RMS_NORM+MUL , 9 +minimax-m3 ,0 ,any ,ADD+ADD , 1 +minimax-m3 ,0 ,any ,RMS_NORM+MUL , 11 +mistral3 ,0 ,any ,RMS_NORM+MUL , 5 +mistral3 ,0 ,any ,RMS_NORM+MUL , 5 +mistral4 ,0 ,any ,ADD+ADD , 1 +mistral4 ,0 ,any ,RMS_NORM+MUL , 9 +mpt ,0 ,any ,NORM+MUL+ADD , 5 +nanbeige ,0 ,any ,RMS_NORM+MUL , 5 +nemotron ,0 ,any ,NORM+MUL+ADD , 5 +nemotron_h ,0 ,any ,RMS_NORM+MUL , 5 +nemotron_h_moe ,1 ,any ,RMS_NORM+MUL , 5 +olmoe ,1 ,any ,RMS_NORM+MUL , 9 +openelm ,0 ,any ,RMS_NORM+MUL , 9 +orion ,0 ,any ,NORM+MUL+ADD , 5 +paddleocr ,0 ,any ,RMS_NORM+MUL , 5 +pangu-embedded ,0 ,any ,RMS_NORM+MUL , 5 +phi2 ,0 ,any ,ADD+ADD , 2 +phi2 ,0 ,any ,NORM+MUL+ADD , 3 +phi3 ,0 ,any ,RMS_NORM+MUL , 5 +phimoe ,1 ,any ,RMS_NORM+MUL+ADD , 5 +plamo ,0 ,any ,ADD+ADD , 2 +plamo ,0 ,any ,RMS_NORM+MUL , 3 +plamo2 ,0 ,any ,RMS_NORM+MUL , 10 +plamo2 ,0 ,any ,RMS_NORM+MUL+ADD , 4 +pockettts ,0 ,any ,NORM+MUL+ADD , 5 +qwen ,0 ,any ,RMS_NORM+MUL , 5 +qwen2 ,0 ,any ,RMS_NORM+MUL , 5 +qwen2moe ,1 ,any ,ADD+ADD , 2 +qwen2moe ,1 ,any ,RMS_NORM+MUL , 5 +qwen2vl ,0 ,any ,RMS_NORM+MUL , 5 +qwen3 ,0 ,any ,RMS_NORM+MUL , 9 +qwen35 ,0 ,any ,GATED_DELTA_NET+CPY , 1 +qwen35 ,0 ,any ,RMS_NORM+MUL , 8 +qwen35moe ,1 ,any ,ADD+ADD , 2 +qwen35moe ,1 ,any ,GATED_DELTA_NET+CPY , 1 +qwen35moe ,1 ,any ,RMS_NORM+MUL , 8 +qwen3moe ,1 ,any ,RMS_NORM+MUL , 9 +qwen3next ,0 ,any ,ADD+ADD , 2 +qwen3next ,0 ,any ,GATED_DELTA_NET+CPY , 1 +qwen3next ,0 ,any ,RMS_NORM+MUL , 8 +qwen3tts ,0 ,any ,RMS_NORM+MUL , 9 +qwen3vl ,0 ,any ,RMS_NORM+MUL , 9 +qwen3vlmoe ,1 ,any ,RMS_NORM+MUL , 9 +qwen4exp ,0 ,any ,ADD+ADD+ADD , 5 +qwen4exp ,0 ,any ,ADD+ADD+ADD+ADD+ADD+ADD+ADD , 9 +qwen4exp ,0 ,any ,GATED_DELTA_NET+CPY , 1 +qwen4exp ,0 ,any ,RMS_NORM+MUL , 5 +refact ,0 ,any ,RMS_NORM+MUL , 5 +refact ,0 ,any ,RMS_NORM+MUL , 5 +rnd1 ,0 ,any ,RMS_NORM+MUL , 9 +seed_oss ,0 ,any ,RMS_NORM+MUL , 5 +smallthinker ,0 ,any ,RMS_NORM+MUL , 5 +smollm3 ,0 ,any ,RMS_NORM+MUL , 5 +stablelm ,0 ,any ,NORM+MUL , 4 +stablelm ,0 ,any ,NORM+MUL+ADD , 5 +starcoder ,0 ,any ,NORM+MUL+ADD , 5 +starcoder2 ,0 ,any ,NORM+MUL+ADD , 5 +talkie ,0 ,any ,ADD+ADD , 2 +xverse ,0 ,any ,RMS_NORM+MUL , 5 diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index ef4fc30ce..15c42a108 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -4638,6 +4638,122 @@ struct test_gated_delta_net : public test_case { } }; +// GGML_OP_GATED_DELTA_NET + GGML_OP_CPY (recurrent cache fusion) +struct test_gated_delta_net_cache_fusion : public test_case { + const ggml_type type; + + const int64_t head_count; + const int64_t head_size; + const int64_t n_seq_tokens; + const int64_t n_seqs; + const int64_t K; // snapshot slot count (>1) + + ggml_tensor * cpy_node = nullptr; + + std::string vars() override { + return VARS_TO_STR6(type, head_count, head_size, n_seq_tokens, n_seqs, K); + } + + test_gated_delta_net_cache_fusion(ggml_type type = GGML_TYPE_F32, + int64_t head_count = 4, int64_t head_size = 32, int64_t n_seq_tokens = 2, int64_t n_seqs = 1, + int64_t K = 2) + : type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs), K(K) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + const int64_t S_v = head_size; + const int64_t H_v = head_count; + const int64_t H_k = head_count; + const int64_t D = S_v * S_v * H_v; + const int64_t n_written = std::min(n_seq_tokens, K); + + ggml_tensor * q = ggml_new_tensor_4d(ctx, type, head_size, H_k, n_seq_tokens, n_seqs); + ggml_tensor * k = ggml_new_tensor_4d(ctx, type, head_size, H_k, n_seq_tokens, n_seqs); + ggml_tensor * v = ggml_new_tensor_4d(ctx, type, head_size, H_v, n_seq_tokens, n_seqs); + ggml_set_name(q, "q"); + ggml_set_name(k, "k"); + ggml_set_name(v, "v"); + ggml_tensor * g = ggml_new_tensor_4d(ctx, type, 1, H_v, n_seq_tokens, n_seqs); + ggml_tensor * beta = ggml_new_tensor_4d(ctx, type, 1, H_v, n_seq_tokens, n_seqs); + ggml_tensor * state = ggml_new_tensor_4d(ctx, type, head_size, head_size, H_v, n_seqs); + ggml_set_name(g, "g"); + ggml_set_name(beta, "beta"); + ggml_set_name(state, "state"); + + q = ggml_l2_norm(ctx, q, 1e-6f); + k = ggml_l2_norm(ctx, k, 1e-6f); + + ggml_tensor * gdn_out = ggml_gated_delta_net(ctx, q, k, v, g, beta, state, K); + ggml_set_name(gdn_out, "gdn_out"); + + // attn scores view (first part of the gdn output) + ggml_tensor * attn = ggml_view_4d(ctx, gdn_out, + S_v, H_v, n_seq_tokens, n_seqs, + ggml_row_size(gdn_out->type, S_v), + ggml_row_size(gdn_out->type, S_v * H_v), + ggml_row_size(gdn_out->type, S_v * H_v * n_seq_tokens), 0); + ggml_set_name(attn, "attn"); + + // snapshot tail view [D, n_seqs, n_written] + const int64_t attn_score_elems = S_v * H_v * n_seq_tokens * n_seqs; + ggml_tensor * src = ggml_view_3d(ctx, gdn_out, + D, n_seqs, n_written, + ggml_row_size(gdn_out->type, D), + ggml_row_size(gdn_out->type, D * n_seqs), + ggml_row_size(gdn_out->type, attn_score_elems)); + + // recurrent cache view [D, n_seqs, n_written] + ggml_tensor * cache = ggml_new_tensor_3d(ctx, type, D, n_seqs, n_written); + ggml_set_name(cache, "cache"); + ggml_tensor * dst = ggml_view_3d(ctx, cache, + D, n_seqs, n_written, + ggml_row_size(cache->type, D), + ggml_row_size(cache->type, D * n_seqs), 0); + + ggml_tensor * cpy = ggml_cpy(ctx, src, dst); + ggml_set_name(cpy, "gdn_cache_cpy"); + cpy_node = cpy; + + // read the cpy output (not the plain dst view, which would not pull the cpy into the graph) + // so that neither the gdn nor the cpy is the graph output + ggml_tensor * out = ggml_sum(ctx, cpy); + return out; + } + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "GATED_DELTA_NET_CACHE_FUSION"; + } + + bool run_whole_graph() override { return true; } + std::vector fusion_test_nodes() override { return { cpy_node }; } + + uint64_t op_flops(ggml_tensor * t) override { + GGML_UNUSED(t); + const uint64_t S_v = head_size; + const uint64_t H_v = head_count; + const uint64_t T = n_seq_tokens; + const uint64_t B = n_seqs; + return (4ull*S_v + 2ull*S_v*S_v) * H_v * T * B; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + if (ggml_is_view_op(t->op)) { continue; } + if (strcmp(t->name, "g") == 0) { + init_tensor_uniform(t, -20.0f, -1e-4f); + } else if (strcmp(t->name, "beta") == 0) { + init_tensor_uniform(t, 0.0f, 1.0f); + } else if (strcmp(t->name, "v") == 0) { + init_tensor_uniform(t, -0.3f, 5.0f); + } else if (strcmp(t->name, "cache") == 0) { + init_tensor_uniform(t, 0.0f, 0.0f); + } else { + init_tensor_uniform(t); + } + } + } +}; + // GGML_OP_GATED_LINEAR_ATTN struct test_gla : public test_case { const ggml_type type; @@ -10741,6 +10857,13 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 32, 8, 1, 1, false, false, /*K=*/3)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 16, 2, 1, false, false, /*K=*/4)); + // gdn + cache cpy fusion (K > 1) + test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 4, 32, 2, 1, 2)); + test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 4, 64, 4, 1, 2)); + test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 4, 32, 4, 1, 4)); + test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 8, 32, 4, 2, 4)); + test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 4, 32, 8, 1, 4)); + #if 0 // these tests are disabled to save execution time, sbut they can be handy for debugging test_cases.emplace_back(new test_llama(2, true)); diff --git a/tests/test-fusion.cpp b/tests/test-fusion.cpp new file mode 100644 index 000000000..467248f0f --- /dev/null +++ b/tests/test-fusion.cpp @@ -0,0 +1,565 @@ +// test-fusion: verify the backend fusion logic against a per-device baseline. +// +// for every dummy model generated by test-llama-archs, the tool runs the model on a single +// device with fusion enabled and disabled, and reports: +// - the per-fusion-type counters for each mode (prefill / decode, merged into "any" when the +// per-graph counts match) +// - the NMSE between the fused and unfused logits +// - the NMSE between the device and a CPU reference +// +// the per-fusion-type counters are compared against a per-device baseline file (CSV) so a +// fusion pattern that silently stops matching (or fires when it should not) is caught as a +// regression. +// +// usage: +// test-fusion --models DIR --device MTL0 --record baseline.csv # generate a baseline +// test-fusion --models DIR --device MTL0 --check baseline.csv # validate against it +// test-fusion --model FILE --device MTL0 --check baseline.csv # validate a single model + +#include "common.h" +#include "log.h" +#include "llama-cpp.h" + +#include "ggml.h" +#include "gguf.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// generic fusion debugging API, resolved through the ad-hoc get_proc_address mechanism +// (not part of the official ggml backend interface yet). a backend that adopts fusion debugging +// exports these exact names. +typedef void * ggml_backend_fusion_t; + +typedef ggml_backend_fusion_t ( * fusion_get_t) (ggml_backend_dev_t); +typedef void ( * fusion_stats_init_t) (ggml_backend_fusion_t); +typedef void ( * fusion_stats_reset_t) (ggml_backend_fusion_t); +typedef int ( * fusion_stats_get_t) (ggml_backend_fusion_t, const char **, uint64_t *, int); +typedef void ( * fusion_set_enabled_t) (ggml_backend_fusion_t, bool); + +static bool silent_model_load_progress(float, void *) { + return true; +} + +struct gguf_context_ptr { + gguf_context * ctx; + gguf_context_ptr(gguf_context * c) : ctx(c) {} + ~gguf_context_ptr() { if (ctx) { gguf_free(ctx); } } + gguf_context * get() const { return ctx; } + gguf_context_ptr(const gguf_context_ptr &) = delete; + gguf_context_ptr & operator=(const gguf_context_ptr &) = delete; +}; + +// NMSE between two vectors (same as tests/test-llama-archs.cpp) +static double nmse(const std::vector & a, const std::vector & b) { + GGML_ASSERT(a.size() == b.size()); + double mse_a_b = 0.0; + double mse_a_0 = 0.0; + + for (size_t i = 0; i < a.size(); i++) { + const float a_i = a[i]; + const float b_i = b[i]; + + mse_a_b += (a_i - b_i) * (a_i - b_i); + mse_a_0 += a_i * a_i; + } + + return mse_a_b / mse_a_0; +} + +// deterministic token sequence +static std::vector get_tokens(const uint32_t n_tokens, const uint32_t n_vocab, const size_t seed) { + std::mt19937 gen(seed); + std::uniform_int_distribution<> dis(0, n_vocab - 1); + std::vector ret; + ret.reserve(n_tokens); + for (uint32_t i = 0; i < n_tokens; i++) { + ret.push_back(dis(gen)); + } + return ret; +} + +// trim leading/trailing whitespace (used when parsing padded CSV columns) +static std::string trim(const std::string & s) { + const size_t b = s.find_first_not_of(" \t\r\n"); + if (b == std::string::npos) { + return ""; + } + const size_t e = s.find_last_not_of(" \t\r\n"); + return s.substr(b, e - b + 1); +} + +static std::string get_arch(const std::string & path) { + gguf_init_params params = { /*no_alloc=*/true, /*ctx=*/nullptr }; + gguf_context_ptr ctx(gguf_init_from_file(path.c_str(), params)); + if (!ctx.get()) { + throw std::runtime_error("failed to read gguf: " + path); + } + const int idx = gguf_find_key(ctx.get(), "general.architecture"); + if (idx < 0) { + return "unknown"; + } + const char * val = gguf_get_val_str(ctx.get(), idx); + return val ? val : "unknown"; +} + +static llama_model_ptr load_model(const std::string & path, ggml_backend_dev_t dev) { + llama_model_params model_params = llama_model_default_params(); + model_params.progress_callback = silent_model_load_progress; + std::vector devs = { dev, nullptr }; + model_params.devices = devs.data(); + model_params.split_mode = LLAMA_SPLIT_MODE_LAYER; + + llama_model_ptr model(llama_model_load_from_file(path.c_str(), model_params)); + if (!model) { + throw std::runtime_error("failed to load model: " + path); + } + return model; +} + +// a fresh context (fresh state) from an already-loaded model +static llama_context_ptr create_ctx(llama_model * model, int n_ubatch) { + llama_context_params ctx_params = llama_context_default_params(); + ctx_params.n_ctx = 0; + ctx_params.n_threads = 4; + ctx_params.n_threads_batch = 4; + ctx_params.n_ubatch = n_ubatch; + ctx_params.n_batch = n_ubatch; + + llama_context_ptr lctx(llama_init_from_model(model, ctx_params)); + if (!lctx) { + throw std::runtime_error("failed to init context"); + } + return lctx; +} + +// decode all tokens in one batch; returns the logits of every token +static std::vector decode_prefill(llama_model * model, llama_context * lctx, const std::vector & tokens) { + const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model)); + llama_batch batch = llama_batch_init(tokens.size(), 0, 1); + for (size_t i = 0; i < tokens.size(); i++) { + common_batch_add(batch, tokens[i], i, { 0 }, true); + } + batch.n_tokens = tokens.size(); + if (llama_decode(lctx, batch)) { + llama_batch_free(batch); + throw std::runtime_error("prefill decode failed"); + } + + std::vector ret; + ret.reserve(tokens.size() * n_vocab); + for (size_t i = 0; i < tokens.size(); i++) { + const float * logits_ith = llama_get_logits_ith(lctx, i); + for (uint32_t j = 0; j < n_vocab; j++) { + ret.push_back(logits_ith[j]); + } + } + llama_batch_free(batch); + return ret; +} + +// decode one token at a time; returns the logits of the last token of each step +static std::vector decode_gen(llama_model * model, llama_context * lctx, const std::vector & tokens) { + const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model)); + llama_batch batch = llama_batch_init(1, 0, 1); + std::vector ret; + for (size_t i = 0; i < tokens.size(); i++) { + common_batch_clear(batch); + common_batch_add(batch, tokens[i], i, { 0 }, true); + if (llama_decode(lctx, batch)) { + llama_batch_free(batch); + throw std::runtime_error("decode failed"); + } + const float * logits = llama_get_logits_ith(lctx, 0); + for (uint32_t j = 0; j < n_vocab; j++) { + ret.push_back(logits[j]); + } + } + llama_batch_free(batch); + return ret; +} + +static void read_counts(fusion_stats_get_t api_stats_get, ggml_backend_fusion_t finfo, + std::vector & labels, std::vector & counts) { + const int n = api_stats_get(finfo, nullptr, nullptr, 0); + labels.assign(n, nullptr); + counts.assign(n, 0); + api_stats_get(finfo, labels.data(), counts.data(), n); +} + +// one row of the per-label report +struct fusion_row { + std::string arch; + bool moe; + std::string mode; + std::string label; + uint64_t count_fused; + uint64_t count_unfused; + uint64_t expected; + double nmse_fus; + double nmse_dev; + bool ok_count; // counts match the baseline + bool ok_nmse; // nmse within epsilon +}; + +static void usage(const char * argv0) { + printf("%s: verify fusion counts on a device against a per-device baseline\n\n", argv0); + printf("usage: %s [options]\n\n", argv0); + printf("options:\n"); + printf(" --models DIR run over all .gguf models in a directory\n"); + printf(" --model FILE run over a single model file (mutually exclusive with --models)\n"); + printf(" --device NAME device to run on (e.g. MTL0, CPU)\n"); + printf(" --record CSV write the golden baseline\n"); + printf(" --check CSV validate the counters against a baseline (default)\n"); + printf(" -h, --help show this message and exit\n"); +} + +int main(int argc, char ** argv) { + std::string models_dir; + std::string model_file; + std::string device_name; + std::string record_path; + std::string check_path; + + for (int i = 1; i < argc; i++) { + const std::string arg = argv[i]; + const auto next = [&](const char * name) -> std::string { + if (i + 1 >= argc) { + LOG_ERR("%s: %s requires an argument\n", __func__, name); + exit(1); + } + return argv[++i]; + }; + if (arg == "-h" || arg == "--help") { + usage(argv[0]); + exit(0); + } + if (arg == "--models") { models_dir = next("--models"); } + else if (arg == "--model") { model_file = next("--model"); } + else if (arg == "--device"){ device_name = next("--device"); } + else if (arg == "--record"){ record_path = next("--record"); } + else if (arg == "--check") { check_path = next("--check"); } + else { + LOG_ERR("%s: unknown argument: %s\n", __func__, arg.c_str()); + return 1; + } + } + + if (device_name.empty()) { + LOG_ERR("%s: --device NAME is required\n", __func__); + return 1; + } + if (models_dir.empty() && model_file.empty()) { + LOG_ERR("%s: --models DIR or --model FILE is required\n", __func__); + return 1; + } + if (!models_dir.empty() && !model_file.empty()) { + LOG_ERR("%s: --models DIR and --model FILE are mutually exclusive\n", __func__); + return 1; + } + if (!record_path.empty() && !check_path.empty()) { + LOG_ERR("%s: --record and --check are mutually exclusive\n", __func__); + return 1; + } + + std::vector models; + if (!model_file.empty()) { + if (!std::filesystem::is_regular_file(model_file)) { + LOG_ERR("%s: model file '%s' does not exist\n", __func__, model_file.c_str()); + return 1; + } + models.push_back(model_file); + } else { + if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) { + LOG_ERR("%s: models directory '%s' does not exist\n", __func__, models_dir.c_str()); + return 1; + } + for (const auto & entry : std::filesystem::directory_iterator(models_dir)) { + if (entry.is_regular_file() && entry.path().extension() == ".gguf") { + models.push_back(entry.path().string()); + } + } + std::sort(models.begin(), models.end()); + + if (models.empty()) { + LOG_ERR("%s: no .gguf models found in '%s'\n", __func__, models_dir.c_str()); + return 1; + } + } + + common_init(); + ggml_backend_load_all(); + + ggml_backend_dev_t dev = ggml_backend_dev_by_name(device_name.c_str()); + if (!dev) { + LOG_WRN("%s: device '%s' not found - skipping (baseline is device-specific)\n", + __func__, device_name.c_str()); + return 0; + } + + // resolve the generic fusion debugging functions through the ad-hoc get_proc_address + // mechanism; a backend that does not adopt fusion debugging exports none of them + auto * reg = ggml_backend_dev_backend_reg(dev); + + // output naming uses the backend base name (e.g. "MTL") rather than the specific device + // name (e.g. "MTL0") the test was invoked with + const std::string base_name = ggml_backend_reg_name(reg); + + auto api_get = (fusion_get_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_get"); + auto api_stats_init = (fusion_stats_init_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_init"); + auto api_stats_reset = (fusion_stats_reset_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_reset"); + auto api_stats_get = (fusion_stats_get_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_get"); + auto api_set_enabled = (fusion_set_enabled_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_set_enabled"); + + if (!api_get || !api_stats_init || !api_set_enabled || !api_stats_reset || !api_stats_get) { + LOG_ERR("%s: device '%s' does not export the generic fusion debugging API " + "(ggml_backend_fusion_*) - cannot run the fusion regression test\n", + __func__, device_name.c_str()); + return 1; + } + + ggml_backend_fusion_t finfo = api_get(dev); + + // enable fusions stats + api_stats_init(finfo); + + const bool has_counts = true; + + // load the baseline (if any): key arch|moe|mode|label -> expected count + std::map baseline; + if (!check_path.empty()) { + std::ifstream in(check_path); + if (!in) { + LOG_ERR("%s: cannot open baseline '%s'\n", __func__, check_path.c_str()); + return 1; + } + std::string line; + while (std::getline(in, line)) { + if (line.empty() || line[0] == '#') { + continue; + } + std::vector cols; + size_t pos = 0; + while ((pos = line.find(',')) != std::string::npos) { + cols.push_back(trim(line.substr(0, pos))); + line.erase(0, pos + 1); + } + cols.push_back(trim(line)); + if (cols.size() != 5) { + continue; + } + baseline[cols[0] + "|" + cols[1] + "|" + cols[2] + "|" + cols[3]] = std::stoull(cols[4]); + } + } + + std::vector rows; + + LOG_INF("%s: running fusion test over %zu models on '%s'\n", __func__, models.size(), base_name.c_str()); + + const size_t seed = 1; + + for (const auto & model_path : models) { + const std::string arch = get_arch(model_path); + const bool moe = arch.find("moe") != std::string::npos; + + llama_model_ptr model; + llama_model_ptr model_cpu; + uint32_t n_vocab = 0; + try { + model = load_model(model_path, dev); + model_cpu = load_model(model_path, ggml_backend_dev_by_name("CPU")); + n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model.get())); + } catch (const std::exception & e) { + LOG_ERR("%s: %s: %s\n", __func__, model_path.c_str(), e.what()); + continue; + } + + struct mode_cfg { + std::string name; + std::vector (*decode)(llama_model *, llama_context *, const std::vector &); + int n_tokens; + int n_graphs; // graph runs per mode (prefill=1, decode=16) + }; + const mode_cfg modes[] = { + { "prefill", decode_prefill, 32, 1 }, + { "decode", decode_gen, 16, 16 }, + }; + + // per-label, per-mode data for this model; prefill and decode are merged into a single + // "any" row when their per-graph counts match + struct mode_data { + bool present; + uint64_t count_fused; // per graph + uint64_t count_unfused; // per graph + double nmse_fus; + double nmse_dev; + bool ok_nmse; + }; + std::map> mdata; + + for (int mi = 0; mi < 2; mi++) { + const mode_cfg & mode = modes[mi]; + const auto tokens = get_tokens(mode.n_tokens, n_vocab, seed); + + // CPU reference for this mode (fresh context, fresh state) + std::vector logits_cpu; + try { + llama_context_ptr ctx = create_ctx(model_cpu.get(), 32); + logits_cpu = mode.decode(model_cpu.get(), ctx.get(), tokens); + } catch (const std::exception & e) { + LOG_WRN("%s: %s: cpu reference: %s\n", __func__, model_path.c_str(), e.what()); + } + + // fused run on a fresh context (fresh state) + std::vector logits_fused; + std::vector labels; + std::vector counts_fused; + { + llama_context_ptr ctx = create_ctx(model.get(), 32); + if (has_counts) { + api_set_enabled(finfo, true); + api_stats_reset(finfo); + } + logits_fused = mode.decode(model.get(), ctx.get(), tokens); + if (has_counts) { + read_counts(api_stats_get, finfo, labels, counts_fused); + } + } + + // unfused run on another fresh context (fresh state) + std::vector logits_unfused; + std::vector counts_unfused; + { + llama_context_ptr ctx = create_ctx(model.get(), 32); + if (has_counts) { + api_set_enabled(finfo, false); + api_stats_reset(finfo); + } + logits_unfused = mode.decode(model.get(), ctx.get(), tokens); + if (has_counts) { + read_counts(api_stats_get, finfo, labels, counts_unfused); + } + } + + const double nmse_fus = nmse(logits_fused, logits_unfused); + const double nmse_dev = logits_cpu.empty() ? 0.0 : nmse(logits_fused, logits_cpu); + + if (has_counts) { + for (int i = 0; i < (int) labels.size(); i++) { + const uint64_t fused = counts_fused[i] / mode.n_graphs; + const uint64_t unfused = counts_unfused[i] / mode.n_graphs; + if (fused == 0 && unfused == 0) { + continue; + } + auto & d = mdata[labels[i]][mi]; + d.present = true; + d.count_fused = fused; + d.count_unfused = unfused; + d.nmse_fus = nmse_fus; + d.nmse_dev = nmse_dev; + d.ok_nmse = nmse_fus <= 1e-4; + } + } else { + rows.push_back({ arch, moe, mode.name, "?", 0, 0, 0, nmse_fus, nmse_dev, true, nmse_fus <= 1e-4 }); + } + } + + // build the per-label rows, merging prefill and decode into "any" when the per-graph + // counts match (they always do for the deterministic fusion table) + if (has_counts) { + for (auto & kv : mdata) { + const std::string & label = kv.first; + const auto & d = kv.second; + const bool both = d[0].present && d[1].present; + const bool match = both && d[0].count_fused == d[1].count_fused; + + if (match) { + // one "any" row; use the worst NMSE across the two modes + const std::string any_key = arch + "|" + (moe ? "1" : "0") + "|any|" + label; + const uint64_t expected = baseline.count(any_key) ? baseline.at(any_key) : 0; + const bool ok_count = check_path.empty() || d[0].count_fused == expected; + const bool ok_nmse = d[0].ok_nmse && d[1].ok_nmse; + const double nmse_fus = std::max(d[0].nmse_fus, d[1].nmse_fus); + const double nmse_dev = std::max(d[0].nmse_dev, d[1].nmse_dev); + rows.push_back({ arch, moe, "any", label, d[0].count_fused, d[0].count_unfused, + expected, nmse_fus, nmse_dev, ok_count, ok_nmse }); + } else { + // counts differ - keep a separate row per mode + for (int mi = 0; mi < 2; mi++) { + if (!d[mi].present) { + continue; + } + const mode_data & a = d[mi]; + const std::string mode_key = arch + "|" + (moe ? "1" : "0") + "|" + modes[mi].name + "|" + label; + const uint64_t expected = baseline.count(mode_key) ? baseline.at(mode_key) : 0; + const bool ok_count = check_path.empty() || a.count_fused == expected; + rows.push_back({ arch, moe, modes[mi].name, label, a.count_fused, a.count_unfused, + expected, a.nmse_fus, a.nmse_dev, ok_count, a.ok_nmse }); + } + } + } + } + + LOG_INF("%s: %-20s (%s) done\n", __func__, arch.c_str(), model_path.c_str()); + } + + // print the report + { + std::ofstream out(record_path); + std::ostream & os = record_path.empty() ? std::cout : out; + if (!record_path.empty()) { + os << "# test-fusion baseline for device " << base_name << "\n"; + os << "# " << std::left + << std::setw(18) << "arch" << ',' + << std::setw(4) << "moe" << ',' + << std::setw(8) << "mode" << ',' + << std::setw(28) << "label" << ',' + << std::right << std::setw(7) << "count" << '\n'; + } + + LOG_INF("%-20s %-4s %-8s %-22s %7s %7s %7s %10s %10s %s\n", + "arch", "moe", "mode", "label", "fused", "unfused", "expected", "nmse_fus", "nmse_dev", "status"); + int n_ok = 0; + int n_bad = 0; + for (const auto & r : rows) { + const bool ok = r.ok_count && r.ok_nmse; + const char * status = ok ? "ok" : "FAIL"; + if (ok) { n_ok++; } else { n_bad++; } + LOG_INF("%-20s %-4s %-8s %-22s %7llu %7llu %7llu %10.2e %10.2e %s\n", + r.arch.c_str(), r.moe ? "moe" : "dense", r.mode.c_str(), r.label.c_str(), + (unsigned long long) r.count_fused, (unsigned long long) r.count_unfused, + (unsigned long long) r.expected, r.nmse_fus, r.nmse_dev, status); + if (!record_path.empty()) { + os << std::left + << std::setw(20) << r.arch << ',' + << std::setw(4) << (r.moe ? "1" : "0") << ',' + << std::setw(8) << r.mode << ',' + << std::setw(28) << r.label << ',' + << std::right << std::setw(7) << r.count_fused << '\n'; + } + } + LOG_INF("summary: %d ok, %d failed\n", n_ok, n_bad); + if (!record_path.empty()) { + LOG_INF("%s: baseline written to '%s'\n", __func__, record_path.c_str()); + } + + if (n_bad && !models_dir.empty() && !check_path.empty()) { + LOG_WRN("%s: if the fusion counts are expected to change, run with --record to update the baseline:\n" + "\n" + "./bin/test-llama-archs -o %s\n" + "%s --device %s --models %s --record %s\n", + __func__, models_dir.c_str(), argv[0], device_name.c_str(), models_dir.c_str(), check_path.c_str()); + } + + return n_bad; + } +} diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index dbed9846f..3496f72e4 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -128,7 +128,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { } else if (arch == LLM_ARCH_CHAMELEON) { n_vocab = 10240; } else if (arch == LLM_ARCH_QWEN3TTS) { - n_vocab = 4096; // must be >= the hard-coded codec head size (3072) + //n_vocab = 4096; // must be >= the hard-coded codec head size (3072) + n_vocab = 3072; // TODO: should be 4096, but user code cannot get `n_vocab_out` yet [TAG_LLAMA_N_VOCAB_OUT] } uint32_t n_head_kv = n_head; diff --git a/tests/test-save-load-state.cpp b/tests/test-save-load-state.cpp index 6179e6c10..74d1ba6c2 100644 --- a/tests/test-save-load-state.cpp +++ b/tests/test-save-load-state.cpp @@ -109,7 +109,7 @@ static bool test_seq_rm_isolated( for (llama_seq_id seq_id = 0; seq_id < 2; ++seq_id) { llama_batch_ptr batch(n_tokens, 0, 1); for (size_t i = 0; i < n_tokens; ++i) { - common_batch_add(batch.get(), tokens[i], i, { seq_id }, false); + common_batch_add(batch.get(), tokens[i], i, { seq_id }, i == n_tokens - 1); } if (llama_decode(ctx.get(), batch.get())) { @@ -373,7 +373,7 @@ static bool test_seq_cp_scatter(struct llama_model * model, const struct common_ auto decode_one = [&](llama_token tok, int pos, llama_seq_id seq) { llama_batch_ptr batch(1, 0, 1); - common_batch_add(batch.get(), tok, pos, { seq }, false); + common_batch_add(batch.get(), tok, pos, { seq }, true); return llama_decode(ctx.get(), batch.get()) == 0; }; From 1dfe94e04875bbab710c1fbcb092ae2901b16d1b Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Fri, 11 Sep 2026 12:59:43 +0200 Subject: [PATCH 12/35] common : fix typo in speculative.cpp comment [no ci] (#28750) --- common/speculative.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/speculative.cpp b/common/speculative.cpp index 7c8a06365..77dfe9535 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -1493,7 +1493,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { const int32_t n_tokens = batch_in.n_tokens; - // remember the frist and last batch index for each sequence + // remember the first and last batch index for each sequence std::fill(i_batch_beg.begin(), i_batch_beg.end(), -1); std::fill(i_batch_end.begin(), i_batch_end.end(), -1); From 3bcfeb700fce9ff38a050dcd3f6a856319e948ba Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Fri, 11 Sep 2026 13:01:29 +0200 Subject: [PATCH 13/35] cmake : add PCH and unity build to improve build times (#28091) * scripts : add initial profiling script (wip) * src : add precompile headers (PCH) for models.h * common : add common.h as PCH * ggml : add PCH for ggml-impl.h * mtmd : use PCH for models.h * scripts : add script to build with Server/Tools/Tests * server : add PCH for common.h * docs: add profiling progress notes (wip) * ggml : add exclude for GCC + SVE on ARM Refs: https://github.com/ggml-org/llama.cpp/actions/runs/33393906061/job/99493756214?pr=28091 * ggml : attempt to fix use of std::hardware_destructive_inference_size Refs: https://github.com/ggml-org/llama.cpp/actions/runs/33396221677/job/99501265689?pr=28091 * squash! ggml : attempt to fix use of std::hardware_destructive_inference_size Add a version check for GCC 12 to conditionally apply the `-Winterference-size` pragma. * editorconfig : exclude profiling reports dir This directory will not be included in the merge later and this commit can be ignore at that point. Just fixing to keep CI happy. * ggml : skip PCH for gcc on non-x86 architectures * tests : add PCH for peg-parser/tests.h There are 7 peg-parser tests that can share one PCH instead of then each parsing the full tests.h. * common : add PCH for chat.h * docs : update linux build profiling full results Just updating after a number of PCH additions. These are not exact figures and will vary a bit from run to run, but they give a general idea of the performance impact of PCH. * cmake : introduce unity build for models This commit introduces a unity build for the models to improve compilation time. The improvements were roughly the following: ```console +------------------------+-----+------------+------------+------------+ | Build | TUs | Frontend | Backend | Total | +------------------------+-----+------------+------------+------------+ | Full, master | 396 | 811.0 s | 692.2 s | 1,503.2 s | | Full, with PCH | 405 | 380.0 s | 664.7 s | 1,044.7 s | | Full, with PCH + UB | 264 | 357.7 s | 635.7 s | 993.4 s | +------------------------+-----+------------+------------+------------+ TU = Translation Unit. Full = includes Server, Tools, and Tests. PCH = precompiled headers. UB = unity build for models. ``` * docs : update linux profiling table with unitiy build results * docs : update mac profiling results to include unity build [no ci] * docs: remove profiling reports * scripts : merge build profile scripts into one script I was lazy before and just copied the first script to enable Tests, Server, and Tools. This now merges them into a single script. * Revert "editorconfig : exclude profiling reports dir" [no ci] This reverts commit 2922a12118a0730d2f7632bcba265b44a0856c59. * src : rename ggml_view_2d_slice to gemma3n_view_2d_slice This is to be consistent with the rename in gemma4.cpp which was required to avoid a name clash. * cmake : add build profile script for windows [no ci] This commit adds a port of the scripts/build-profile.sh script to windows powershell. This was developed on Windows on ARM but should work on X64 as well but needs to be tested there as well. --- common/CMakeLists.txt | 2 + docs/build-profiling.md | 122 +++++++++++++++++++++++++++ ggml/src/ggml-cpu/CMakeLists.txt | 6 ++ ggml/src/ggml-cpu/ops.h | 8 ++ scripts/build-profile.ps1 | 136 +++++++++++++++++++++++++++++++ scripts/build-profile.sh | 122 +++++++++++++++++++++++++++ src/CMakeLists.txt | 75 +++++++++-------- src/models/gemma3n.cpp | 20 ++--- src/models/gemma4.cpp | 4 +- tests/CMakeLists.txt | 2 + tools/mtmd/CMakeLists.txt | 7 ++ tools/server/CMakeLists.txt | 2 + 12 files changed, 462 insertions(+), 44 deletions(-) create mode 100644 docs/build-profiling.md create mode 100644 scripts/build-profile.ps1 create mode 100755 scripts/build-profile.sh diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 1506bf647..9a43911d3 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -134,6 +134,8 @@ set_target_properties(${TARGET} PROPERTIES target_include_directories(${TARGET} PUBLIC .) target_link_libraries (${TARGET} PUBLIC vendor::nlohmann vendor::sheredom) target_compile_features (${TARGET} PUBLIC cxx_std_17) +target_precompile_headers (${TARGET} PRIVATE common.h) +target_precompile_headers (${TARGET} PRIVATE chat.h) if (LLAMA_SUBPROCESS) target_compile_definitions(${TARGET} PUBLIC LLAMA_SUBPROCESS) diff --git a/docs/build-profiling.md b/docs/build-profiling.md new file mode 100644 index 000000000..839e7cca4 --- /dev/null +++ b/docs/build-profiling.md @@ -0,0 +1,122 @@ +## Build profiling +This page is a working document for analyzing the current build and try to +identify ways to improve the build time. + +### Requirements +The profiling script requires clang to be used as the compiler tool chain and +also requires that ClangBuildAnalyzer is installed. + +Mac: +```console +brew install clang-build-analyzer +``` + +Linux: +```console +git clone https://github.com/aras-p/ClangBuildAnalyzer.git +cd ClangBuildAnalyzer +cmake -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build -j$(nproc) +sudo cp build/ClangBuildAnalyzer /usr/local/bin/ +``` + +Windows: install LLVM/clang and Ninja (e.g. via the +[LLVM releases page](https://github.com/llvm/llvm-project/releases) and +`winget install Ninja-build.Ninja`), then build ClangBuildAnalyzer the same +way as on Linux: +```console +git clone https://github.com/aras-p/ClangBuildAnalyzer.git +cd ClangBuildAnalyzer +cmake -B build -G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Release +cmake --build build --config Release +``` +Then add `ClangBuildAnalyzer\build` to `PATH`. + +### Usage +Mac/Linux: +```console +$ ./scripts/build-profile.sh +``` + +Windows: +```console +> .\scripts\build-profile.ps1 +``` + +Both accept `--full`/`-Full` (include Server, Tools, and Tests) and a jobs +override (`-jN` / `-Jobs N`). + +Note: on Windows, `cmake` defaults to the Visual Studio generator, which +ignores `CMAKE_C_COMPILER`/`CMAKE_CXX_COMPILER` and silently falls back to +MSVC. `build-profile.ps1` passes `-G Ninja` so clang is actually used, this +is required on ARM64. + +### Linux (Ubuntu 24.04) + +Environment: +- Clang: 18.1.3 (Ubuntu clang version 18.1.3 (1ubuntu1)) +- libstdc++: GCC 13.3.0 (Ubuntu 13.3.0-6ubuntu2~24.04.1) +- Target: x86_64-pc-linux-gnu + +```console ++------------------------+-----+------------+------------+------------+ +| Build | TUs | Frontend | Backend | Total | ++------------------------+-----+------------+------------+------------+ +| Minimal, master | 249 | 468.2 s | 270.3 s | 738.5 s | +| Minimal, with PCH | 253 | 177.1 s | 265.8 s | 442.9 s | +| Full, master | 396 | 811.0 s | 692.2 s | 1,503.2 s | +| Full, with PCH | 405 | 380.0 s | 664.7 s | 1,044.7 s | +| Full, with PCH + UB | 264 | 357.7 s | 635.7 s | 993.4 s | ++------------------------+-----+------------+------------+------------+ + +PCH = precompiled header. +Full = includes building Server, Tools, and Tests. +UB = unity build for models +``` +Note that the number of translation units (TUs) increases when using precompiled +headers — each PCH target adds one extra TU for the precompilation step itself. + +### Mac (Apple M3) + +Environment: +- Clang: Apple clang version 17.0.0 (clang-1700.3.19.1) +- libc++: ships with Apple clang 17.0.0 (Xcode toolchain) +- Target: arm64-apple-macosx15.6 + +```console ++------------------------+-----+------------+------------+------------+ +| Build | TUs | Frontend | Backend | Total | ++------------------------+-----+------------+------------+------------+ +| Minimal, master | 256 | 154.5 s | 94.8 s | 249.3 s | +| Minimal, with PCH | 261 | 65.9 s | 90.0 s | 155.9 s | +| Full, master | 407 | 265.7 s | 209.7 s | 475.4 s | +| Full, with PCH | 414 | 154.6 s | 197.5 s | 352.1 s | +| Full, with PCH + UB | 274 | 143.0 s | 192.2 s | 335.2 s | ++------------------------+-----+------------+------------+------------+ + +PCH = precompiled header. +Full = includes building Server, Tools, and Tests. +UB = unity build for models +``` + +### Windows (ARM64) + +Environment: +- Clang: clang version 22.1.8 (LLVM, `C:\Program Files\LLVM`) +- STL: MSVC STL (Visual Studio 2022 Build Tools 14.44.35207) +- Target: aarch64-pc-windows-msvc + +```console ++------------------------+-----+------------+------------+------------+ +| Build | TUs | Frontend | Backend | Total | ++------------------------+-----+------------+------------+------------+ +| Minimal, master | 249 | 159.4 s | 82.2 s | 241.6 s | +| Full, master | 373 | 337.2 s | 167.4 s | 504.6 s | +| Minimal, with PCH + UB | 113 | 62.3 s | 82.4 s | 144.7 s | +| Full, with PCH + UB | 240 | 233.0 s | 185.1 s | 418.1 s | ++------------------------+-----+------------+------------+------------+ + +PCH = precompiled header. +Full = includes building Server, Tools, and Tests. +UB = unity build for models +``` diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index 1c7338eea..83088e147 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -675,6 +675,12 @@ function(ggml_add_cpu_backend_variant_impl tag_name) target_compile_options(${GGML_CPU_NAME} PRIVATE ${ARCH_FLAGS}) target_compile_definitions(${GGML_CPU_NAME} PRIVATE ${ARCH_DEFINITIONS}) + if (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND NOT GGML_SYSTEM_ARCH STREQUAL "x86") + message(STATUS "Skipping PCH for ${GGML_CPU_NAME}: GCC PCH is only enabled for x86 (arch: ${GGML_SYSTEM_ARCH})") + else() + target_precompile_headers(${GGML_CPU_NAME} PRIVATE ggml-impl.h) + endif() + if (EMSCRIPTEN) set_target_properties(${GGML_CPU_NAME} PROPERTIES COMPILE_FLAGS "-msimd128") endif() diff --git a/ggml/src/ggml-cpu/ops.h b/ggml/src/ggml-cpu/ops.h index 4c1642a67..ce2b3e870 100644 --- a/ggml/src/ggml-cpu/ops.h +++ b/ggml/src/ggml-cpu/ops.h @@ -18,7 +18,15 @@ #endif #endif +// -Winterference-size was introduced in GCC 12 +#if defined(__cplusplus) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winterference-size" +#endif static const size_t CACHE_LINE_SIZE_F32 = CACHE_LINE_SIZE/sizeof(float); +#if defined(__cplusplus) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12 +#pragma GCC diagnostic pop +#endif // Work buffer size for im2col operations in CONV2D #define GGML_IM2COL_WORK_SIZE (16 * 1024 * 1024) diff --git a/scripts/build-profile.ps1 b/scripts/build-profile.ps1 new file mode 100644 index 000000000..410ead39d --- /dev/null +++ b/scripts/build-profile.ps1 @@ -0,0 +1,136 @@ +# Compile-time profiling using clang -ftime-trace + ClangBuildAnalyzer. +# +# Usage: +# .\scripts\build-profile.ps1 [-Full] [-Jobs N] +# +# -Full : include Server, Tools, and Tests (default: minimal build) +# -Jobs : number of parallel jobs (default: all cores) +# +# Requires ClangBuildAnalyzer: +# https://github.com/aras-p/ClangBuildAnalyzer + +param( + [switch]$Full, + [int]$Jobs = [Environment]::ProcessorCount +) + +$ErrorActionPreference = "Stop" + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$RootDir = Split-Path -Parent $ScriptDir + +if ($Full) { + $BuildDir = Join-Path $RootDir "build-profile-full" + $Report = Join-Path $BuildDir "profile-report-full.txt" +} else { + $BuildDir = Join-Path $RootDir "build-profile-baseline" + $Report = Join-Path $BuildDir "profile-report.txt" +} + +$OutputBin = Join-Path $BuildDir "clang_analysis.bin" + +if (-not (Get-Command clang++ -ErrorAction SilentlyContinue)) { + Write-Error "clang++ not found" + exit 1 +} + +if (-not (Get-Command ninja -ErrorAction SilentlyContinue)) { + Write-Error "ninja not found (required so cmake does not fall back to the Visual Studio/MSVC generator)" + exit 1 +} + +if (-not (Get-Command ClangBuildAnalyzer -ErrorAction SilentlyContinue)) { + Write-Error "ClangBuildAnalyzer not found`n https://github.com/aras-p/ClangBuildAnalyzer/releases" + exit 1 +} + +$ClangVer = (clang++ --version | Select-Object -First 1) +Write-Host "compiler : $ClangVer" +Write-Host "build dir: $BuildDir" +Write-Host "output : $OutputBin" +Write-Host "jobs : $Jobs" +Write-Host "" + +if (Get-Command ccache -ErrorAction SilentlyContinue) { + Write-Host "clearing ccache..." + ccache -C -z +} + +$env:CCACHE_DISABLE = "1" + +$TestsFlag = if ($Full) { "ON" } else { "OFF" } +$ToolsFlag = if ($Full) { "ON" } else { "OFF" } +$ServerFlag = if ($Full) { "ON" } else { "OFF" } + +cmake --fresh ` + -S $RootDir ` + -B $BuildDir ` + -G "Ninja" ` + -DCMAKE_BUILD_TYPE=Release ` + -DCMAKE_C_COMPILER=clang ` + -DCMAKE_CXX_COMPILER=clang++ ` + -DCMAKE_C_FLAGS="-ftime-trace" ` + -DCMAKE_CXX_FLAGS="-ftime-trace" ` + -DGGML_CCACHE=OFF ` + -DGGML_OPENMP=ON ` + -DGGML_NATIVE=OFF ` + "-DLLAMA_BUILD_TESTS=$TestsFlag" ` + -DLLAMA_BUILD_EXAMPLES=OFF ` + "-DLLAMA_BUILD_TOOLS=$ToolsFlag" ` + "-DLLAMA_BUILD_SERVER=$ServerFlag" ` + -DLLAMA_BUILD_APP=OFF + +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + +$StrayTrace = Join-Path $RootDir "-.json" +if (Test-Path $StrayTrace) { + Remove-Item $StrayTrace -Force +} + +Write-Host "" +Write-Host "Initializing ClangBuildAnalyzer..." +ClangBuildAnalyzer --start $BuildDir +Write-Host "" + +Write-Host "building..." +Write-Host "" + +$StartTime = Get-Date + +cmake --build $BuildDir --clean-first -j $Jobs + +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + +$Elapsed = (Get-Date) - $StartTime + +Write-Host "" +Write-Host ("build time: {0}s ({1}m {2}s)" -f [int]$Elapsed.TotalSeconds, [int]$Elapsed.TotalMinutes, $Elapsed.Seconds) +Write-Host "" + +Write-Host "Aggregating profile metrics..." +ClangBuildAnalyzer --stop $BuildDir $OutputBin | Out-Null + +Write-Host "" +Write-Host ("=" * 80) + +$TUs = "?" +if (Test-Path $Report) { + $Match = Select-String -Path $Report -Pattern "Compilation \((\d+)" | Select-Object -First 1 + if ($Match) { $TUs = $Match.Matches[0].Groups[1].Value } +} + +ClangBuildAnalyzer --analyze $OutputBin | Tee-Object -FilePath $Report + +Write-Host "" +Write-Host "translation units: $TUs" +Write-Host "" +Write-Host "largest trace files (top 20 by size):" + +Get-ChildItem -Path $BuildDir -Recurse -Filter "*.json" | + Where-Object { $_.Name -ne "compile_commands.json" } | + Sort-Object Length -Descending | + Select-Object -First 20 | + ForEach-Object { "{0,8:F1} KB {1}" -f ($_.Length / 1024), $_.FullName } + +Write-Host "" +Write-Host "ClangBuildAnalyzer report was generated: $Report" diff --git a/scripts/build-profile.sh b/scripts/build-profile.sh new file mode 100755 index 000000000..942994989 --- /dev/null +++ b/scripts/build-profile.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# Compile-time profiling using clang -ftime-trace + ClangBuildAnalyzer. +# +# Usage: +# ./scripts/build-profile.sh [--full] [-jN] +# +# --full: include Server, Tools, and Tests (default: minimal build) +# -jN : number of parallel jobs (default: all cores) +# +# Requires ClangBuildAnalyzer: +# macOS: brew install clang-build-analyzer +# Linux: https://github.com/aras-p/ClangBuildAnalyzer.git + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" + +FULL=0 +JOBS="-j$(nproc 2>/dev/null || sysctl -n hw.ncpu)" + +for arg in "$@"; do + case "${arg}" in + --full) FULL=1 ;; + -j*) JOBS="${arg}" ;; + *) echo "error: unknown argument: ${arg}" >&2; exit 1 ;; + esac +done + +if [ "${FULL}" -eq 1 ]; then + BUILD_DIR="${ROOT_DIR}/build-profile-full" + REPORT="${BUILD_DIR}/profile-report-full.txt" +else + BUILD_DIR="${ROOT_DIR}/build-profile-baseline" + REPORT="${BUILD_DIR}/profile-report.txt" +fi + +OUTPUT_BIN="${BUILD_DIR}/clang_analysis.bin" + +if ! command -v clang++ &>/dev/null; then + echo "error: clang++ not found" >&2 + exit 1 +fi + +if ! command -v ClangBuildAnalyzer &>/dev/null; then + echo "error: ClangBuildAnalyzer not found" >&2 + echo " brew install clangbuildanalyzer (macOS)" >&2 + echo " or: https://github.com/aras-p/ClangBuildAnalyzer/releases" >&2 + exit 1 +fi + +CLANG_VER=$(clang++ --version | head -1) +echo "compiler : ${CLANG_VER}" +echo "build dir: ${BUILD_DIR}" +echo "output : ${OUTPUT_BIN}" +echo "jobs : ${JOBS}" +echo + +if command -v ccache &>/dev/null; then + echo "clearing ccache..." + ccache -C -z +fi + +export CCACHE_DISABLE=1 + +cmake --fresh \ + -S "${ROOT_DIR}" \ + -B "${BUILD_DIR}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_C_FLAGS="-ftime-trace" \ + -DCMAKE_CXX_FLAGS="-ftime-trace" \ + -DGGML_CCACHE=OFF \ + -DGGML_OPENMP=ON \ + -DGGML_NATIVE=OFF \ + -DLLAMA_BUILD_TESTS=$([ "${FULL}" -eq 1 ] && echo ON || echo OFF) \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_TOOLS=$([ "${FULL}" -eq 1 ] && echo ON || echo OFF) \ + -DLLAMA_BUILD_SERVER=$([ "${FULL}" -eq 1 ] && echo ON || echo OFF) \ + -DLLAMA_BUILD_APP=OFF + +echo + +echo "Initializing ClangBuildAnalyzer..." +ClangBuildAnalyzer --start "${BUILD_DIR}" +echo + +echo "building..." +echo + +START=$(date +%s) + +cmake --build "${BUILD_DIR}" --clean-first "${JOBS}" + +END=$(date +%s) +ELAPSED=$((END - START)) + +echo +printf "build time: %ds (%dm %ds)\n" "${ELAPSED}" "$((ELAPSED / 60))" "$((ELAPSED % 60))" +echo + +echo "Aggregating profile metrics..." +ClangBuildAnalyzer --stop "${BUILD_DIR}" "${OUTPUT_BIN}" > /dev/null + +echo +echo "================================================================================" +TUS=$(grep -oP "Compilation \(\K[0-9]+" "${REPORT}" 2>/dev/null || echo "?") +ClangBuildAnalyzer --analyze "${OUTPUT_BIN}" | tee "${REPORT}" + +echo +echo "translation units: ${TUS}" +echo +echo "largest trace files (top 20 by size):" +find "${BUILD_DIR}" -name "*.json" ! -name "compile_commands.json" \ + | xargs ls -l 2>/dev/null \ + | awk 'NF>5 {print $5, $NF}' \ + | sort -rn \ + | awk 'NR<=20 {printf "%8.1f KB %s\n", $1/1024, $2}' + +echo +echo "ClangBuildAnalyzer report was generated: ${REPORT}" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 221e14f7f..bc922b6a7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -8,40 +8,44 @@ llama_add_compile_flags() file(GLOB LLAMA_MODELS_SOURCES "models/*.cpp") +set(LLAMA_CORE_SOURCES + llama.cpp + llama-adapter.cpp + llama-arch.cpp + llama-batch.cpp + llama-chat.cpp + llama-context.cpp + llama-cparams.cpp + llama-grammar.cpp + llama-graph.cpp + llama-hparams.cpp + llama-impl.cpp + llama-io.cpp + llama-kv-cache.cpp + llama-kv-cache-iswa.cpp + llama-kv-cache-dsa.cpp + llama-kv-cache-dsa-iswa.cpp + llama-kv-cache-msa.cpp + llama-kv-cache-dsv4.cpp + llama-memory.cpp + llama-memory-hybrid.cpp + llama-memory-hybrid-iswa.cpp + llama-memory-hybrid-idx.cpp + llama-memory-recurrent.cpp + llama-mmap.cpp + llama-model-loader.cpp + llama-model-saver.cpp + llama-model.cpp + llama-quant.cpp + llama-sampler.cpp + llama-vocab.cpp + unicode-data.cpp + unicode.cpp +) + add_library(llama ../include/llama.h - llama.cpp - llama-adapter.cpp - llama-arch.cpp - llama-batch.cpp - llama-chat.cpp - llama-context.cpp - llama-cparams.cpp - llama-grammar.cpp - llama-graph.cpp - llama-hparams.cpp - llama-impl.cpp - llama-io.cpp - llama-kv-cache.cpp - llama-kv-cache-iswa.cpp - llama-kv-cache-dsa.cpp - llama-kv-cache-dsa-iswa.cpp - llama-kv-cache-msa.cpp - llama-kv-cache-dsv4.cpp - llama-memory.cpp - llama-memory-hybrid.cpp - llama-memory-hybrid-iswa.cpp - llama-memory-hybrid-idx.cpp - llama-memory-recurrent.cpp - llama-mmap.cpp - llama-model-loader.cpp - llama-model-saver.cpp - llama-model.cpp - llama-quant.cpp - llama-sampler.cpp - llama-vocab.cpp - unicode-data.cpp - unicode.cpp + ${LLAMA_CORE_SOURCES} unicode.h ${LLAMA_MODELS_SOURCES} ) @@ -50,13 +54,20 @@ set_target_properties(llama PROPERTIES VERSION ${LLAMA_VERSION_BASE} SOVERSION ${LLAMA_VERSION_MAJOR} MACHO_CURRENT_VERSION 0 # keep macOS linker from seeing oversized version number + UNITY_BUILD ON + UNITY_BUILD_BATCH_SIZE 16 ) +# exclude non-model sources from unity build +set_source_files_properties(${LLAMA_CORE_SOURCES} ../include/llama.h unicode.h + PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) + configure_file(llama-version.h.in ${CMAKE_CURRENT_BINARY_DIR}/llama-version.h @ONLY) target_include_directories(llama PRIVATE . ${CMAKE_CURRENT_BINARY_DIR}) target_include_directories(llama PUBLIC ../include) target_compile_features (llama PRIVATE cxx_std_17) # don't bump +target_precompile_headers (llama PRIVATE models/models.h) target_link_libraries(llama PUBLIC ggml) diff --git a/src/models/gemma3n.cpp b/src/models/gemma3n.cpp index ea616db3b..bb628203a 100644 --- a/src/models/gemma3n.cpp +++ b/src/models/gemma3n.cpp @@ -82,7 +82,7 @@ std::unique_ptr llama_model_gemma3n::build_arch_graph(const l } // get 2D slice view from a 3D tensor, the idx corresponds to the 3rd dim -static ggml_tensor * ggml_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) { +static ggml_tensor * gemma3n_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) { GGML_ASSERT(idx < (int) x->ne[2]); return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], ggml_row_size(x->type, x->ne[0]), idx * x->ne[0] * x->ne[1] * ggml_element_size(x)); @@ -139,7 +139,7 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par ggml_tensor * predictions = altup_predict(cur, il); // [n_embd, n_tokens, n_altup] // predicted value will go through self-attention and laurel - ggml_tensor * active_prediction = ggml_view_2d_slice(ctx0, predictions, i_altup_act); // [n_embd, n_tokens] + ggml_tensor * active_prediction = gemma3n_view_2d_slice(ctx0, predictions, i_altup_act); // [n_embd, n_tokens] cur = active_prediction; cb(cur, "active_prediction", il); @@ -236,13 +236,13 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par ggml_tensor * first_prediction; // [n_embd, n_tokens] { - first_prediction = ggml_view_2d_slice(ctx0, corrected, i_altup_act); // [n_embd, n_tokens] + first_prediction = gemma3n_view_2d_slice(ctx0, corrected, i_altup_act); // [n_embd, n_tokens] first_prediction = ggml_mul(ctx0, first_prediction, model.layers[il].altup_correct_scale); first_prediction = build_lora_mm(model.layers[il].per_layer_inp_gate, first_prediction); first_prediction = ggml_gelu(ctx0, first_prediction); // [n_embd_altup, n_tokens] cb(first_prediction, "first_prediction_gated", il); - ggml_tensor * inp_this_layer = ggml_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_altup, n_tokens] + ggml_tensor * inp_this_layer = gemma3n_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_altup, n_tokens] first_prediction = ggml_mul(ctx0, first_prediction, inp_this_layer); // [n_embd_altup, n_tokens] cb(first_prediction, "first_prediction_scaled", il); @@ -253,7 +253,7 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par } // equivalent to python code: corrected_predictions[1:] += first_prediction { - ggml_tensor * slice_first = ggml_view_2d_slice(ctx0, corrected, 0); + ggml_tensor * slice_first = gemma3n_view_2d_slice(ctx0, corrected, 0); ggml_tensor * slice_rest = ggml_view_3d( ctx0, corrected, n_embd, n_tokens, n_altup - 1, ggml_row_size(corrected->type, n_embd), ggml_row_size(corrected->type, n_embd * n_tokens), n_embd * n_tokens * ggml_element_size(corrected)); @@ -271,7 +271,7 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par // cur now has multiple altup(s), we want to merge them back to 1 altup { - ggml_tensor * target_magnitude = calc_magnitude(ggml_view_2d_slice(ctx0, cur, i_altup_act)); // [n_embd, n_tokens] + ggml_tensor * target_magnitude = calc_magnitude(gemma3n_view_2d_slice(ctx0, cur, i_altup_act)); // [n_embd, n_tokens] // do a view to skip the first slice (active altup) ggml_tensor * alt_slice = ggml_view_3d(ctx0, cur, n_embd, n_tokens, n_altup - 1, ggml_row_size(cur->type, n_embd), @@ -283,9 +283,9 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par cb(altup_unembd, "altup_unembd", -1); // equivalent to torch.mean(hidden_states, dim=0) - cur = ggml_view_2d_slice(ctx0, cur, 0); // [n_embd, n_tokens] + cur = gemma3n_view_2d_slice(ctx0, cur, 0); // [n_embd, n_tokens] for (int i = 0; i < n_altup - 1; ++i) { - cur = ggml_add(ctx0, cur, ggml_view_2d_slice(ctx0, altup_unembd, i)); + cur = ggml_add(ctx0, cur, gemma3n_view_2d_slice(ctx0, altup_unembd, i)); } cur = ggml_scale(ctx0, cur, 1.0f / float(n_altup)); // [n_embd, n_tokens] cb(cur, "unembd_merged", -1); @@ -419,7 +419,7 @@ ggml_tensor * llama_model_gemma3n::graph::altup_compute_router_modalities(ggml_t // input cur shape: [n_embd, n_tokens, n_altup] // output shape: [n_embd, n_tokens, n_altup] ggml_tensor * llama_model_gemma3n::graph::altup_predict(ggml_tensor * cur, int il) { - ggml_tensor * activated = ggml_view_2d_slice(ctx0, cur, i_altup_act); // [n_embd, n_tokens] + ggml_tensor * activated = gemma3n_view_2d_slice(ctx0, cur, i_altup_act); // [n_embd, n_tokens] ggml_tensor * modalities = altup_compute_router_modalities(activated, il); // [n_altup, n_tokens] cb(modalities, "modalities", il); @@ -447,7 +447,7 @@ ggml_tensor * llama_model_gemma3n::graph::altup_correct(ggml_tensor * prediction ggml_tensor * modalities = altup_compute_router_modalities(activated, il); // [n_altup, n_tokens] cb(modalities, "modalities", il); - ggml_tensor * active_prediction = ggml_view_2d_slice(ctx0, predictions, i_altup_act); + ggml_tensor * active_prediction = gemma3n_view_2d_slice(ctx0, predictions, i_altup_act); ggml_tensor * innovation = ggml_sub(ctx0, activated, active_prediction); // [n_embd, n_tokens] cb(innovation, "innovation", il); diff --git a/src/models/gemma4.cpp b/src/models/gemma4.cpp index 388126e26..39e899aa6 100644 --- a/src/models/gemma4.cpp +++ b/src/models/gemma4.cpp @@ -145,7 +145,7 @@ std::unique_ptr llama_model_gemma4::build_arch_graph(const ll } // get 2D slice view from a 3D tensor, the idx corresponds to the 3rd dim -static ggml_tensor * ggml_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) { +static ggml_tensor * gemma4_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) { GGML_ASSERT(idx < (int) x->ne[2]); return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], ggml_row_size(x->type, x->ne[0]), idx * x->ne[0] * x->ne[1] * ggml_element_size(x)); @@ -372,7 +372,7 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para cur = build_lora_mm(model.layers[il].per_layer_inp_gate, cur); // [n_embd_per_layer, n_tokens] cur = ggml_gelu(ctx0, cur); - ggml_tensor * inp_this_layer = ggml_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_per_layer, n_tokens] + ggml_tensor * inp_this_layer = gemma4_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_per_layer, n_tokens] // TODO @ngxson : improve this if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 920c58c73..0c4e4d5a9 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -278,6 +278,8 @@ llama_build_and_test( peg-parser/test-unicode.cpp peg-parser/tests.h ) +target_precompile_headers(test-peg-parser PRIVATE peg-parser/tests.h) + if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "s390x") set(MODEL_NAME "tinyllamas/stories15M-q4_0.gguf") diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index 907468e87..176eb1505 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -84,6 +84,13 @@ target_link_libraries (mtmd PUBLIC ggml llama) target_link_libraries (mtmd PRIVATE Threads::Threads vendor::hash vendor::miniaudio vendor::stb vendor::sheredom) target_include_directories(mtmd PUBLIC .) target_compile_features (mtmd PRIVATE cxx_std_17) +target_precompile_headers (mtmd PRIVATE models/models.h) + +set_source_files_properties( + mtmd-helper.cpp + mtmd-helper-gen.cpp + PROPERTIES SKIP_PRECOMPILE_HEADERS ON +) if (MTMD_VIDEO) target_compile_definitions(mtmd PRIVATE MTMD_VIDEO) diff --git a/tools/server/CMakeLists.txt b/tools/server/CMakeLists.txt index 43c245633..f02a2ba3b 100644 --- a/tools/server/CMakeLists.txt +++ b/tools/server/CMakeLists.txt @@ -32,6 +32,7 @@ endif() target_include_directories(${TARGET} PRIVATE ../mtmd) target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}) target_link_libraries(${TARGET} PUBLIC llama-common mtmd ${CMAKE_THREAD_LIBS_INIT}) +target_precompile_headers(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/common/common.h) # llama-server-impl: server logic, reusable by app @@ -49,6 +50,7 @@ set_target_properties(${TARGET} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON) target_include_directories(${TARGET} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories(${TARGET} PRIVATE ../mtmd ${CMAKE_SOURCE_DIR}) target_link_libraries(${TARGET} PUBLIC server-context llama-ui cpp-httplib ${CMAKE_THREAD_LIBS_INIT}) +target_precompile_headers(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/common/common.h) add_dependencies(${TARGET} llama-ui-assets) From 5bda51bfbc62e64193221e639f6ad4e08767d760 Mon Sep 17 00:00:00 2001 From: Foad Abo Dahood <32059146+masterFoad@users.noreply.github.com> Date: Fri, 11 Sep 2026 14:12:55 +0300 Subject: [PATCH 14/35] metal : skip the empty half of the mul_mm_id token tile (#28301) kernel_mul_mm_id splits its NR1 = 32 token tile into two 16-row halves and skips the upper half when the expert did not fill it, on both the tensor and simdgroup paths. The tB extents are corrected to (NK, NR1H) for the [NR1][NK] row-major tile. The B tile is staged unconditionally, as on master: rows past nr1 restage a clamped duplicate of a valid row, lie in the output-row dimension so they never contribute to a valid row, and are dropped by the final store loop. test-backend-ops: re-draw the expert ids between perf iterations of test_mul_mat_id so MoE perf numbers are not warm-cache, and add token-tile boundary coverage using n_used == n_mats, which routes every token to every expert so each expert receives exactly n rows; n = 32, 33, 47, 48, 49 reach mul_mm_id and leave a last tile of 32, 1, 15, 16 and 17 rows. --- ggml/src/ggml-metal/kernels/mul_mm.metal | 99 +++++++++++++++--------- tests/test-backend-ops.cpp | 55 ++++++++++--- 2 files changed, 107 insertions(+), 47 deletions(-) diff --git a/ggml/src/ggml-metal/kernels/mul_mm.metal b/ggml/src/ggml-metal/kernels/mul_mm.metal index ee848eed6..0a45bb1bb 100644 --- a/ggml/src/ggml-metal/kernels/mul_mm.metal +++ b/ggml/src/ggml-metal/kernels/mul_mm.metal @@ -496,6 +496,13 @@ kernel void kernel_mul_mm_id( + args.nb11*i11 + args.nb10*iy); + // skip the upper half of the token tile when the expert did not fill it + constexpr short NR1H = NR1/2; + + const bool has_hi = nr1 > NR1H; + + const short lb1 = (short) tiitg/NL1; // 0 .. NR1-1, this thread's row of the B tile + #ifndef GGML_METAL_HAS_TENSOR S0_8x8 ma[4]; S1_8x8 mb[2]; @@ -505,15 +512,22 @@ kernel void kernel_mul_mm_id( for (short i = 0; i < 8; i++){ mc[i] = make_filled_simdgroup_matrix(0.f); } + + // simdgroups 2,3 own rows NR1H..NR1-1 + const bool sg_active = has_hi || sgitg < 2; #else - auto tA = tensor, tensor_inline>(sa, dextents(NK, NR0)); - auto tB = tensor, tensor_inline>(sb, dextents(NR1, NK )); + auto tA = tensor, tensor_inline>(sa, dextents(NK, NR0)); + + // sb is [NR1][NK] row-major + auto tB0 = tensor, tensor_inline>(sb, dextents(NK, NR1H)); + auto tB1 = tensor, tensor_inline>(sb + NR1H*NK, dextents(NK, NR1H)); mpp::tensor_ops::matmul2d< - mpp::tensor_ops::matmul2d_descriptor(NR1, NR0, NK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate), + mpp::tensor_ops::matmul2d_descriptor(NR1H, NR0, NK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate), execution_simdgroups<4>> mm; - auto cT = mm.get_destination_cooperative_tensor(); + auto cT0 = mm.get_destination_cooperative_tensor(); + auto cT1 = mm.get_destination_cooperative_tensor(); #endif for (int loop_k = 0; loop_k < args.ne00; loop_k += NK) { @@ -656,37 +670,45 @@ kernel void kernel_mul_mm_id( threadgroup_barrier(mem_flags::mem_threadgroup); #ifndef GGML_METAL_HAS_TENSOR - // load matrices from threadgroup memory and conduct outer products - threadgroup const S0 * lsma = (sa + 4*64*(sgitg%2)); - threadgroup const S1 * lsmb = (sb + 2*64*(sgitg/2)); + if (sg_active) { + // load matrices from threadgroup memory and conduct outer products + threadgroup const S0 * lsma = (sa + 4*64*(sgitg%2)); + threadgroup const S1 * lsmb = (sb + 2*64*(sgitg/2)); - FOR_UNROLL (short ik = 0; ik < NK/8; ik++) { - simdgroup_barrier(mem_flags::mem_none); + FOR_UNROLL (short ik = 0; ik < NK/8; ik++) { + simdgroup_barrier(mem_flags::mem_none); - FOR_UNROLL (short i = 0; i < 4; i++) { - simdgroup_load(ma[i], lsma + 64*i, 8, 0, false); + FOR_UNROLL (short i = 0; i < 4; i++) { + simdgroup_load(ma[i], lsma + 64*i, 8, 0, false); + } + + simdgroup_barrier(mem_flags::mem_none); + + FOR_UNROLL (short i = 0; i < 2; i++) { + simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false); + } + + simdgroup_barrier(mem_flags::mem_none); + + FOR_UNROLL (short i = 0; i < 8; i++){ + simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]); + } + + lsma += 8*64; + lsmb += 4*64; } - - simdgroup_barrier(mem_flags::mem_none); - - FOR_UNROLL (short i = 0; i < 2; i++) { - simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false); - } - - simdgroup_barrier(mem_flags::mem_none); - - FOR_UNROLL (short i = 0; i < 8; i++){ - simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]); - } - - lsma += 8*64; - lsmb += 4*64; } #else - auto sA = tA.slice(0, 0); - auto sB = tB.slice(0, 0); + auto sA = tA.slice(0, 0); + auto sB0 = tB0.slice(0, 0); - mm.run(sB, sA, cT); + mm.run(sB0, sA, cT0); + + if (has_hi) { + auto sB1 = tB1.slice(0, 0); + + mm.run(sB1, sA, cT1); + } #endif } @@ -694,13 +716,20 @@ kernel void kernel_mul_mm_id( threadgroup_barrier(mem_flags::mem_threadgroup); #ifdef GGML_METAL_HAS_TENSOR - auto tC = tensor, tensor_inline>(sc, dextents(NR0, NR1)); - cT.store(tC); -#else - threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; + auto tC0 = tensor, tensor_inline>(sc, dextents(NR0, NR1H)); + cT0.store(tC0); - for (short i = 0; i < 8; i++) { - simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); + if (has_hi) { + auto tC1 = tensor, tensor_inline>(sc + NR1H*NR0, dextents(NR0, NR1H)); + cT1.store(tC1); + } +#else + if (sg_active) { + threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; + + for (short i = 0; i < 8; i++) { + simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); + } } #endif diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 15c42a108..b63b3773e 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -1219,6 +1219,11 @@ struct test_case { } } + // re-draw data-dependent inputs between timed perf iterations + virtual void reinit_perf_iter(ggml_context * ctx) { + GGML_UNUSED(ctx); + } + virtual size_t op_size(ggml_tensor * t) { size_t size = ggml_nbytes(t); // add source tensors @@ -1653,6 +1658,9 @@ struct test_case { total_time_us += end_time - start_time; total_mem += mem; total_runs += n_runs; + + // re-draw any data-dependent inputs (expert ids) outside the timed region + reinit_perf_iter(ctx.get()); } while (total_time_us < 1000*1000); // run for at least 1 second // Create test result @@ -5000,25 +5008,31 @@ struct test_mul_mat_hadamard : public test_mul_mat { } }; -static void init_mul_mat_id_tensors(ggml_context * ctx, int n_mats) { +static void init_mul_mat_id_ids(ggml_context * ctx, int n_mats) { std::random_device rd; std::default_random_engine rng(rd()); for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (t->type == GGML_TYPE_I32) { - if (ggml_is_view_op(t->op)) { continue; } - // ids - for (int64_t r = 0; r < ggml_nrows(t); r++) { - std::vector data(t->ne[0]); - for (int i = 0; i < t->ne[0]; i++) { - data[i] = i % n_mats; - } - std::shuffle(data.begin(), data.end(), rng); - ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t)); + if (t->type != GGML_TYPE_I32 || ggml_is_view_op(t->op)) { + continue; + } + for (int64_t r = 0; r < ggml_nrows(t); r++) { + std::vector data(t->ne[0]); + for (int i = 0; i < t->ne[0]; i++) { + data[i] = i % n_mats; } - } else { + std::shuffle(data.begin(), data.end(), rng); + ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t)); + } + } +} + +static void init_mul_mat_id_tensors(ggml_context * ctx, int n_mats) { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->type != GGML_TYPE_I32) { init_tensor_uniform(t); } } + init_mul_mat_id_ids(ctx, n_mats); } // GGML_OP_MUL_MAT_ID @@ -5085,6 +5099,10 @@ struct test_mul_mat_id : public test_case { void initialize_tensors(ggml_context * ctx) override { init_mul_mat_id_tensors(ctx, n_mats); } + + void reinit_perf_iter(ggml_context * ctx) override { + init_mul_mat_id_ids(ctx, n_mats); + } }; // GGML_OP_MUL_MAT_ID + GGML_OP_ADD or GGML_OP_MUL @@ -9890,6 +9908,19 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 16, 256, {2, 3}, {1, 1}, {0, 1, 3, 2})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 16, 256, {2, 3}, {1, 1}, {0, 3, 2, 1})); + // token-tile boundary coverage. With n_used == n_mats every token routes to every expert, so + // each expert receives exactly n rows, with no dependence on the random draw. mul_mm_id is used + // from 32 tokens up: n = 32, 33, 47, 48, 49 reach it, leaving a last tile of 32, 1, 15, 16 and + // 17 rows - 16 and 17 straddle the point where the upper half stops being skipped. The smaller + // n cover the same row counts on the mat-vec path. + for (ggml_type type_a : {GGML_TYPE_Q4_K, GGML_TYPE_IQ2_XS, GGML_TYPE_F16}) { + for (int n : {1, 15, 16, 17, 31, 32, 33, 47, 48, 49}) { + test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 4, 4, false, 512, n, 256)); + } + // experts that receive no rows at all + test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 8, 1, false, 512, 1, 256)); + } + for (ggml_type type_a : other_types) { for (ggml_type type_b : {GGML_TYPE_F32}) { if (ggml_blck_size(type_a) != 256) { From 43f3dda6237a453a587a8f00230d52decfeaa8e5 Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Fri, 11 Sep 2026 21:17:08 +0800 Subject: [PATCH 15/35] ggml: skip 0-sized ids tensor when offloading selected experts (#28739) --- ggml/src/ggml-backend.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index 40e50c5c9..6faa68047 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -1705,6 +1705,10 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s ggml_tensor * ids_tensor = node->src[2]; ggml_backend_t ids_backend = split_backend; + if (ggml_nelements(ids_tensor) == 0) { + continue; + } + // if the ids tensor is also an input of the split, it may not have been copied yet to the split backend // in that case, we use the original ids tensor for (int i = input_id + 1; i < split->n_inputs; i++) { From 8172e6577ac2b35de1ec1e5d1c0aaad6c4a2129f Mon Sep 17 00:00:00 2001 From: Pascal Date: Fri, 11 Sep 2026 15:50:12 +0200 Subject: [PATCH 16/35] tests: tolerate a shared pool abort in test_completion_unified (#28759) The expected success table holds when the four requests enter the shared pool together. On a loaded runner they are admitted tens of milliseconds apart, the slot lifetimes overlap differently and the pool overflows while a short request is still resident. The decode failure aborts every slot, so a request the table marks as successful comes back with the context error instead of its generation. Such a request now passes on that error too, while any other status, a different error or a truncated generation still fails the test. --- tools/server/tests/unit/test_completion.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/server/tests/unit/test_completion.py b/tools/server/tests/unit/test_completion.py index 9375e0110..01732eb16 100644 --- a/tools/server/tests/unit/test_completion.py +++ b/tools/server/tests/unit/test_completion.py @@ -394,7 +394,12 @@ def test_completion_unified(n_ctx, n_slots, n_predict_vals, expected_success): results = parallel_function_calls(tasks) for res, n_predict, expect_ok in zip(results, n_predict_vals, expected_success): if expect_ok: - assert res.status_code == 200 + # the pool is aborted as a whole, so a request that fits on its own + # is still dropped when the slots overlap, and it says so explicitly + assert res.status_code == 200 or ( + res.status_code == 500 + and "context size has been exceeded" in res.body["error"]["message"].lower() + ) # note: https://github.com/ggml-org/llama.cpp/pull/18700#issuecomment-3728695581 if res.status_code == 200: From 982937a3337f7e97ef08fd5603f4157575ece7e1 Mon Sep 17 00:00:00 2001 From: Rohanjames1997 Date: Fri, 11 Sep 2026 13:19:37 -0500 Subject: [PATCH 17/35] tests: extend test-quantize-fns to test nrc=2 (i8mm) kernels (#16234) * Test for nrc=2 as well | i8mm kernels * Trigger only on supported HW * Remove trailing whitespace * Address review comment * test: properly prepare nrc=2 inputs with independent data per row * tests : make nrc=2 dot product inputs distinct Assisted-by: Kiro * tests : use non-trivial strides in nrc=2 dot product test * tests : fail nrc=2 dot product test on non-finite errors --- tests/test-quantize-fns.cpp | 69 ++++++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cpp index 9510ac14c..570fca89a 100644 --- a/tests/test-quantize-fns.cpp +++ b/tests/test-quantize-fns.cpp @@ -5,6 +5,8 @@ #undef NDEBUG #include +#include +#include #include #include #include @@ -32,9 +34,9 @@ static const char* RESULT_STR[] = {"ok", "FAILED"}; // Generate synthetic data -static void generate_data(float offset, size_t n, float * dst) { +static void generate_data(float offset, size_t n, float * dst, float amplitude = 2.0f) { for (size_t i = 0; i < n; i++) { - dst[i] = 0.1 + 2*cosf(i + offset); + dst[i] = 0.1 + amplitude*cosf(i + offset); } } @@ -83,23 +85,50 @@ static float dot_product(const float * a1, const float * a2, size_t test_size) { } // Total dot product error -static float dot_product_error(const ggml_type_traits * qfns, const ggml_type_traits_cpu * qfns_cpu, size_t test_size, const float * test_data1, const float * test_data2) { - GGML_UNUSED(qfns); - - std::vector tmp_q1(2*test_size); - std::vector tmp_q2(2*test_size); - +static float dot_product_error(const ggml_type_traits_cpu * qfns_cpu, ggml_type src0_type, size_t test_size, + const float * test_data1, const float * test_data2, + const float * test_data3, const float * test_data4, + const int nrc) { const auto * vdot = ggml_get_type_traits_cpu(qfns_cpu->vec_dot_type); + const size_t pad = 64; + const size_t bx = ggml_row_size(src0_type, test_size) + pad; + const size_t by = ggml_row_size(qfns_cpu->vec_dot_type, test_size) + pad; + + std::vector tmp_q1(bx * nrc); + std::vector tmp_q2(by * nrc); qfns_cpu->from_float(test_data1, tmp_q1.data(), test_size); vdot->from_float(test_data2, tmp_q2.data(), test_size); - float result = INFINITY; - qfns_cpu->vec_dot(test_size, &result, 0, tmp_q1.data(), 0, tmp_q2.data(), 0, 1); + if (nrc == 1) { + float result = INFINITY; + qfns_cpu->vec_dot(test_size, &result, 0, tmp_q1.data(), 0, tmp_q2.data(), 0, 1); - const float dot_ref = dot_product(test_data1, test_data2, test_size); + const float dot_ref = dot_product(test_data1, test_data2, test_size); + return fabsf(result - dot_ref) / test_size; + } - return fabsf(result - dot_ref) / test_size; + // nrc == 2: kernel computes a 2x2 dot product matrix + // Output layout: s[0]=dot(vx0,vy0), s[1]=dot(vx1,vy0), s[bs]=dot(vx0,vy1), s[bs+1]=dot(vx1,vy1) + // row and output strides are padded, same as in the mul_mat path + qfns_cpu->from_float(test_data3, tmp_q1.data() + bx, test_size); + vdot->from_float(test_data4, tmp_q2.data() + by, test_size); + + const size_t bs = 16; + std::vector result(bs + 2, INFINITY); + qfns_cpu->vec_dot(test_size, result.data(), bs, tmp_q1.data(), bx, tmp_q2.data(), by, 2); + + const float ref00 = dot_product(test_data1, test_data2, test_size); + const float ref10 = dot_product(test_data3, test_data2, test_size); + const float ref01 = dot_product(test_data1, test_data4, test_size); + const float ref11 = dot_product(test_data3, test_data4, test_size); + + const auto err = [test_size](float val, float ref) { + const float e = fabsf(val - ref) / test_size; + return std::isfinite(e) ? e : INFINITY; + }; + + return std::max({err(result[0], ref00), err(result[1], ref10), err(result[bs], ref01), err(result[bs + 1], ref11)}); } static int test_vec_dot_f32(bool verbose) { @@ -133,9 +162,13 @@ static int test_vec_dot_q(bool verbose) { std::vector test_data(test_size); std::vector test_data2(test_size); + std::vector test_data3(test_size); + std::vector test_data4(test_size); generate_data(0.0, test_data.size(), test_data.data()); generate_data(1.0, test_data2.size(), test_data2.data()); + generate_data(3.0, test_data3.size(), test_data3.data(), 1.0f); + generate_data(4.0, test_data4.size(), test_data4.data(), 1.5f); for (int i = 0; i < GGML_TYPE_COUNT; i++) { ggml_type type = (ggml_type) i; @@ -178,7 +211,7 @@ static int test_vec_dot_q(bool verbose) { printf("%5s reference implementation error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], reference_error); } - const float vec_dot_error = dot_product_error(qfns, qfns_cpu, test_size, test_data.data(), test_data2.data()); + const float vec_dot_error = dot_product_error(qfns_cpu, type, test_size, test_data.data(), test_data2.data(), nullptr, nullptr, 1); const float max_allowed_error = type == GGML_TYPE_Q2_K || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ2_S ? MAX_DOT_PRODUCT_ERROR_LOWBIT @@ -194,6 +227,16 @@ static int test_vec_dot_q(bool verbose) { if (failed || verbose) { printf("%5s dot product error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error); } + + // Test nrc=2 path for types that support it + if (qfns_cpu->nrows == 2) { + const float vec_dot_error_nrc2 = dot_product_error(qfns_cpu, type, test_size, test_data.data(), test_data2.data(), test_data3.data(), test_data4.data(), 2); + failed = !(vec_dot_error_nrc2 < max_allowed_error); + num_failed += failed; + if (failed || verbose) { + printf("%5s dot product error (nrc=2): %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error_nrc2); + } + } } } From b78a39a2f93b13a79a3e01aff3f14274efb43afc Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 11 Sep 2026 22:00:57 +0300 Subject: [PATCH 18/35] ci : run test-backend-ops as a dedicated ci/run.sh test (#28740) * ci : run test-backend-ops as a dedicated gg test Run test-backend-ops as a separate gg test in ci/run.sh so it is executed outside ctest. With GG_BUILD_HIGH_PERF it keeps the existing CPU-only invocation (-b CPU); otherwise it runs all available backends without a backend filter. Remove the dedicated backend-ops workflow and keep test-backend-ops as a built target that is not registered with ctest to avoid duplicate runs. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp * ci : run test-backend-ops earlier and enable high-perf on kleidiai Move the test-backend-ops gg test before test-llama-archs. Enable GG_BUILD_HIGH_PERF and LLAMA_ARG_THREADS on the Graviton4 KleidiAI job and use the standard self-hosted results/mnt paths. Add TODO markers for decoupling tests from libllama. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp * ci : run test-backend-ops in parallel Pass -j $(nproc) to test-backend-ops in both high-perf and all-backend modes. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp * ci : disable parallel tests for ROCm * cont : disable parallel tests with MoltenVK --- .github/workflows/build-openvino.yml | 2 +- .github/workflows/build-self-hosted.yml | 10 +++++-- .github/workflows/build-vulkan.yml | 4 +-- .github/workflows/build-webgpu.yml | 4 +-- ci/run.sh | 38 ++++++++++++++++++------- tests/CMakeLists.txt | 12 ++------ 6 files changed, 41 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build-openvino.yml b/.github/workflows/build-openvino.yml index 8879a6af1..86aba456c 100644 --- a/.github/workflows/build-openvino.yml +++ b/.github/workflows/build-openvino.yml @@ -33,7 +33,7 @@ env: LLAMA_ARG_LOG_PREFIX: 1 LLAMA_ARG_LOG_TIMESTAMPS: 1 # TODO: fix failing tests on OpenVINO backend - CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-|test-backend-ops|test-save-load-state" + CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-|test-save-load-state" jobs: ubuntu-24-openvino: diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml index fda4879e2..02a38466f 100644 --- a/.github/workflows/build-self-hosted.yml +++ b/.github/workflows/build-self-hosted.yml @@ -395,7 +395,11 @@ jobs: - name: Test id: ggml-ci run: | - LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + LLAMA_ARG_THREADS=$(nproc) \ + GG_BUILD_HIGH_PERF=1 \ + GG_BUILD_NO_BF16=1 \ + GG_BUILD_EXTRA_TESTS_0=1 \ + bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp cpu-arm64-graviton4-kleidiai: runs-on: ah-ubuntu_22_04-c8g_8x @@ -434,6 +438,8 @@ jobs: - name: Test id: ggml-ci run: | + LLAMA_ARG_THREADS=$(nproc) \ GG_BUILD_KLEIDIAI=1 \ GG_BUILD_EXTRA_TESTS_0=1 \ - bash ./ci/run.sh ./tmp/results ./tmp/mnt + GG_BUILD_HIGH_PERF=1 \ + bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp diff --git a/.github/workflows/build-vulkan.yml b/.github/workflows/build-vulkan.yml index 9de52e990..21d2a7735 100644 --- a/.github/workflows/build-vulkan.yml +++ b/.github/workflows/build-vulkan.yml @@ -164,9 +164,7 @@ jobs: export GGML_VK_VISIBLE_DEVICES=0 export GGML_VK_DISABLE_F16=1 export GGML_VK_DISABLE_COOPMAT=1 - # This is using llvmpipe and runs slower than other backends - # test-backend-ops is too slow on llvmpipe, skip it - ctest -L main -E test-backend-ops --verbose --timeout 900 + ctest -L main --verbose --timeout 900 windows: runs-on: windows-2025 diff --git a/.github/workflows/build-webgpu.yml b/.github/workflows/build-webgpu.yml index e624e3ba8..8277abcc4 100644 --- a/.github/workflows/build-webgpu.yml +++ b/.github/workflows/build-webgpu.yml @@ -190,6 +190,4 @@ jobs: id: cmake_test run: | cd build - # This is using llvmpipe and runs slower than other backends - # test-backend-ops is too slow on llvmpipe, skip it - ctest -L main -E test-backend-ops --verbose --timeout 900 + ctest -L main --verbose --timeout 900 diff --git a/ci/run.sh b/ci/run.sh index 294cbe57b..1ceb19fd5 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -190,7 +190,7 @@ if [ ! -z ${GG_BUILD_OPENVINO} ]; then CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON" # TODO: fix failing tests on OpenVINO backend - CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-|test-backend-ops|test-save-load-state" + CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-|test-save-load-state" fi ## helpers @@ -250,7 +250,7 @@ function gg_run_ctest_debug { (cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log (time cmake --build . --config Debug -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log - (time ctest -C Debug --output-on-failure -L main -E "test-opt|test-backend-ops|test-llama-archs" ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log + (time ctest -C Debug --output-on-failure -L main -E "test-opt|test-llama-archs" ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log set +e } @@ -768,25 +768,43 @@ function gg_check_build_requirements { fi } -function gg_run_test_backend_ops_cpu { +function gg_run_test_backend_ops { cd ${SRC} cd build-ci-release set -e - (time ./bin/test-backend-ops -b CPU ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops-cpu.log + local args_extra="-j $(nproc)" + + # TODO: fix multi-threaded for ROCm + # https://github.com/ggml-org/llama.cpp/actions/runs/34576278519/job/103297889044?pr=28740#step:3:4865 + if [ ! -z ${GG_BUILD_ROCM} ]; then + args_extra="" + fi + + # TODO: MoltenVK bug? + # https://github.com/ggml-org/llama.cpp/actions/runs/34611260059/job/103302413736?pr=28740#step:3:5897 + if [ ! -z "${GG_BUILD_VULKAN}" ] && [ "$(uname -s)" = "Darwin" ]; then + args_extra="" + fi + + if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then + (time ./bin/test-backend-ops ${args_extra} -b CPU) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log + else + (time ./bin/test-backend-ops ${args_extra} ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log + fi set +e } -function gg_sum_test_backend_ops_cpu { +function gg_sum_test_backend_ops { gg_printf '### %s\n\n' "${ci}" - gg_printf 'Runs test-backend-ops for CPU backend\n' + gg_printf 'Runs test-backend-ops\n' gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" gg_printf '```\n' - gg_printf '%s\n' "$(cat $OUT/${ci}-test-backend-ops-cpu.log)" + gg_printf '%s\n' "$(cat $OUT/${ci}-test-backend-ops.log)" gg_printf '```\n' gg_printf '\n' } @@ -819,13 +837,11 @@ ret=0 test $ret -eq 0 && gg_run ctest_debug test $ret -eq 0 && gg_run ctest_release +test $ret -eq 0 && gg_run test_backend_ops + test $ret -eq 0 && gg_run test_llama_archs_models test $ret -eq 0 && gg_run test_llama_archs_tensor_split -if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then - test $ret -eq 0 && gg_run test_backend_ops_cpu -fi - if [ -z ${GG_BUILD_LOW_PERF} ]; then test $ret -eq 0 && gg_run embd_bge_small test $ret -eq 0 && gg_run rerank_tiny diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0c4e4d5a9..b3559a173 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -10,7 +10,7 @@ function(llama_build source) endif() add_executable(${TEST_TARGET} ${TEST_SOURCES}) - target_link_libraries(${TEST_TARGET} PRIVATE llama llama-common) + target_link_libraries(${TEST_TARGET} PRIVATE llama llama-common) # TODO: [TAG_TESTS_LLAMA_LINK] if (LLAMA_TESTS_INSTALL) install(TARGETS ${TEST_TARGET} RUNTIME) endif() @@ -310,15 +310,9 @@ if (NOT LLAMA_SANITIZE_ADDRESS AND NOT GGML_SCHED_NO_REALLOC) # TODO: repair known memory leaks llama_build_and_test(test-opt.cpp) endif() -llama_build_and_test(test-backend-ops.cpp) -# the tensor API kernels come from a separate metallib - check they produce correct results -# ref: https://github.com/ggml-org/llama.cpp/issues/27473 -if (GGML_METAL AND NOT GGML_METAL_EMBED_LIBRARY) - llama_test(test-backend-ops NAME test-backend-ops-metallib-tensor - ARGS test -b MTL0 -o MUL_MAT -p type_a=q6_K) - set_tests_properties(test-backend-ops-metallib-tensor PROPERTIES ENVIRONMENT GGML_METAL_TENSOR_ENABLE=1) -endif() +# TODO: make this test (and others) not link `libllama` as it is not needed [TAG_TESTS_LLAMA_LINK] +llama_build(test-backend-ops.cpp) llama_build_and_test(test-model-load-cancel.cpp LABEL "model") llama_build_and_test(test-autorelease.cpp LABEL "model") From 8ea290247c87ced2ab245b056ffe96dbcf90d36c Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Fri, 11 Sep 2026 21:36:52 +0200 Subject: [PATCH 19/35] cmake : skip PCH for llama-server PCH when using MSVC (#28763) This commit fixes an issue that I introduced when adding PCH (precompiled headers) in Commit 3bcfeb700fce9ff38a050dcd3f6a856319e948ba ("cmake : add PCH and unity build to improve build times (#28091)". See linked issue for details. Co-authored-by: mjungnickel18 Co-authored-by: Pascal Resolves: https://github.com/ggml-org/llama.cpp/issues/28758 Refs: https://github.com/ggml-org/llama.cpp/actions/runs/34592933983/job/103262608990#step:9:1284 --- tools/server/CMakeLists.txt | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tools/server/CMakeLists.txt b/tools/server/CMakeLists.txt index f02a2ba3b..02607c838 100644 --- a/tools/server/CMakeLists.txt +++ b/tools/server/CMakeLists.txt @@ -1,5 +1,13 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) +# MSVC emits a PCH bookkeeping symbol that WINDOWS_EXPORT_ALL_SYMBOLS exports as an ambiguous "__" + +set(LLAMA_SERVER_PCH ON) + +if (BUILD_SHARED_LIBS AND CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + set(LLAMA_SERVER_PCH OFF) +endif() + # server-context containing the core server logic, used by llama-server and CLI set(TARGET server-context) @@ -32,7 +40,10 @@ endif() target_include_directories(${TARGET} PRIVATE ../mtmd) target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}) target_link_libraries(${TARGET} PUBLIC llama-common mtmd ${CMAKE_THREAD_LIBS_INIT}) -target_precompile_headers(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/common/common.h) + +if (LLAMA_SERVER_PCH) + target_precompile_headers(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/common/common.h) +endif() # llama-server-impl: server logic, reusable by app @@ -50,7 +61,10 @@ set_target_properties(${TARGET} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON) target_include_directories(${TARGET} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories(${TARGET} PRIVATE ../mtmd ${CMAKE_SOURCE_DIR}) target_link_libraries(${TARGET} PUBLIC server-context llama-ui cpp-httplib ${CMAKE_THREAD_LIBS_INIT}) -target_precompile_headers(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/common/common.h) + +if (LLAMA_SERVER_PCH) + target_precompile_headers(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/common/common.h) +endif() add_dependencies(${TARGET} llama-ui-assets) From 82d6bb284d1ff1c6ef37f29a4c3b63d1a8b11806 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Sat, 12 Sep 2026 00:53:07 +0200 Subject: [PATCH 20/35] server: refactor subproc handling (#28555) * server: refactor subproc handling * fix Windows build * download: keep concurrent downloads of one blob apart Every process writes the same path + .downloadInProgress, so a second download of the same blob finds that file, takes it for its own partial transfer and asks for the bytes after it, which produces a corrupt result. The in-progress file now carries the pid of the process writing it. std::rename also replaces an existing destination on POSIX but fails on Windows, so a download whose blob appeared in the meantime is dropped after every retry and an etag rewrite silently keeps the old value. std::filesystem::rename has the POSIX behaviour everywhere, and the error now carries the reason reported by the system. * Revert "download: keep concurrent downloads of one blob apart" This reverts commit 917b83f149c625527f872fb2cf41289358fa5371. * tests: serialize the router tests that download the same model Parallel workers share one cache, so the two tests fetch the same blob into the same in-progress file and race to rename it. They now take a file lock around the download, like the session fixture does for the preset models. * Revert "tests: serialize the router tests that download the same model" This reverts commit c368a4a98c677938ca87002edb6186ff2c02fd83. --------- Co-authored-by: Pascal --- tools/server/server-common.cpp | 145 ++++++++++ tools/server/server-common.h | 38 +++ tools/server/server-models.cpp | 471 +++++++++++++++++++-------------- tools/server/server-models.h | 26 +- 4 files changed, 473 insertions(+), 207 deletions(-) diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index 2ac98b6fd..eade7db21 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -16,6 +16,21 @@ #include #include +#ifdef _WIN32 +// windows.h defines min and max as macros, which breaks std::min and std::max +#define WIN32_LEAN_AND_MEAN +#ifndef NOMINMAX +# define NOMINMAX +#endif +#include +#include +#else +#include +#include +#include +#include +#endif + json format_error_response(const std::string & message, const enum error_type type) { std::string type_str; int code = 500; @@ -1832,3 +1847,133 @@ server_tokens format_prompt_rerank( return result; } + +// +// server_subproc +// + +bool server_subproc::has_output() { + if (out_handle >= 0) { + return true; + } + FILE * f = sproc.stdout_file(); // combined stdout/stderr + if (!f) { + return false; + } +#ifdef _WIN32 + HANDLE h = (HANDLE) _get_osfhandle(_fileno(f)); + if (h != INVALID_HANDLE_VALUE) { + out_handle = (intptr_t) h; + } +#else + int fd = fileno(f); + if (fd >= 0) { + fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK); + out_handle = fd; + } +#endif + return out_handle >= 0; +} + +int server_subproc::read_output(char * buf, size_t len) { + if (!has_output()) { + return -1; + } +#ifdef _WIN32 + HANDLE h = (HANDLE) out_handle; + DWORD avail = 0; + if (!PeekNamedPipe(h, NULL, 0, NULL, &avail, NULL)) { + return -1; // pipe broken, child gone + } + if (avail == 0) { + return 0; + } + DWORD to_read = avail < (DWORD) len ? avail : (DWORD) len; + DWORD got = 0; + if (!ReadFile(h, buf, to_read, &got, NULL) || got == 0) { + return -1; + } + return (int) got; +#else + while (true) { + ssize_t r = read((int) out_handle, buf, len); + if (r > 0) { + return (int) r; + } + if (r == 0) { + return -1; // EOF + } + if (errno == EINTR) { + continue; + } + if (errno == EAGAIN || errno == EWOULDBLOCK) { + return 0; + } + return -1; + } +#endif +} + +server_subproc::waiter::waiter() { +#ifndef _WIN32 + int fds[2]; + GGML_ASSERT(pipe(fds) == 0); + for (int fd : fds) { + fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK); + } + wake_fd[0] = fds[0]; + wake_fd[1] = fds[1]; +#endif +} + +server_subproc::waiter::~waiter() { +#ifndef _WIN32 + close((int) wake_fd[0]); + close((int) wake_fd[1]); +#endif +} + +void server_subproc::waiter::wake() { +#ifndef _WIN32 + char c = 1; + (void) !write((int) wake_fd[1], &c, 1); +#endif +} + +void server_subproc::waiter::wait(const std::vector & procs, std::vector & ready, int64_t timeout_ms) { + ready.assign(procs.size(), false); +#ifdef _WIN32 + // no waitable wait exists for anonymous pipes, so poll them in 50 ms steps + bool any = false; + for (size_t i = 0; i < procs.size(); i++) { + DWORD avail = 0; + if (!procs[i]->has_output() || !PeekNamedPipe((HANDLE) procs[i]->out_handle, NULL, 0, NULL, &avail, NULL) || avail > 0) { + ready[i] = true; // data or broken pipe, read_output() tells which + any = true; + } + } + if (!any) { + int64_t step = timeout_ms < 0 ? 50 : std::min(timeout_ms, 50); + std::this_thread::sleep_for(std::chrono::milliseconds(step)); + } +#else + std::vector pfds; + pfds.reserve(procs.size() + 1); + pfds.push_back({ (int) wake_fd[0], POLLIN, 0 }); + for (auto * p : procs) { + pfds.push_back({ p->has_output() ? (int) p->out_handle : -1, POLLIN, 0 }); // poll() skips negative fds + } + int timeout = timeout_ms < 0 ? -1 : (int) std::min(timeout_ms, std::numeric_limits::max()); + int r = poll(pfds.data(), pfds.size(), timeout); + if (r < 0 && errno != EINTR) { + LOG_ERR("%s: poll() failed: %s\n", __func__, strerror(errno)); + } + if (pfds[0].revents) { + char buf[64]; + while (read((int) wake_fd[0], buf, sizeof(buf)) > 0) {} + } + for (size_t i = 0; i < procs.size(); i++) { + ready[i] = pfds[i + 1].fd < 0 || pfds[i + 1].revents != 0; + } +#endif +} diff --git a/tools/server/server-common.h b/tools/server/server-common.h index 6c681a2cf..9894f5f06 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -6,6 +6,7 @@ #include "chat.h" #include "mtmd.h" #include "mtmd-helper.h" +#include "subproc.h" #include "json.h" @@ -13,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -611,3 +613,39 @@ struct server_pipe { return true; } }; + +// wrapper around common_subproc to manage a child server process +// mainly used by router mode +struct server_subproc { + common_subproc sproc; + std::atomic stopped{false}; // set by the monitor once the process exited and was reaped + + bool is_alive() { return sproc.alive(); } + void terminate() { sproc.terminate(); } + int join() { return sproc.join(); } + + // true if the child's combined stdout/stderr pipe is available (call after create()) + bool has_output(); + + // non-blocking read + // returns the number of bytes read, 0 when nothing is available, -1 when the pipe is closed or broken + int read_output(char * buf, size_t len); + + // wait until one of a set of children has output, wake() is called, or a timeout passes + struct waiter { + waiter(); + ~waiter(); + + // thread-safe; on Windows this is a no-op, wait() returns within 50 ms anyway + void wake(); + + // timeout_ms < 0 waits until data or wake(); ready[i] is set for each child with data (or a broken pipe) + void wait(const std::vector & procs, std::vector & ready, int64_t timeout_ms); + + private: + intptr_t wake_fd[2] = { -1, -1 }; // POSIX self-pipe + }; + +private: + intptr_t out_handle = -1; // fd on POSIX, HANDLE on Windows; taken lazily from sproc +}; diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 4d2592b25..f1783c083 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -44,30 +44,215 @@ extern char **environ; #define CMD_ROUTER_TO_CHILD_EXIT "cmd_router_to_child:exit" #define CMD_CHILD_TO_ROUTER_STATE "cmd_child_to_router:state:" // followed by json string +// note: SIGPIPE is ignored by the server +static void request_child_exit(server_subproc & proc) { + FILE * stdin_file = proc.sproc.stdin_file(); + if (stdin_file) { + fprintf(stdin_file, "%s\n", CMD_ROUTER_TO_CHILD_EXIT); + fflush(stdin_file); + } +} + // address for child process, this is needed because router may run on 0.0.0.0 // ref: https://github.com/ggml-org/llama.cpp/issues/17862 #define CHILD_ADDR "127.0.0.1" -struct server_subproc { - common_subproc sproc; // not yet spawned while in DOWNLOADING state - std::atomic stopped{false}; // set to cancel a download or signal child process exit - - bool is_alive() { - return sproc.alive(); +// single-threaded, watching all child processes at once +struct server_monitor { + server_monitor(server_models & models) : models(models) { + th = std::thread([this]() { run(); }); } - void request_exit() { - FILE * stdin_file = sproc.stdin_file(); - if (stdin_file) { - fprintf(stdin_file, "%s\n", CMD_ROUTER_TO_CHILD_EXIT); - fflush(stdin_file); + ~server_monitor() { + push({ cmd_t::QUIT, {}, "", 0, false }); + th.join(); + } + + // thread-safe + void watch(const std::string & name, std::shared_ptr proc, server_child_mode mode, int port) { + child_t c; + c.name = name; + c.proc = std::move(proc); + c.mode = mode; + c.port = port; + if (!c.proc->has_output()) { + SRV_ERR("failed to get stdout/stderr of child process for name=%s\n", name.c_str()); + c.eof = true; } - stopped.store(true, std::memory_order_relaxed); + push({ cmd_t::WATCH, std::move(c), "", 0, false }); } - void terminate() { - sproc.terminate(); + // thread-safe + void stop(const std::string & name, int stop_timeout, bool send_exit) { + push({ cmd_t::STOP, {}, name, stop_timeout, send_exit }); } + +private: + struct child_t { + std::string name; + std::shared_ptr proc; + server_child_mode mode = SERVER_CHILD_MODE_NORMAL; + int port = 0; + std::string buf; // partial line + bool eof = false; // output closed, waiting for the process to be reaped + int64_t deadline = 0; // force-kill time in ms, 0 when no stop is pending + }; + + struct cmd_t { + enum { WATCH, STOP, QUIT } type; + child_t child; + std::string name; + int stop_timeout; + bool send_exit; + }; + + void push(cmd_t && cmd) { + { + std::lock_guard lk(mu); + cmds.push_back(std::move(cmd)); + } + waiter.wake(); + } + + // returns true if the loop should exit + bool handle_commands() { + std::deque batch; + { + std::lock_guard lk(mu); + batch.swap(cmds); + } + for (auto & cmd : batch) { + switch (cmd.type) { + case cmd_t::WATCH: + children.push_back(std::move(cmd.child)); + break; + case cmd_t::STOP: + // the newest child with this name is the one the registry knows + for (auto it = children.rbegin(); it != children.rend(); ++it) { + if (it->name != cmd.name) { + continue; + } + if (cmd.send_exit && !it->eof) { + request_child_exit(*it->proc); + } + it->deadline = ggml_time_ms() + (int64_t) cmd.stop_timeout * 1000; + break; + } + break; + case cmd_t::QUIT: + return true; + } + } + return false; + } + + // read what the child wrote, forward complete lines + void read_output(child_t & c) { + char chunk[4096]; + while (!c.eof) { + int n = c.proc->read_output(chunk, sizeof(chunk)); + if (n < 0) { + c.eof = true; + break; + } + if (n == 0) { + break; + } + c.buf.append(chunk, (size_t) n); + size_t start = 0; + while (true) { + size_t nl = c.buf.find('\n', start); + if (nl == std::string::npos) { + break; + } + std::string line = c.buf.substr(start, nl + 1 - start); + start = nl + 1; + on_line(c, line); + } + c.buf.erase(0, start); + if (c.buf.size() > max_line) { + c.buf.clear(); // a child that never writes a newline must not grow this without bound + } + } + if (c.eof && !c.buf.empty()) { + on_line(c, c.buf); + c.buf.clear(); + } + } + + void on_line(child_t & c, const std::string & line) { + if (string_starts_with(line, CMD_CHILD_TO_ROUTER_STATE)) { + LOG_DBG("[%5d] %s", c.port, line.c_str()); // prevent spamming the log + models.handle_child_state(c.name, line); + } else { + LOG("[%5d] %s", c.port, line.c_str()); // forward log + } + } + + void run() { + while (true) { + if (handle_commands()) { + return; + } + + // wait for output, a wakeup, or the next deadline; + // a child whose output closed is polled for its exit every 50 ms + int64_t now = ggml_time_ms(); + int64_t timeout = -1; + for (const auto & c : children) { + if (c.eof) { + timeout = timeout < 0 ? 50 : std::min(timeout, 50); + } + if (c.deadline) { + int64_t d = std::max(0, c.deadline - now); + timeout = timeout < 0 ? d : std::min(timeout, d); + } + } + std::vector procs; + std::vector owners; + for (auto & c : children) { + if (!c.eof) { + procs.push_back(c.proc.get()); + owners.push_back(&c); + } + } + std::vector ready; + waiter.wait(procs, ready, timeout); + for (size_t i = 0; i < owners.size(); i++) { + if (ready[i]) { + read_output(*owners[i]); + } + } + + // deadlines and exits + now = ggml_time_ms(); + for (auto it = children.begin(); it != children.end();) { + if (it->deadline && now >= it->deadline && !it->proc->stopped.load(std::memory_order_acquire)) { + SRV_WRN("force-killing model instance name=%s after timeout\n", it->name.c_str()); + it->proc->terminate(); + it->deadline = 0; + } + if (it->eof && !it->proc->is_alive()) { + int exit_code = it->proc->join(); + it->proc->stopped.store(true, std::memory_order_release); + models.on_child_exit(it->name, it->proc, it->mode, exit_code); + SRV_INF("instance name=%s exited with status %d\n", it->name.c_str(), exit_code); + it = children.erase(it); + } else { + ++it; + } + } + } + } + + static constexpr size_t max_line = 1024 * 1024; + + server_models & models; + std::mutex mu; + std::deque cmds; + std::vector children; // monitor thread only + server_subproc::waiter waiter; + std::thread th; }; struct server_lru_sched { @@ -395,7 +580,8 @@ server_models::server_models( base_params(params), base_env(get_environment()), base_preset(ctx_preset.load_from_args(argc, argv)), - sched(std::make_unique(*this)) { + sched(std::make_unique(*this)), + monitor(std::make_unique(*this)) { // clean up base preset unset_reserved_args(base_preset, true); // set binary path @@ -412,6 +598,10 @@ server_models::server_models( server_models::~server_models() = default; +void server_models::instance_t::request_exit() const { + request_child_exit(*subproc); +} + void server_models::add_model(server_model_meta && meta) { if (mapping.find(meta.name) != mapping.end()) { throw std::runtime_error(string_format("model '%s' appears multiple times", meta.name.c_str())); @@ -466,7 +656,6 @@ void server_models::add_model(server_model_meta && meta) { std::string name = meta.name; mapping[name] = instance_t{ /* subproc */ std::make_shared(), - /* th */ std::thread(), /* meta */ std::move(meta) }; } @@ -621,9 +810,7 @@ void server_models::load_models() { }; // Phase 2: acquire the lock once for all mapping mutations. - // We temporarily release it only when calling functions that acquire it internally - // (unload, load) or when joining threads (the monitoring thread calls update_status - // which locks the mutex, so joining while holding it would deadlock). + // We temporarily release it only when calling functions that acquire it internally (unload) std::unique_lock lk(mutex); need_reload = false; @@ -708,49 +895,15 @@ void server_models::load_models() { return true; }); - // collect all threads to join in one pass while the lock is held: - // - monitoring threads from just-unloaded models (to_unload) - // - threads of finished downloads (DOWNLOADED), they acquire the mutex on exit - // - threads of already-UNLOADED models that are being removed from source - std::vector threads_to_join; - for (const auto & name : to_unload) { - auto it = mapping.find(name); - if (it != mapping.end() && it->second.th.joinable()) { - threads_to_join.push_back(std::move(it->second.th)); - } - } - for (auto & [name, inst] : mapping) { - if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { - continue; // downloading models are not from config sources, leave them alone - } - if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADED) { - // joining this thread under the lock deadlocks: it locks the mutex on its way out - if (inst.th.joinable()) { - threads_to_join.push_back(std::move(inst.th)); - } - continue; - } - if (final_presets.find(name) == final_presets.end() && !inst.meta.is_running() && inst.th.joinable()) { - threads_to_join.push_back(std::move(inst.th)); - } - } - - // join outside the lock - monitoring thread calls update_status (needs lock) - lk.unlock(); - for (auto & th : threads_to_join) th.join(); - lk.lock(); - // erase models no longer in any source for (auto it = mapping.begin(); it != mapping.end(); ) { if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { ++it; // download thread is still busy, skip } else if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADED) { - // download finished, thread is joined above, safe to erase - GGML_ASSERT(!it->second.th.joinable()); + // download finished, safe to erase it = mapping.erase(it); } else if (final_presets.find(it->first) == final_presets.end()) { SRV_INF("(reload) removing model name=%s (no longer in source)\n", it->first.c_str()); - GGML_ASSERT(!it->second.th.joinable()); // must have been joined above it = mapping.erase(it); } else { ++it; @@ -1030,117 +1183,12 @@ void server_models::load(const std::string & name, const load_options & opts) { } } - // start a thread to manage the child process - // captured variables are guaranteed to be destroyed only after the thread is joined - inst.th = std::thread([ - this, name, - child_proc = inst.subproc, - port = inst.meta.port, - stop_timeout = inst.meta.stop_timeout, - child_mode = opts.mode - ]() { - FILE * stdin_file = child_proc->sproc.stdin_file(); - FILE * stdout_file = child_proc->sproc.stdout_file(); // combined stdout/stderr - - std::thread log_thread([&]() { - // read stdout/stderr and forward to main server log - // also handle status report from child process - std::vector vec_buf(128 * 1024); // large buffer for storing info - char * buffer = vec_buf.data(); - if (stdout_file) { - while (fgets(buffer, vec_buf.size(), stdout_file) != nullptr) { - std::string str(buffer); - if (string_starts_with(buffer, CMD_CHILD_TO_ROUTER_STATE)) { - LOG_DBG("[%5d] %s", port, buffer); // prevent spamming the log - this->handle_child_state(name, str); - } else { - // forward log - LOG("[%5d] %s", port, buffer); - } - } - } else { - SRV_ERR("failed to get stdout/stderr of child process for name=%s\n", name.c_str()); - } - }); - - std::thread stopping_thread([&]() { - // thread to monitor explicit stop requests; child crash is signalled via child_proc->stopped - auto is_stopping = [this, &name]() { - return this->stopping_models.find(name) != this->stopping_models.end(); - }; - { - std::unique_lock lk(this->mutex); - this->cv_stop.wait(lk, [&]() { - return is_stopping() || child_proc->stopped.load(std::memory_order_acquire); - }); - } - // child crashed or finished on its own, skip graceful shutdown sequence - if (child_proc->stopped.load(std::memory_order_acquire)) { - return; - } - SRV_INF("stopping model instance name=%s\n", name.c_str()); - fprintf(stdin_file, "%s\n", CMD_ROUTER_TO_CHILD_EXIT); - fflush(stdin_file); - int64_t start_time = ggml_time_ms(); - while (true) { - std::unique_lock lk(this->mutex); - if (!is_stopping() || child_proc->stopped.load(std::memory_order_acquire)) { - return; - } - int64_t elapsed = ggml_time_ms() - start_time; - if (elapsed >= stop_timeout * 1000) { - lk.unlock(); - SRV_WRN("force-killing model instance name=%s after %d seconds timeout\n", name.c_str(), stop_timeout); - child_proc->terminate(); - return; - } - this->cv_stop.wait_for(lk, std::chrono::seconds(1), [&]() { - return !is_stopping() || child_proc->stopped.load(std::memory_order_acquire); - }); - } - }); - - // we reach here when the child process exits (stdout EOF) - // note: we cannot join() prior to this point because it will close stdin_file - if (log_thread.joinable()) { - log_thread.join(); - } - - child_proc->stopped.store(true, std::memory_order_release); - { - std::lock_guard lk(this->mutex); - stopping_models.erase(name); - cv_stop.notify_all(); - } - if (stopping_thread.joinable()) { - stopping_thread.join(); - } - - // get the exit code - int exit_code = child_proc->sproc.join(); - - // update status and exit code - if (child_mode == SERVER_CHILD_MODE_DOWNLOAD) { - // instance will be cleaned up on next load_models() call - } else { - this->update_status(name, { - SERVER_MODEL_STATUS_UNLOADED, - exit_code - }); - } - SRV_INF("instance name=%s exited with status %d\n", name.c_str(), exit_code); - }); - - // clean up old process/thread if exists + // old process should have exited already, but just in case, we clean it up here { - auto & old_instance = mapping[name]; - // old process should have exited already, but just in case, we clean it up here - if (old_instance.subproc && old_instance.subproc->is_alive()) { + auto it = mapping.find(name); + if (it != mapping.end() && it->second.subproc && it->second.subproc->is_alive()) { SRV_WRN("old process for model name=%s is still alive, this is unexpected\n", name.c_str()); - old_instance.subproc->terminate(); // force kill - } - if (old_instance.th.joinable()) { - old_instance.th.join(); + it->second.subproc->terminate(); // force kill } } @@ -1148,13 +1196,41 @@ void server_models::load(const std::string & name, const load_options & opts) { {"status", server_model_status_to_string(inst.meta.status)}, }); + auto proc = inst.subproc; + int port = inst.meta.port; mapping[name] = std::move(inst); + monitor->watch(name, proc, opts.mode, port); cv.notify_all(); } -void server_models::request_stop(const std::string & name) { +void server_models::request_stop(const std::string & name, bool send_exit) { + auto it = mapping.find(name); + if (it == mapping.end() || stopping_models.count(name)) { + return; + } stopping_models.insert(name); - cv_stop.notify_all(); + monitor->stop(name, it->second.meta.stop_timeout, send_exit); +} + +void server_models::on_child_exit(const std::string & name, const std::shared_ptr & proc, server_child_mode mode, int exit_code) { + { + std::lock_guard lk(mutex); + stopping_models.erase(name); + auto it = mapping.find(name); + if (it == mapping.end() || it->second.subproc != proc) { + return; // entry erased, or a newer instance took the name + } + } + if (mode == SERVER_CHILD_MODE_DOWNLOAD) { + // instance will be cleaned up on next load_models() call + std::lock_guard lk(mutex); + cv.notify_all(); + } else { + update_status(name, { + SERVER_MODEL_STATUS_UNLOADED, + exit_code + }); + } } void server_models::unload(const std::string & name) { @@ -1163,20 +1239,21 @@ void server_models::unload(const std::string & name) { if (it != mapping.end()) { if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { SRV_INF("cancelling download for model name=%s\n", name.c_str()); - it->second.subproc->request_exit(); + it->second.request_exit(); // for convenience, we wait the status change here wait(lk, name, [](const server_model_meta & new_meta) { return new_meta.status != SERVER_MODEL_STATUS_DOWNLOADING; }); } else if (it->second.meta.is_running()) { SRV_INF("stopping model instance name=%s\n", name.c_str()); - if (it->second.meta.status == SERVER_MODEL_STATUS_LOADING) { + bool loading = it->second.meta.status == SERVER_MODEL_STATUS_LOADING; + if (loading) { // special case: if model is in loading state, unloading means force-killing it SRV_WRN("model name=%s is still loading, force-killing\n", name.c_str()); it->second.subproc->terminate(); } - request_stop(name); - // status change will be handled by the managing thread + request_stop(name, !loading); + // status change will be handled by the monitor } else { SRV_WRN("model instance name=%s is not running\n", name.c_str()); } @@ -1184,27 +1261,29 @@ void server_models::unload(const std::string & name) { } void server_models::unload_all() { - std::vector to_join; - { - std::lock_guard lk(mutex); - for (auto & [name, inst] : mapping) { - if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { - SRV_INF("cancelling download for model name=%s\n", name.c_str()); - inst.subproc->stopped.store(true, std::memory_order_relaxed); - } else if (inst.meta.is_running()) { - SRV_INF("stopping model instance name=%s\n", name.c_str()); - request_stop(name); - // status change will be handled by the managing thread + std::unique_lock lk(mutex); + for (auto & [name, inst] : mapping) { + if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { + SRV_INF("cancelling download for model name=%s\n", name.c_str()); + inst.request_exit(); + } else if (inst.meta.is_running()) { + SRV_INF("stopping model instance name=%s\n", name.c_str()); + bool loading = inst.meta.status == SERVER_MODEL_STATUS_LOADING; + if (loading) { + inst.subproc->terminate(); } - // moving the thread to join list to avoid deadlock - to_join.push_back(std::move(inst.th)); + request_stop(name, !loading); } } - for (auto & th : to_join) { - if (th.joinable()) { - th.join(); + // wait for every child to exit, the monitor force-kills the ones that ignore the exit command + cv.wait(lk, [this]() { + for (const auto & [name, inst] : mapping) { + if (inst.meta.is_running() || inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { + return false; + } } - } + return true; + }); } void server_models::update_status(const std::string & name, const update_status_args & args) { @@ -1291,18 +1370,18 @@ bool server_models::remove(const std::string & name) { if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { // cancel in-flight download SRV_INF("cancelling download for model name=%s\n", name.c_str()); - it->second.subproc->request_exit(); + it->second.request_exit(); } else if (it->second.meta.is_running()) { // stop running instance SRV_INF("stopping model instance name=%s\n", name.c_str()); - stopping_models.insert(name); - if (it->second.meta.status == SERVER_MODEL_STATUS_LOADING) { + bool loading = it->second.meta.status == SERVER_MODEL_STATUS_LOADING; + if (loading) { it->second.subproc->terminate(); } - cv_stop.notify_all(); + request_stop(name, !loading); } - // wait until the monitoring thread finishes + // wait until the child is gone wait(lk, name, [](const server_model_meta & meta) { return meta.status == SERVER_MODEL_STATUS_UNLOADED || meta.status == SERVER_MODEL_STATUS_DOWNLOADED; @@ -1311,8 +1390,7 @@ bool server_models::remove(const std::string & name) { // re-find after wait - load_models() may have erased the entry during the wait it = mapping.find(name); if (it == mapping.end()) { - // load_models() already joined the thread and erased the entry; - // we just need to clean up the cached files on disk + // load_models() already erased the entry; we just need to clean up the cached files on disk lk.unlock(); bool ok = common_download_remove(name); SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "partial"); @@ -1320,11 +1398,6 @@ bool server_models::remove(const std::string & name) { return true; } - // join before erasing - thread no longer acquires this mutex - if (it->second.th.joinable()) { - it->second.th.join(); - } - // remove from disk (best-effort: cancelled downloads may have no cached files) bool ok = common_download_remove(name); mapping.erase(name); @@ -1539,7 +1612,7 @@ void server_models::handle_child_state(const std::string & name, const std::stri std::lock_guard lk(mutex); auto it = mapping.find(name); if (it != mapping.end()) { - return it->second.subproc->request_exit(); + return it->second.request_exit(); } }; if (result == "download_finished") { diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 7f6c26b35..90161bf34 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -107,27 +108,29 @@ struct server_model_meta { }; struct server_models_routes; -struct server_subproc; // defined in server-models.cpp struct server_lru_sched; // defined in server-models.cpp +struct server_monitor; // defined in server-models.cpp struct server_models { friend struct server_models_routes; friend struct server_lru_sched; + friend struct server_monitor; private: struct instance_t { - std::shared_ptr subproc; // shared between main thread and monitoring thread - std::thread th; + std::shared_ptr subproc; // shared with the monitor thread server_model_meta meta; int req_count = 0; // number of active proxy requests + + // ask the child to exit (it handles the command on its stdin, see server_child::setup) + void request_exit() const; }; std::mutex mutex; std::condition_variable cv; std::map mapping; - // for stopping models - std::condition_variable cv_stop; + // models asked to stop, still counted as running until the monitor records their exit std::set stopping_models; // set to true while load_models() is executing a reload; load() will wait until clear @@ -216,9 +219,12 @@ private: // not thread-safe, caller must hold mutex void add_model(server_model_meta && meta); - // ask the monitoring thread to stop a running instance + // ask the monitor to stop a running instance; send_exit is false for a child that was already force-killed // not thread-safe, caller must hold mutex - void request_stop(const std::string & name); + void request_stop(const std::string & name, bool send_exit = true); + + // called by the monitor once a child exited and was reaped + void on_child_exit(const std::string & name, const std::shared_ptr & proc, server_child_mode mode, int exit_code); // notify SSE clients void notify_sse(const std::string & event, const std::string & model_id, const json & data = nullptr); @@ -297,12 +303,16 @@ public: // handle message sent from server_child::notify_to_router() // raw input must starts with CMD_CHILD_TO_ROUTER_STATE, followed by a JSON string - // this function is not thread-safe, must be called from instance's monitoring thread + // called from the monitor thread // payload per state: // state = loading -> payload = {} (TODO: add progress info) // state = ready -> payload = model_info (json), or {} if wakeup from sleeping // state = sleeping -> payload = {} void handle_child_state(const std::string & name, const std::string & raw_input); + +private: + // one thread watching every child; keep last, the destructor joins the thread + std::unique_ptr monitor; }; struct server_child { From d3146f2b56c2db4711ac8391871c9e529d1946d7 Mon Sep 17 00:00:00 2001 From: Mendy Berger <12537668+MendyBerger@users.noreply.github.com> Date: Fri, 11 Sep 2026 21:47:29 -0400 Subject: [PATCH 21/35] ggml-webgpu: Update to a recent version of Dawn (#28683) * ggml-webgpu: Update to a recent version of Dawn * No module scanning * Accept review suggestion to update comment Co-authored-by: Masashi Yoshimura --------- Co-authored-by: Masashi Yoshimura --- .github/workflows/build-self-hosted.yml | 8 ++++---- .github/workflows/build-wasm.yml | 2 +- .github/workflows/build-webgpu.yml | 8 ++++---- docs/build.md | 2 +- ggml/src/ggml-webgpu/CMakeLists.txt | 6 ++++++ ggml/src/ggml-webgpu/ggml-webgpu.cpp | 21 +++++++++++---------- 6 files changed, 27 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml index 02a38466f..c7f992540 100644 --- a/.github/workflows/build-self-hosted.yml +++ b/.github/workflows/build-self-hosted.yml @@ -160,10 +160,10 @@ jobs: - name: Dawn Dependency id: dawn-depends run: | - DAWN_VERSION="v20260317.182325" + DAWN_VERSION="v20260908.214631" DAWN_OWNER="google" DAWN_REPO="dawn" - DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-ubuntu-latest-Release" + DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-ubuntu-latest-Release" echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" curl -L -o artifact.tar.gz \ "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" @@ -246,10 +246,10 @@ jobs: - name: Dawn Dependency id: dawn-depends run: | - DAWN_VERSION="v20260317.182325" + DAWN_VERSION="v20260908.214631" DAWN_OWNER="google" DAWN_REPO="dawn" - DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-macos-latest-Release" + DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-macos-latest-Release" echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" curl -L -o artifact.tar.gz \ "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 81b943df7..5a3166ce6 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -68,7 +68,7 @@ jobs: - name: Fetch emdawnwebgpu run: | - DAWN_TAG="v20260317.182325" + DAWN_TAG="v20260908.214631" EMDAWN_PKG="emdawnwebgpu_pkg-${DAWN_TAG}.zip" echo "Downloading ${EMDAWN_PKG}" curl -L -o emdawn.zip \ diff --git a/.github/workflows/build-webgpu.yml b/.github/workflows/build-webgpu.yml index 8277abcc4..ec582ff27 100644 --- a/.github/workflows/build-webgpu.yml +++ b/.github/workflows/build-webgpu.yml @@ -77,10 +77,10 @@ jobs: - name: Dawn Dependency id: dawn-depends run: | - DAWN_VERSION="v20260317.182325" + DAWN_VERSION="v20260908.214631" DAWN_OWNER="google" DAWN_REPO="dawn" - DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-macos-latest-Release" + DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-macos-latest-Release" echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" curl -L -o artifact.tar.gz \ "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" @@ -147,10 +147,10 @@ jobs: id: dawn-depends run: | sudo apt-get install -y libxrandr-dev libxinerama-dev libxcursor-dev mesa-common-dev libx11-xcb-dev libxi-dev - DAWN_VERSION="v20260317.182325" + DAWN_VERSION="v20260908.214631" DAWN_OWNER="google" DAWN_REPO="dawn" - DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-ubuntu-latest-Release" + DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-ubuntu-latest-Release" echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" curl -L -o artifact.tar.gz \ "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" diff --git a/docs/build.md b/docs/build.md index 28dcbc2e5..70fc17af2 100644 --- a/docs/build.md +++ b/docs/build.md @@ -806,7 +806,7 @@ To read documentation for how to build on Android, [click here](./android.md) ## WebGPU -The WebGPU backend relies on [Dawn](https://dawn.googlesource.com/dawn). Follow the instructions [here](https://dawn.googlesource.com/dawn/+/refs/heads/main/docs/quickstart-cmake.md) to install Dawn locally so that llama.cpp can find it using CMake. The current implementation is up-to-date with Dawn commit `18eb229`. +The WebGPU backend relies on [Dawn](https://dawn.googlesource.com/dawn). Follow the instructions [here](https://dawn.googlesource.com/dawn/+/refs/heads/main/docs/quickstart-cmake.md) to install Dawn locally so that llama.cpp can find it using CMake. The current implementation is up-to-date with Dawn commit `94c3c9c`. In the llama.cpp directory, build with CMake: diff --git a/ggml/src/ggml-webgpu/CMakeLists.txt b/ggml/src/ggml-webgpu/CMakeLists.txt index 1503a1ef8..2eacca7f2 100644 --- a/ggml/src/ggml-webgpu/CMakeLists.txt +++ b/ggml/src/ggml-webgpu/CMakeLists.txt @@ -39,6 +39,12 @@ ggml_add_backend_library(ggml-webgpu add_dependencies(ggml-webgpu generate_shaders) +# Dawn needs C++20 (https://dawn.googlesource.com/dawn/+/refs/heads/main/docs/quickstart-cmake.md#prerequisites) +target_compile_features(ggml-webgpu PRIVATE cxx_std_20) + +# Disable C++20 module scanning since emscan-deps fails to find webgpu_cpp.h +set_target_properties(ggml-webgpu PROPERTIES CXX_SCAN_FOR_MODULES OFF) + if(EMSCRIPTEN) set(EMDAWNWEBGPU_DIR "" CACHE PATH "Path to emdawnwebgpu_pkg") diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp index f06a9c872..13db0b856 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp @@ -4006,16 +4006,17 @@ static void ggml_backend_webgpu_request_adapter(wgpu::Instance & instance, wgpu: options.nextInChain = &adapterTogglesDesc; #endif - instance.WaitAny(instance.RequestAdapter( - &options, wgpu::CallbackMode::AllowSpontaneous, - [&adapter](wgpu::RequestAdapterStatus status, wgpu::Adapter _adapter, const char * message) { - if (status != wgpu::RequestAdapterStatus::Success) { - GGML_LOG_ERROR("ggml_webgpu: Failed to get an adapter: %s\n", message); - return; - } - adapter = std::move(_adapter); - }), - UINT64_MAX); + instance.WaitAny( + instance.RequestAdapter( + &options, wgpu::CallbackMode::AllowSpontaneous, + [&adapter](wgpu::RequestAdapterStatus status, wgpu::Adapter _adapter, wgpu::StringView message) { + if (status != wgpu::RequestAdapterStatus::Success) { + GGML_LOG_ERROR("ggml_webgpu: Failed to get an adapter: %s\n", std::string(message).c_str()); + return; + } + adapter = std::move(_adapter); + }), + UINT64_MAX); } static void create_webgpu_device(ggml_backend_webgpu_reg_context * ctx) { From eafe15a5e3d87dd68ae33acf6a7cbd9415a0ac5e Mon Sep 17 00:00:00 2001 From: Max Krasnyansky Date: Fri, 11 Sep 2026 20:46:51 -0700 Subject: [PATCH 22/35] hexagon: support for multi-device model split (aka row-split) (#28589) * hex-row-split: add support for multi-device row spliting Co-authored-by: Max Krasnyansky * hex-mdev: add work splitting to fused kernels * hex-mdev: use mdev_ prefix for all multi-device state * hex-mdev: make device configuration more expressive to support device groups * hex-mdev: fix mdev session init * hex-mdev: fused nx (2x,3x) matmuls must update row counts for each w/o * hex-mdev: fix MUL_MAT work partitioning bugs introduced by mdev * hex-cont: fix crashes with new tests due to wrong striding * hex-mdev: move fences after l2flushes * hex-cont: fix work splitting for mnpu -- align chunks to cachelines * hex-mdev: fix CPY tests with multi-dev * hex-mmid: fix work partitioning with mnpu * hex-mm: fix test failures with mdev * hex-binary: fix work partitioning for mdev * hex-argsort: fix mdev partitioning * hex-mdev: fix work partitioning and general updates for all simple ops * hex-fa: fix mdev work splitting issues * hex-mdev: fixing more failing ops test * hex-mdev: update the rest of the ops * hex-mdev: refactor all mdev splitting logic to be contained within if (mdev_count > 1) {...} * hex-mdev: fix macros * hex-mdev: simplify session flush logic * hex-sync: fix recursion in session flush * hex-mdev: factor out fence buffer and allocator * hex-fence: make fence allocation more robust with reserved slots for mdev * hex-mdev: keep all mdev state in htp_mdev_group * hex-mdev: further cleanup mdev group handling at the host * hex-mdev: update group idx in the opbatch before serializing * hex-batch: remove separate op_pending and use batch_req/rsp_seq * hex-async: workaround another missing tensor_init in ggml-meta * hex-fence: cleanup and robustify fences and error handling in multi-device scenarios * hex-ar: improve ALLREDUCE error handling * hex-async: robust error handling for op_cpy_fence * hex-async: use seq0 from allreduce context to allocate fence_seq * hex-mdev: fix remaining issues with fence and barrier clearing in CPY_FENCE * hex-misc: realign macros and fix misplaces trace events * hex-misc: align macros * hex-mdev: fix unclone buffer re-entrancy * hex-glu: fix mdev partitioning logic * hex-mdev: make buffer uncloning/cleanup work with tensor-split scenarios * hex-mdev: tighten up the can_split check in act-ops * hex-mdev: factor out common bits of the partitioning logic * hex-mm: minor realignment of the macros * hex-bufs: fix incorrectly placed assert for MAX_BUFS * hex-pad: tighten up gating checks for PAD * hex-kparams: make sure all kernels properly use kparams->n_threads * hex-docs: update user and developer docs with new features and detailed guide for ops development * hex-scripts: update run script to properly parse dev groups * hex-misc: formatting * hex-sess: minor cleanup for session init * hex-ar: fix vtcm size calc in allreduce kparams * hex-scripts: fix flake8 warnings * hex-rope: update ROPE to support mdev work split * hex-ops: remove redunant checks and minor reformat * hex-dev-guide: update dev-guide to avoid redundant null checks * hex-async: improve event_wait, event_sync and fence implementations * hex-async: remove synchronous flush from event_sync * hex-async: symplify fence recovery protocol and make sync more robust * hex-async: futher simplify error recovery for fences * hex-err: return status instead of just -1 * hex-async: print all seq nums in hex * hex-async: make sure fences flush dirty ranges * hex-async: add dirty ranges merging to reduce fence flushes * hex-async: properly sync before freeing the event * hex-async: make sure fence owner session is not overriden * hex-async: more fence write order more robust * hex-async: make sure not to fuse ALLREDUCE+ADD if their dsts overlap * hex-fusion: cleanup redundant checks --------- Co-authored-by: Alexander Lu --- docs/backend/snapdragon/README.md | 141 +- docs/backend/snapdragon/developer.md | 360 ++++- ggml/src/ggml-hexagon/ggml-hexagon.cpp | 1163 ++++++++++++----- ggml/src/ggml-hexagon/htp-opnode.h | 6 + ggml/src/ggml-hexagon/htp/act-ops.c | 233 ++-- ggml/src/ggml-hexagon/htp/allreduce-ops.c | 113 +- ggml/src/ggml-hexagon/htp/allreduce-ops.h | 11 + ggml/src/ggml-hexagon/htp/argsort-ops.c | 50 +- ggml/src/ggml-hexagon/htp/binary-ops.c | 228 ++-- ggml/src/ggml-hexagon/htp/concat-ops.c | 81 +- ggml/src/ggml-hexagon/htp/cpy-ops.c | 476 ++++--- ggml/src/ggml-hexagon/htp/cumsum-ops.c | 105 +- ggml/src/ggml-hexagon/htp/diag-ops.c | 101 +- ggml/src/ggml-hexagon/htp/fill-ops.c | 69 +- ggml/src/ggml-hexagon/htp/flash-attn-ops.c | 84 +- ggml/src/ggml-hexagon/htp/flash-attn-ops.h | 1 + .../ggml-hexagon/htp/gated-delta-net-ops.c | 74 +- ggml/src/ggml-hexagon/htp/get-rows-ops.c | 62 +- ggml/src/ggml-hexagon/htp/hex-common.h | 9 + ggml/src/ggml-hexagon/htp/hex-utils.h | 1 - ggml/src/ggml-hexagon/htp/hmx-utils.h | 14 +- ggml/src/ggml-hexagon/htp/htp-ctx.h | 51 +- ggml/src/ggml-hexagon/htp/htp-fence.h | 89 ++ ggml/src/ggml-hexagon/htp/htp-ops.h | 23 +- ggml/src/ggml-hexagon/htp/htp-tensor.c | 137 +- ggml/src/ggml-hexagon/htp/htp-tensor.h | 109 ++ ggml/src/ggml-hexagon/htp/hvx-arith.h | 274 ++-- ggml/src/ggml-hexagon/htp/hvx-div.h | 94 +- ggml/src/ggml-hexagon/htp/hvx-inverse.h | 46 +- ggml/src/ggml-hexagon/htp/hvx-scale.h | 44 +- ggml/src/ggml-hexagon/htp/hvx-sigmoid.h | 80 +- ggml/src/ggml-hexagon/htp/im2col-ops.c | 105 +- ggml/src/ggml-hexagon/htp/main.c | 103 +- ggml/src/ggml-hexagon/htp/matmul-ops.c | 371 ++++-- ggml/src/ggml-hexagon/htp/pad-ops.c | 159 +-- ggml/src/ggml-hexagon/htp/repeat-ops.c | 49 +- ggml/src/ggml-hexagon/htp/rope-ops.c | 41 +- ggml/src/ggml-hexagon/htp/set-rows-ops.c | 54 +- ggml/src/ggml-hexagon/htp/softmax-ops.c | 53 +- ggml/src/ggml-hexagon/htp/solve-tri-ops.c | 92 +- ggml/src/ggml-hexagon/htp/ssm-conv.c | 169 +-- ggml/src/ggml-hexagon/htp/sum-rows-ops.c | 80 +- ggml/src/ggml-hexagon/htp/unary-ops.c | 278 ++-- .../snapdragon/ggml-hexagon-align-macros.py | 296 +++++ scripts/snapdragon/run.py | 128 +- 45 files changed, 4352 insertions(+), 1955 deletions(-) create mode 100644 ggml/src/ggml-hexagon/htp/htp-fence.h create mode 100755 scripts/snapdragon/ggml-hexagon-align-macros.py diff --git a/docs/backend/snapdragon/README.md b/docs/backend/snapdragon/README.md index 391c8bf23..5d32a5877 100644 --- a/docs/backend/snapdragon/README.md +++ b/docs/backend/snapdragon/README.md @@ -188,7 +188,7 @@ llama_memory_breakdown_print: | - Host | 439 = Op test for MUL_MAT: ``` -~/src/llama.cpp$ ./scripts/snapdragon/run.py --target adb --hex-hostbuf 0 --devices HTP0:0 -- test-backend-ops -b HTP0:0 -o MUL_MAT +~/src/llama.cpp$ ./scripts/snapdragon/run.py --target adb --devices HTP0:0 -- test-backend-ops -b HTP0:0 -o MUL_MAT ... Backend 2/3: HTP0:0 Device description: Hexagon @@ -213,14 +213,109 @@ ggml-hex: new session: HTP0 : session-id 0 domain-id 3 uri file:///libggml-htp-v | llama 1B Q4_0 | 729.75 MiB | 1.24 B | HTP | 99 | 4 | 128 | 0 | tg64 | 51.54 ± 1.13 | ``` +## Multi-Device Execution Modes + +The Hexagon backend supports multiple execution and partitioning modes to accommodate different model sizes, memory +constraints, and single- or multi-NPU hardware topologies: + +### 1. Single-Device Mode with Dynamic Buffer Mapping + +Runs the model on a single NPU session (e.g. `HTP0` or `HTP0:0`). + +A single NPU session provides ~3.5GB of available virtual address space. For models larger than 3.5GB, the backend +automatically maps and unmaps weight buffers during graph execution. This allows large models to run on a single NPU +without manual configuration: + +```bash +./scripts/snapdragon/run.py --target adb --devices HTP0:0 -- \ + llama-cli -m models/Llama-3.2-3B-Instruct-Q4_0.gguf -ngl 99 -p "Hello" +``` + +### 2. Layer-Split Mode across Virtual Sessions (`HTP0,HTP1,...` or `HTP0:0,HTP0:1,...`) + +Partitions model layers at load time across multiple virtual sessions hosted on a single physical NPU. + +Each virtual session acts as an independent backend device from llama.cpp's perspective (similar to multiple GPUs). +Because layers are permanently distributed across sessions, each session's allocated weights remain within its private 3.5GB +address space window, eliminating runtime buffer re-mapping overhead. + +Here is an example of running the GPT-OSS-20B model on a Snapdragon device using 4 virtual sessions on a single NPU: + +```bash +./scripts/snapdragon/run.py --target adb \ + --devices HTP0:0,HTP0:1,HTP0:2,HTP0:3 -- \ + llama-cli --load-mode none -m /data/local/tmp/gguf/gpt-oss-20b-Q4_0.gguf -t 4 \ + --ctx-size 8192 --batch-size 128 -ctk q8_0 -ctv q8_0 -fa on -ngl 99 -no-cnv -f surfing.txt +``` + +Log output snippet: + +``` +... +llama_model_loader: - type f32: 289 tensors +llama_model_loader: - type q4_0: 96 tensors +llama_model_loader: - type q8_0: 2 tensors +llama_model_loader: - type mxfp4: 72 tensors +... +load_tensors: offloaded 25/25 layers to GPU +load_tensors: CPU model buffer size = 1182.09 MiB +load_tensors: HTP0:1 model buffer size = 2512.58 MiB +load_tensors: HTP0:3 model buffer size = 2093.83 MiB +load_tensors: HTP0:0 model buffer size = 2931.34 MiB +load_tensors: HTP0:2 model buffer size = 2512.58 MiB +... +llama_perf_context_print: prompt eval time = 3843.67 ms / 197 tokens ( 19.51 ms per token, 51.25 tokens per second) +llama_perf_context_print: eval time = 1686.13 ms / 31 runs ( 54.39 ms per token, 18.39 tokens per second) +llama_perf_context_print: total time = 6266.30 ms / 228 tokens +llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | +llama_memory_breakdown_print: | - HTP0:0 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - HTP0:1 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - HTP0:2 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - HTP0:3 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - Host | 1476 = 1208 + 105 + 162 | +``` + +### 3. Tensor-Split Mode across Physical Devices (`HTP0:0,HTP1:0,...`) + +Distributes model tensors across distinct physical NPU hardware cores using llama.cpp's tensor parallelism +(`--split-mode tensor`). + +Tensors are partitioned across physical NPUs for parallel execution (proportions are distributed equally by default without +needing an explicit `--tensor-split` option): + +```bash +./scripts/snapdragon/run.py --target adb \ + --devices HTP0:0,HTP1:0 -- \ + llama-cli -m models/Llama-3.2-3B-Instruct-Q4_0.gguf --split-mode tensor -ngl 99 -p "Hello" +``` + +### 4. Row-Split Multi-Device Mode via Device Grouping (`HTP0[0-1]`) + +Groups multiple physical NPU cores into a single logical device using bracket notation (`HTP0[0-1]` or `HTP0[0,1]`). + +Unlike host-level tensor-splitting, row-splitting is executed entirely inside the Hexagon backend: + +```bash +./scripts/snapdragon/run.py --target adb \ + --devices 'HTP0[0-1]' -- \ + llama-cli -m models/Llama-3.2-3B-Instruct-Q4_0.gguf -ngl 99 -p "Hello" +``` + +You can also combine row-splitting with layer-splitting across multiple grouped devices (e.g. `--devices 'HTP0[0-1],HTP1[2-3]'` +on 4 physical NPUs, or `--devices 'HTP0[0-1:0],HTP1[0-1:1]'` on 2 physical NPUs using virtual sessions 0 and 1). + ## Environment variables - `GGML_HEXAGON_DEVICES` (default: not set, defaults to HTP0 session) - Controls which NPU devices and sessions to allocate. Can be configured as: - - A single integer `N`: Allocates `N` sessions named `HTP0`, `HTP1`, ..., `HTP` (behaves identically to `GGML_HEXAGON_NDEV=N`). - - A comma-separated list of device names in `HTP:` format (or legacy `HTP` format). For example, `HTP0:0,HTP0:1` creates two virtual - sessions on the first physical NPU (useful for memory limits). `HTP0:0,HTP1:0` allocates one session on each of the two physical NPUs - on a dual-NPU device. + Controls which NPU devices and sessions to allocate. Configurable via `--devices` in `run.py`: + - `N` (single integer): Allocates `N` virtual sessions named `HTP0`, `HTP1`, ..., `HTP` on physical NPU 0. + - `HTP:,...`: Comma-separated list of individual devices specifying physical and virtual index: + - `HTP0:0,HTP0:1`: Two virtual sessions on physical NPU 0 (layer-split on single NPU). + - `HTP0:0,HTP1:0`: One session on physical NPU 0 and one on physical NPU 1 (tensor-split across physical cores). + - `HTP[]`: Device grouping syntax for row-split multi-device execution: + - `HTP0[0-1]`: A single logical device `HTP0` that groups physical cores 0 and 1. + - `HTP0[0-1],HTP1[2-3]`: Two layer-split devices across 4 physical NPUs (cores 0-1 and 2-3). + - `HTP0[0-1:0],HTP1[0-1:1]`: Two layer-split devices across 2 physical NPUs using virtual sessions 0 and 1. - `GGML_HEXAGON_NDEV` (deprecated) Replaced by `GGML_HEXAGON_DEVICES`. Controls the number of virtual sessions to allocate on physical NPU `0`. @@ -229,9 +324,8 @@ ggml-hex: new session: HTP0 : session-id 0 domain-id 3 uri file:///libggml-htp-v - `GGML_HEXAGON_NHVX=0` Controls the number of HVX hardware threads to use. The default is all (actual number varies depending on the hardware version). -- `GGML_HEXAGON_HOSTBUF=1` - Controls whether the Hexagon backend allocates host buffers. By default, all buffers except for REPACK are host buffers. - This option is required for testing Ops that require REPACK buffers (MUL_MAT and MUL_MAT_ID). +- `GGML_HEXAGON_HOSTBUF=1` (default: 0, disabled) + Enables allocating host buffers for debugging. By default, host buffers are disabled. - `GGML_HEXAGON_VERBOSE=1` Enables verbose logging of Ops from the backend. Example output: @@ -246,23 +340,26 @@ ggml-hex: new session: HTP0 : session-id 0 domain-id 3 uri file:///libggml-htp-v ``` - `GGML_HEXAGON_PROFILE=1` - Enables Op profiling: + Enables Op profiling (configurable via `--hex-profile` in `run.py`): - - `1` Basic profile with per-op `usecs` and `cycles` counters - - `2` Extended profile with per-op `usecs`, `cycles` and default PMU counter data - - `0x1,...,0x8` Extended profile with per-op `usecs`, `cycles` and custom PMU counter data + - `1`: Basic profile with per-op `usecs` and `cycles` counters + - `2`: Extended profile with per-op `usecs`, `cycles` and default PMU counter data + - `0x1,...,0x8`: Extended profile with per-op `usecs`, `cycles` and custom PMU counter data - The logging output can be either saved into a file for post-processing or it can be piped directly into the post-processing tool - to generate the report. - Examples: + The logging output can be saved to a file or piped directly into the post-processing script: - `GGML_HEXAGON_PROFILE=1 ./scripts/snapdragon/run.py --target adb -- llama-cli ... |& ./scripts/snapdragon/ggml-hexagon-profile.py -` + ```bash + ./scripts/snapdragon/run.py --target adb --hex-profile 1 -- llama-cli ... |& \ + ./scripts/snapdragon/ggml-hexagon-profile.py - + ``` - `GGML_HEXAGON_OPFILTER=regex` - Allows filtering (disabling) Ops that match the regex pattern: + Filters (disables) Ops matching the regex pattern (configurable via `--hex-opfilter` in `run.py`): - Examples: - - `GGML_HEXAGON_OPFILTER="FLASH_ATTN_EXT" ./scripts/snapdragon/run.py --target adb -- llama-cli ...` - Disable Flash Attention on Hexagon (falls back to CPU or GPU) - `GGML_HEXAGON_OPFILTER="ADD\|SUB" ./scripts/snapdragon/run.py --target adb -- llama-cli ...` - Disable ADD and SUB on Hexagon (fall back to CPU or GPU) + ```bash + # Disable Flash Attention on Hexagon (falls back to CPU or GPU) + ./scripts/snapdragon/run.py --target adb --hex-opfilter "FLASH_ATTN_EXT" -- llama-cli ... + # Disable ADD and SUB on Hexagon (fall back to CPU or GPU) + ./scripts/snapdragon/run.py --target adb --hex-opfilter "ADD|SUB" -- llama-cli ... + ``` diff --git a/docs/backend/snapdragon/developer.md b/docs/backend/snapdragon/developer.md index d7d9f2a27..633643c16 100644 --- a/docs/backend/snapdragon/developer.md +++ b/docs/backend/snapdragon/developer.md @@ -2,16 +2,16 @@ ## Backend libraries -The Hexagon backend consist of two parts: +The Hexagon backend consists of two parts: - `libggml-hexagon` - This is the regular CPU-side GGML backend library, either shared or statically linked + This is the regular CPU-side GGML backend library, either shared or statically linked. - `libggml-htp-vNN` This is the NPU-side (HTP stands for Hexagon Tensor Processor) shared library that contains the Op dispatcher and kernels. The correct library is selected automatically at runtime based on the HW version. -Here is an example of the build artifacts +Here is an example of the build artifacts: ``` ~/src/llama.cpp$ ls -l pkg-adb/llama.cpp/lib/libggml* @@ -26,75 +26,307 @@ pkg-adb/llama.cpp/lib/libggml-htp-v81.so ## Memory buffers -Hexagon NPU backend takes advantage of the Snapdragon's unified memory model where all buffers are fully accessible by the CPU and GPU. -The NPU does have a dedicated tightly-coupled memory called VTCM but that memory is used only for intermediate data (e.g. dynamically -quantized tensors) or temporary data (chunks of the weight tensors fetched via DMA). - -Please note that currently the Hexagon backend does not implement SET/GET_ROWS Ops because there is no advantage in offloading those -to the NPU at this point. - -The backend does allocates non-host buffers for the tensors with datatypes that require repacking: Q4_0, Q8_0, MXFP4. -From the MMU perspective these buffers are still regular buffers (normal access by the CPU) they are marked as non-host simply to force -the repacking. +The Hexagon NPU backend takes advantage of Snapdragon unified memory where all DDR buffers are accessible by CPU, GPU, and NPU. +The NPU has dedicated tightly-coupled memory called VTCM (Vector Tightly-Coupled Memory). VTCM is used for intermediate data (such as +dynamically quantized activations) and streaming buffers (chunks of weight and activation tensors fetched via DMA). ## Large model handling -Hexagon NPU sessions (aka Process Domains (PD) in the Hexagon SDK) are limited to a maximum memory mapping window of around 3.5GB. +Hexagon NPU sessions have a 32-bit virtual address space window of around 3.5GB. In llama.cpp/GGML, each Hexagon session is mapped to a single GGML backend device (e.g., `HTP0:0`, `HTP0:1`, etc. when using `GGML_HEXAGON_DEVICES`, or `HTP0`, `HTP1` in legacy mode). -To support running models larger than 3.5GB on a single device, the Hexagon backend dynamically maps and unmaps execution buffers -during the graph execution cycle to stay within the Process Domain window. This enables large models to run successfully on a single -NPU device. +To support running models larger than 3.5GB on a single device, the Hexagon backend dynamically maps and unmaps buffers: +- Buffers are allocated in shared DDR (RPCMEM) via file descriptors (`fastrpc_mmap` using `FASTRPC_MAP_FD_DELAYED`). +- Pinned buffers (such as KV cache and active compute buffers) remain mapped throughout execution. +- Inactive weight buffers are dynamically mapped into the NPU session via `HAP_mmap()` during batch buffer preparation + (`prep_op_bufs()` in `htp/main.c`) and unmapped via `htp_iface_munmap()` when no longer needed by the active batch. +- This dynamic sliding window allows a single NPU session to execute models that exceed the 3.5GB window. -Alternatively, users can choose to use standard llama.cpp/GGML layer-splitting mode to partition and split the model across -multiple Hexagon devices or virtual sessions (which behave like multiple GPUs from the offload and splitting perspective). +Alternatively, users can partition and split the model across multiple virtual sessions or physical NPUs using layer-splitting, +tensor-splitting, or row-splitting modes. For user-facing execution modes and examples, see the +[Snapdragon user guide](README.md#multi-device-execution-modes). -Here is an example of running GPT-OSS-20B model on a Snapdragon device using 4 virtual sessions on a single NPU (physical index 0). +## Op and Kernel Development Guidelines + +Writing high-performance operators for Hexagon requires following specific guidelines. + +### DDR -> DMA -> VTCM Execution Pipeline + +- Strongly prefer the `DDR -> DMA -> VTCM -> compute (HVX/HMX) -> VTCM -> DMA -> DDR` data flow. +- Direct HVX reads/writes from/to DDR are less efficient and should only be used as a fallback. +- The DMA queue is a strict FIFO where operations must be pushed and popped in strict order. +- Follow the pipelined multi-buffering sequence properly (typically 2x to 16x buffering) so every push has a corresponding pop: + + 1. In the prologue, push initial DDR -> VTCM transfers to prime the pipeline. + 2. In the loop body, wait for buffer N via DMA pop, launch HVX/HMX compute on buffer N, push VTCM -> DDR writeback of result N, + and push DDR -> VTCM prefetch of buffer N+2. + 3. In the epilogue, pop all remaining in-flight transfers to drain the pipeline. + +- Because every push must be matched by a pop, `dma_queue_flush()` is not required when the pipeline sequence is followed + properly. Flushing is only used in rare exceptions where a batch of operations is pushed without individual pops. +- Use the DMA queue interface from [`dma-queue.h`](../../../ggml/src/ggml-hexagon/htp/dma-queue.h) + (`dma_queue_push_ddr_to_vtcm`, `dma_queue_pop`, `dma_queue_push_vtcm_to_ddr`). + See [`cumsum-ops.c`](../../../ggml/src/ggml-hexagon/htp/cumsum-ops.c) and + [`act-ops.c`](../../../ggml/src/ggml-hexagon/htp/act-ops.c) for reference implementations. + +### Avoid Scalar Reads and Writes to VTCM + +- Access VTCM data using DMA transfers or HVX/HMX vector instructions rather than scalar reads and writes. + +### Avoid Scalar Division in Inner Loops + +- Hexagon cores do not have hardware division instructions. +- For recurring divisions across iterations or threads, use `fastdiv` from + [`hex-fastdiv.h`](../../../ggml/src/ggml-hexagon/htp/hex-fastdiv.h) with precomputed divisors (such as + `octx->ctx->mdev.count_div` or `octx->n_threads_div`). +- Do not call `init_fastdiv_values()` for single-use divisions; use standard compiler division (`/`) instead. + +### Host-Side Precomputation via `kernel_params` + +- Precompute tensor shapes, strides, scale conversions, tiling layouts, and validation checks on the host CPU during graph + preparation in [`ggml-hexagon.cpp`](../../../ggml/src/ggml-hexagon/ggml-hexagon.cpp). +- Pack precomputed parameters into the operator's fixed `kernel_params` structure in `htp_op_node` (such as + `htp_mm_kernel_params`, `htp_unary_kernel_params`, `htp_fa_kernel_params`, `htp_get_rows_kernel_params`). +- The NPU executes directly using `octx->kernel_params` without redundant runtime metadata extraction or validation. +- **Strict Host-Kernel Alignment**: + - Verify that parameters calculated by the host CPU are strictly honored by the NPU kernel. + - Ensure the kernel does not ignore host-computed fields (for example, falling back to `octx->n_threads` instead of + using `kparams->n_threads`, or ignoring precomputed `tasks_per_thread` and chunk counts). + - Both human developers and coding agents must audit both sides of the interface: ensure fields populated in `kernel_params` + in [`ggml-hexagon.cpp`](../../../ggml/src/ggml-hexagon/ggml-hexagon.cpp) are actively and consistently utilized by the + corresponding operator entry point and worker threads in `htp/*-ops.c`. + +### Tracing Instrumentation + +- All kernels must include trace events for performance profiling and timeline visualization in Perfetto + ([`hex-profile.h`](../../../ggml/src/ggml-hexagon/htp/hex-profile.h)). +- Surround compute sections with `htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) info)` and + `htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) info)`. +- Use specific event types for major phases: + - `HTP_TRACE_EVT_HVX_COMP`: Vector compute execution. + - `HTP_TRACE_EVT_DMA`: DMA transfer wait or poll cycles. + - `HTP_TRACE_EVT_FENCE`: Multi-device fence barrier synchronization. + - `HTP_TRACE_EVT_L2FLUSH`: L2 cache cleaning operations. +- Pass meaningful progress metrics (such as row index, chunk index, or token index) in the 16-bit `info` parameter. + +### Work Queue and Threading + +- Distribute parallel work across NPU worker threads using the thread pool work queue: + + ```c + work_queue_run(ctx->work_queue, worker_func, &op_ctx, n_threads); + ``` + +- Keep worker functions independent and re-entrant. Worker threads should only operate on their designated chunk of rows or elements. + +### Avoid Redundant Defensive NULL Checks + +- Do not add defensive NULL checks or assertions for internal framework pointers or required graph operands and outputs. + Internal pointers include `ctx`, `octx`, local context structs like `*ctx`, `kparams`, and worker callback `data`. +- These pointers are architectural invariants during kernel execution and host-side graph preparation. + Graph compute receives allocated nodes with valid required `node->src[N]` and `node->data` pointers. +- Do not turn an invariant violation into an unsupported operation or missed fusion. + Checks such as `if (!octx || !octx->ctx)` clutter the code, obscure intent, and hide upstream errors. +- **Distinction**: `octx->src[N]` pointers *can* be NULL by design and must be checked when optional. + Examples include attention masks, optional bias or weights in fused kernels, and frequency factors. + +### Multiline Macro Formatting + +- Keep trailing backslashes in multiline `#define` macros cleanly aligned to a consistent column. +- Avoid trailing whitespace after macro backslashes. +- Use [`scripts/snapdragon/ggml-hexagon-align-macros.py`](../../../scripts/snapdragon/ggml-hexagon-align-macros.py) to inspect, diff, + or automatically align macro definitions across Hexagon kernel sources: + + ```bash + # Check for misaligned macros + python3 scripts/snapdragon/ggml-hexagon-align-macros.py ggml/src/ggml-hexagon/htp/ + + # Fix misaligned macros in-place + python3 scripts/snapdragon/ggml-hexagon-align-macros.py --fix ggml/src/ggml-hexagon/htp/ + ``` + +## Multi-Device Partitioning (mdev) + +Multi-device (mdev) mode enables row-level tensor parallel execution across multiple physical NPU cores or virtual NPU +sessions. + +### 128-Byte Cache Line Alignment + +- Shared tensor buffers reside in DDR (RPCMEM) with a 128-byte cache line granularity + (`HEX_L2_LINE_SIZE` = 128 bytes, `HTP_TENSOR_MDEV_LINE_SIZE`). +- **Rule**: Multi-device work partitions must align destination write regions to 128-byte cache line boundaries so distinct + devices never share or overwrite the same cache line. + +### Partitioning Helpers in `htp-tensor.h` + +Common partitioning logic is factored into reusable inline helpers in +[`htp-tensor.h`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h): + +1. [`htp_tensor_mdev_rows_per_chunk`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L67): + Determines the minimum number of rows per chunk so that the chunk byte size is a multiple of 128 bytes: + + ``` + rows_per_chunk = 128 / hex_gcd_u32(row_size, 128) + ``` + + If row stride `nb[1]` is already a multiple of 128 bytes, `rows_per_chunk = 1`. + Returns `false` if the tensor cannot be safely row-partitioned (such as unaligned base pointer, permuted layout, + or non-128-byte aligned outer strides). + +2. [`htp_tensor_mdev_partition`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L94): + Calculates the per-device work range `struct htp_tensor_mdev_range { uint32_t start; uint32_t count; }` given + `total_units`, `units_per_chunk`, `mdev_idx`, `mdev_count`, and the precomputed `mdev_count_div`. + Handles chunk distribution across devices, assigns remainder units to the last device, and automatically triggers + single-device fallback when partitioning is unsafe. + +### Row-Partitioned Operators + +For row-wise operators +(such as activations in [`act-ops.c`](../../../ggml/src/ggml-hexagon/htp/act-ops.c), +binary ops in [`binary-ops.c`](../../../ggml/src/ggml-hexagon/htp/binary-ops.c), +unary ops in [`unary-ops.c`](../../../ggml/src/ggml-hexagon/htp/unary-ops.c), and +sameshape copies in [`cpy-ops.c`](../../../ggml/src/ggml-hexagon/htp/cpy-ops.c)): + +```c +const uint32_t total_rows = ne01 * ne02 * ne03; +const size_t dst_row_size = dst->ne[0] * elem_size; + +uint32_t row_start = 0; +uint32_t nrows = total_rows; + +if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, elem_size, (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; +} + +if (nrows == 0) { + return HTP_STATUS_OK; +} +``` + +### Element-Partitioned Operators + +For flat element-wise operations (such as reshape copies in +[`cpy-ops.c`](../../../ggml/src/ggml-hexagon/htp/cpy-ops.c)): +- Partition total linear elements N = ne0 * ne1 * ne2 * ne3 in 128-byte cache line chunks (`elems_per_line = (elem_size == 4) ? 32 : 64`). +- Requires strict 1D contiguity: + [`htp_tensor_is_contiguous(dst, elem_size)`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L28) + and 128-byte aligned destination pointer + [`htp_tensor_mdev_data_aligned(dst)`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L47). +- If contiguous and aligned, pass `elems_per_line` to + [`htp_tensor_mdev_partition`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L94); + otherwise pass 0 to trigger Device 0 fallback. + +### Single-Device Fallback (Device 0) + +- Fallback to Device 0 (`mdev.idx == 0`) when partitioning would cause cache line tearing or when work cannot be evenly distributed. +- Triggers: + 1. Destination tensor cannot be safely partitioned (`rows_per_chunk == 0` or non-contiguous/unaligned buffer). + 2. Total aligned chunks < `mdev_count`. +- Device 0 processes the entire tensor `[0, total_units)`. +- Devices 1 ... N-1 receive `count = 0` and return `HTP_STATUS_OK` immediately. + +### Flatten Outer Dimensions Globally + +- **Never partition solely on `ne01` (dimension 1).** +- Partitioning only on `ne01` repeats the device boundary across every 2D slice (`ne02`, `ne03`). If each 2D slice is small, + false sharing occurs repeatedly throughout the tensor. +- Always flatten outer dimensions globally: `total_rows = ne01 * ne02 * ne03` and partition once across the combined row space. + +### Stateless Starting Coordinates + +- Do not use incremental state variables across slices that assume the thread or device starts at index 0. +- Precompute starting multidimensional coordinates at `r = row_start` (or `e = elem_start`) once using `fastdiv`. +- In inner loops, step base pointers directly (`ptr += stride`) or reset/wrap coordinates explicitly (`if (++i01 == ne01) { ... }`). + +### Clean Range Encapsulation + +- Initialize single-device default ranges at declaration: + + ```c + uint32_t row_start = 0; + uint32_t nrows = total_rows; + ``` + +- Encapsulate all multi-device logic inside `if (octx->ctx->mdev.count > 1)`. If the block is omitted or compiled out, + the operator runs standard single-device execution untouched. +- Do not propagate `mdev_` prefixes to worker functions or context structs. Worker threads are device-agnostic and + should only receive standard range parameters (`ctx.row_start`, `ctx.nrows`). +- In worker threads, calculate row intervals using standard arithmetic: + + ```c + const uint32_t ir0 = ctx->row_start + dr * ith; + const uint32_t ir1 = MIN(ir0 + dr, ctx->row_start + ctx->nrows); + ``` + + In single-device mode (`row_start == 0`), this naturally simplifies to `dr * ith` and `MIN(ir0 + dr, ctx->nrows)` with zero overhead. + +## Multi-Device Synchronization + +Multi-device execution synchronizes worker sessions across devices using explicit barriers and tensor cache flushing. + +### Synchronization Fence Protocol + +Multi-device execution synchronizes worker sessions through atomic fence slots and barriers defined in +[`htp-fence.h`](../../../ggml/src/ggml-hexagon/htp/htp-fence.h): ``` -~/src/llama.cpp$ ./scripts/snapdragon/run.py --target adb --devices HTP0:0,HTP0:1,HTP0:2,HTP0:3 -- llama-cli --load-mode none -m /data/local/tmp/gguf/gpt-oss-20b-Q4_0.gguf -t 4 --ctx-size 8192 --batch-size 128 -ctk q8_0 -ctv q8_0 -fa on -ngl 99 -no-cnv -f surfing.txt -... -llama_model_loader: - type f32: 289 tensors -llama_model_loader: - type q4_0: 96 tensors -llama_model_loader: - type q8_0: 2 tensors -llama_model_loader: - type mxfp4: 72 tensors -... -load_tensors: offloaded 25/25 layers to GPU -load_tensors: CPU model buffer size = 1182.09 MiB -load_tensors: HTP0:1 model buffer size = 2512.58 MiB -load_tensors: HTP0:3 model buffer size = 2093.83 MiB -load_tensors: HTP0:0 model buffer size = 2931.34 MiB -load_tensors: HTP0:2 model buffer size = 2512.58 MiB -... -llama_context: n_ctx_per_seq (8192) < n_ctx_train (131072) -- the full capacity of the model will not be utilized -llama_context: CPU output buffer size = 0.77 MiB -llama_kv_cache_iswa: creating non-SWA KV cache, size = 8192 cells -llama_kv_cache: HTP0:1 KV buffer size = 25.50 MiB -llama_kv_cache: HTP0:3 KV buffer size = 25.50 MiB -llama_kv_cache: HTP0:0 KV buffer size = 25.50 MiB -llama_kv_cache: HTP0:2 KV buffer size = 25.50 MiB -llama_kv_cache: size = 102.00 MiB ( 8192 cells, 12 layers, 1/1 seqs), K (q8_0): 51.00 MiB, V (q8_0): 51.00 MiB -llama_kv_cache_iswa: creating SWA KV cache, size = 256 cells -llama_kv_cache: HTP0:1 KV buffer size = 0.80 MiB -llama_kv_cache: HTP0:3 KV buffer size = 0.53 MiB -llama_kv_cache: HTP0:0 KV buffer size = 1.06 MiB -llama_kv_cache: HTP0:2 KV buffer size = 0.80 MiB -llama_kv_cache: size = 3.19 MiB ( 256 cells, 12 layers, 1/1 seqs), K (q8_0): 1.59 MiB, V (q8_0): 1.59 MiB -llama_context: HTP0:0 compute buffer size = 16.06 MiB -llama_context: HTP0:1 compute buffer size = 16.06 MiB -llama_context: HTP0:2 compute buffer size = 16.06 MiB -llama_context: HTP0:3 compute buffer size = 16.06 MiB -llama_context: CPU compute buffer size = 98.19 MiB -... -llama_perf_context_print: prompt eval time = 3843.67 ms / 197 tokens ( 19.51 ms per token, 51.25 tokens per second) -llama_perf_context_print: eval time = 1686.13 ms / 31 runs ( 54.39 ms per token, 18.39 tokens per second) -llama_perf_context_print: total time = 6266.30 ms / 228 tokens -llama_perf_context_print: graphs reused = 30 -llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | -llama_memory_breakdown_print: | - HTP0:0 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | -llama_memory_breakdown_print: | - HTP0:1 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | -llama_memory_breakdown_print: | - HTP0:2 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | -llama_memory_breakdown_print: | - HTP0:3 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | -llama_memory_breakdown_print: | - Host | 1476 = 1208 + 105 + 162 | +[NPU Session 0] [NPU Session 1] + | | + (Input Prep) (Input Prep) + | | + Pre-Op Barrier ----------------------------- Pre-Op Barrier + (mdev_sync_fence) (mdev_sync_fence) + | | + Kernel Execution Kernel Execution + (Output Slice 0) (Output Slice 1) + | | + Tensor Cache Flush Tensor Cache Flush + (htp_tensor_flush_all) (htp_tensor_flush_all) + | | + Post-Op/Batch Barrier ---------------------- Post-Op/Batch Barrier + (htp_mdev_group_barrier) (htp_mdev_group_barrier) + | | + Return Response to Host Return Response to Host ``` + +### Atomic Fence Slots and Cache Invalidation + +- Fence synchronization operates on dedicated RPCMEM shared memory mapped across all participating sessions (`ctx->mdev.fence_base`). +- Each device owns a dedicated 128-byte cache-line aligned fence slot: + + ```c + atomic_uint * my_fence = htp_mdev_fence_slot(fence_base, mdev_idx); + ``` + +- **Writing to fence ([`htp_fence_write`](../../../ggml/src/ggml-hexagon/htp/htp-fence.h#L18))**: + Stores `seq` and `status`, issues a `syncht` thread synchronization barrier, and flushes/invalidates the line + using `Q6_dccleaninva_A(fence)`. +- **Reading from peer fence ([`htp_fence_read`](../../../ggml/src/ggml-hexagon/htp/htp-fence.h#L26))**: + Executes `Q6_dccleaninva_A(fence)` and `syncht` before reading atomic values to ensure fresh data from DDR. + +### Deterministic Monotonic Sequence Numbers + +- Barrier fences use monotonically increasing sequence numbers: + + ```c + const uint32_t seq = ++ctx->mdev.fence_seq; + ``` + +- Comparing sequence numbers with signed arithmetic `(int32_t)(peer_seq - seq) >= 0` prevents race conditions or + misaligned barrier arrivals across iterations. +- If any peer reports an error status (`peer_status > HTP_STATUS_OK`), the barrier propagates the error and unblocks immediately. + +### Tensor Cache Flush and Pipeline Completion + +- In the kernel, ensure all pushed DMA operations have been popped in strict FIFO order to drain the queue. +- Use [`htp_tensor_flush_all()`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h) to flush specific dirty tensors back to DDR: + - [`htp_tensor_flush_all()`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h) flushes only modified tensor address ranges, + ensuring peer devices and the host CPU observe consistent data in DDR. +- Never signal completion before all DMA transfers are drained and dirty tensor flushes have completed. + diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 112e9bae6..ec7801388 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -66,7 +66,6 @@ using u32vec = std::vector; #define GGML_HEXAGON_MAX_SESSIONS 16 -#define GGML_HEXAGON_FENCE_BUFFER_SIZE 8192 #define GGML_HEXAGON_FENCE_SLOT_SIZE 128 struct ggml_hexagon_device_config { @@ -75,6 +74,8 @@ struct ggml_hexagon_device_config { int domain_id = 0; std::string domain_name; std::string name; + + std::vector mdev_group; }; static ggml_hexagon_device_config opt_device_configs[GGML_HEXAGON_MAX_SESSIONS]; @@ -350,27 +351,48 @@ struct ggml_hexagon_tensor_extra { }; static inline bool ggml_hexagon_tensor_is_fuseable(const struct ggml_tensor * t) { - if (!t || !t->extra) return false; + if (!t->extra) return false; auto extra = (const struct ggml_hexagon_tensor_extra *) t->extra; return (extra->flags & GGML_HEXAGON_TENSOR_FUSEABLE) != 0; } +static inline bool ggml_hexagon_tensors_overlap(const struct ggml_tensor * a, const struct ggml_tensor * b) { + const uintptr_t a0 = (uintptr_t) a->data; + const uintptr_t b0 = (uintptr_t) b->data; + const uintptr_t a1 = a0 + ggml_nbytes(a); + const uintptr_t b1 = b0 + ggml_nbytes(b); + + return a0 < b1 && b0 < a1; +} + struct htp_opnode; struct ggml_hexagon_opbatch; struct ggml_hexagon_opqueue; struct ggml_hexagon_shared_buffer; +struct ggml_hexagon_fence_buffer; struct ggml_hexagon_session; +struct ggml_backend_hexagon_device_context; + +struct ggml_hexagon_mdev_group { + uint32_t idx = 0; + uint32_t count = 1; + std::vector> sessions; +}; struct ggml_backend_hexagon_comm_context { std::vector backends; size_t n_backends = 0; - uint32_t fence_seq = 0; + volatile uint32_t * fence_slots[GGML_HEXAGON_MAX_SESSIONS] = {}; + ggml_tensor fence_tensors[GGML_HEXAGON_MAX_SESSIONS] = {}; }; struct ggml_hexagon_event { - ggml_hexagon_session * sess = nullptr; - uint64_t seq = 0; + ggml_hexagon_session * sess = nullptr; + ggml_hexagon_session * fence_sess = nullptr; + volatile uint32_t * fence_slot = nullptr; + ggml_tensor fence_tensor = {}; + uint32_t seq = 0; }; struct ggml_hexagon_session { @@ -387,12 +409,12 @@ struct ggml_hexagon_session { bool valid_queue; bool valid_iface; - std::atomic op_pending; ggml_hexagon_opbatch* op_batch; ggml_hexagon_opqueue* op_queue; std::unordered_map> cloned_buffers; - std::unordered_set sync_peers; + std::unordered_set virt_peers; + std::unordered_set phys_peers; uint32_t n_threads = 0; uint32_t n_hvx = 0; @@ -400,14 +422,23 @@ struct ggml_hexagon_session { uint64_t vtcm_size = 0; size_t max_vmem = 0; size_t max_bufsize = 0; - uint32_t fence_seq; + uint32_t fence_seq = 0; + + std::atomic batch_req_seq{0}; + std::atomic batch_rsp_seq{0}; + std::atomic last_error{HTP_STATUS_OK}; uint64_t cached_uid = 0; std::vector cached_nodes; mutable std::unordered_set needs_repack; - ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev = nullptr) noexcept(false); + ggml_hexagon_mdev_group mdev; + ggml_backend_dev_t dev = nullptr; + ggml_backend_hexagon_device_context * dev_ctx = nullptr; + ggml_hexagon_fence_buffer * fence_buf = nullptr; + + ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev = nullptr, uint32_t mdev_idx = 0, uint32_t mdev_count = 0) noexcept(false); ~ggml_hexagon_session() noexcept(true); const char* c_name() const { return name.c_str(); } @@ -415,31 +446,36 @@ struct ggml_hexagon_session { void allocate(const ggml_hexagon_device_config & config) noexcept(false); void release() noexcept(true); + uint8_t * alloc_fence(uint32_t n_slots = 1); + void free_fence(void * ptr, uint32_t n_slots = 1); + + uint8_t * mdev_fence_slot = nullptr; + std::unordered_map cpy_fence_slots; + + void enqueue_mdev_group(); void enqueue_op(const htp_opnode & node); void enqueue_cpy(const ggml_tensor * src, ggml_tensor * dst, const ggml_tensor * sync_tensor = nullptr, uint32_t fence_seq = 0); - void enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq = 0); - void enqueue_allreduce(const ggml_tensor * dst, const std::vector & src_tensors, const std::vector & sync_tensors, uint32_t rank, uint32_t n_ranks, uint32_t fence_seq_entry = 0, uint32_t fence_seq_exit = 0); + void enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq = 0, bool wait = true); + void enqueue_allreduce(const ggml_tensor * dst, const std::vector & src_tensors, + const std::vector & sync_tensors, uint32_t rank, uint32_t n_ranks, + uint32_t fence_seq_entry = 0, uint32_t fence_seq_exit = 0); - void flush(bool all = true); - void flush_pending(bool all = false); + void flush_sync(bool all = true); + void flush_async(); void flush_batch(size_t min_ops = 1); - - uint64_t record_event(); - void wait_event(uint64_t seq); + void flush_peers(); + void flush_pending(bool all = true); bool clone_buffer(const ggml_hexagon_shared_buffer*); + void release_buffer(const ggml_hexagon_shared_buffer*); + void unclone_buffer(const ggml_hexagon_shared_buffer*); - void add_sync_peer(ggml_hexagon_session * peer) { - sync_peers.insert(peer); - } - - void flush_sync_peers() { - if (sync_peers.empty()) return; - - for (auto * peer : sync_peers) { - peer->flush_batch(); + void add_peer(ggml_hexagon_session * peer) { + if (this->phys_idx == peer->phys_idx) { + virt_peers.insert(peer); + } else { + phys_peers.insert(peer); } - sync_peers.clear(); } }; @@ -451,8 +487,9 @@ struct ggml_backend_hexagon_device_context { ggml_backend_dev_t dev = nullptr; size_t max_bufsize = 0; - ggml_backend_buffer_type buffer_type = {}; - ggml_backend_buffer_type host_buffer_type = {}; + ggml_backend_buffer_type buffer_type = {}; + ggml_backend_buffer_type host_buffer_type = {}; + ggml_backend_buffer_type fence_buffer_type = {}; std::unique_ptr sess; @@ -484,6 +521,8 @@ struct ggml_hexagon_rpcmem_block { int fd = -1; size_t size = 0; + std::unordered_set mapped_clones; + ggml_hexagon_rpcmem_block(size_t size) { base = (uint8_t *) rpcmem_alloc2(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, size); if (!base) { @@ -508,8 +547,6 @@ struct ggml_hexagon_shared_buffer { ggml_hexagon_session * sess; std::shared_ptr mem; std::vector tensor_extra; - uint32_t fence_head = 0; - size_t fences_size = 0; bool mapped; bool pinned; @@ -518,16 +555,6 @@ struct ggml_hexagon_shared_buffer { size_t size() const { return mem ? mem->size : 0; } int fd() const { return mem ? mem->fd : -1; } - uint8_t * alloc_fence() { - if (fences_size == 0) return nullptr; - int max_slots = fences_size / GGML_HEXAGON_FENCE_SLOT_SIZE; - uint32_t slot = (fence_head++) % max_slots; - - size_t guard_offset = size() - fences_size; - uint8_t * fence_ptr = base() + guard_offset + (size_t)slot * GGML_HEXAGON_FENCE_SLOT_SIZE; - return fence_ptr; - } - void mmap() { if (!this->mem) return; fastrpc_map_flags flags = this->pinned ? FASTRPC_MAP_FD : FASTRPC_MAP_FD_DELAYED; @@ -581,29 +608,24 @@ struct ggml_hexagon_shared_buffer { this->mem = nullptr; } - ggml_hexagon_shared_buffer(ggml_hexagon_session * sess, size_t size, bool pinned = false, size_t fence_size = 0) { - this->sess = sess; - this->mapped = false; - this->pinned = pinned; - this->fences_size = fence_size; + ggml_hexagon_shared_buffer(ggml_hexagon_session * sess, size_t size, bool pinned = false) { + this->sess = sess; + this->mapped = false; + this->pinned = pinned; - // Size adjustment inside the buffer class + // Size adjustment inside the buffer class: 4K aligned data size + 4K guard page size_t guard_offset = (size + 4095) & ~4095; - size_t total_size = guard_offset; - if (fence_size > 0) { - total_size += 4096 + fence_size; - } + size_t total_size = guard_offset + 4096; alloc(total_size); } // Clone constructor for cross-session mapping ggml_hexagon_shared_buffer(ggml_hexagon_session * sess, const ggml_hexagon_shared_buffer & other) { - this->sess = sess; - this->mem = other.mem; - this->mapped = false; - this->pinned = other.pinned; - this->fences_size = other.fences_size; + this->sess = sess; + this->mem = other.mem; + this->mapped = false; + this->pinned = other.pinned; } ~ggml_hexagon_shared_buffer() { @@ -614,6 +636,59 @@ struct ggml_hexagon_shared_buffer { } }; +struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer { + uint32_t slot_count = 0; + uint32_t slot_head = 0; + std::vector free_slots; + ggml_backend_buffer backend_buffer{}; + + ggml_hexagon_fence_buffer(ggml_hexagon_session * sess, ggml_backend_buffer_type_t buft, size_t size) + : ggml_hexagon_shared_buffer(sess, size, false /* pinned */), + slot_count(size / GGML_HEXAGON_FENCE_SLOT_SIZE), + slot_head(0) { + backend_buffer.buft = buft; + backend_buffer.context = static_cast(this); + backend_buffer.size = size; + } + + uint8_t * alloc_slot(uint32_t n_slots = 1) { + uint8_t * ptr = nullptr; + if (n_slots == 1 && !free_slots.empty()) { + uint32_t slot = free_slots.back(); + free_slots.pop_back(); + ptr = base() + (size_t) slot * GGML_HEXAGON_FENCE_SLOT_SIZE; + } else if (slot_head + n_slots <= slot_count) { + uint32_t slot = slot_head; + slot_head += n_slots; + ptr = base() + (size_t) slot * GGML_HEXAGON_FENCE_SLOT_SIZE; + } + if (ptr) { + memset(ptr, 0, (size_t) n_slots * GGML_HEXAGON_FENCE_SLOT_SIZE); + } + return ptr; + } + + void free_slot(void * ptr, uint32_t n_slots = 1) { + if (!ptr) return; + uint32_t slot = ((uint8_t *) ptr - base()) / GGML_HEXAGON_FENCE_SLOT_SIZE; + for (uint32_t i = 0; i < n_slots; i++) { + free_slots.push_back(slot + i); + } + } +}; + +inline uint8_t * ggml_hexagon_session::alloc_fence(uint32_t n_slots) { + uint8_t * ptr = fence_buf->alloc_slot(n_slots); + GGML_ASSERT(ptr); + return ptr; +} + +inline void ggml_hexagon_session::free_fence(void * ptr, uint32_t n_slots) { + if (fence_buf) { + fence_buf->free_slot(ptr, n_slots); + } +} + static ggml_hexagon_session * ggml_backend_hexagon_buffer_get_sess(ggml_backend_buffer_t buffer) { auto sbuf = static_cast(buffer->context); return sbuf->sess; @@ -621,6 +696,7 @@ static ggml_hexagon_session * ggml_backend_hexagon_buffer_get_sess(ggml_backend_ static void ggml_backend_hexagon_buffer_free_buffer(ggml_backend_buffer_t buffer) { auto sbuf = static_cast(buffer->context); + sbuf->sess->unclone_buffer(sbuf); delete sbuf; } @@ -1537,7 +1613,7 @@ static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer( auto dev_ctx = static_cast(buffer_type->context)->dev_ctx; auto sess = dev_ctx->session(); try { - ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false, GGML_HEXAGON_FENCE_BUFFER_SIZE); + ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false); return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_buffer_interface, sbuf, size); } catch (const std::exception & exc) { GGML_LOG_ERROR("ggml-hex: %s failed to allocate device buffer context: %s\n", dev_ctx->c_name(), exc.what()); @@ -1550,7 +1626,7 @@ static ggml_backend_buffer_t ggml_backend_hexagon_host_buffer_type_alloc_buffer( auto dev_ctx = static_cast(buffer_type->context)->dev_ctx; auto sess = dev_ctx->session(); try { - ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false, GGML_HEXAGON_FENCE_BUFFER_SIZE); + ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false); return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_host_buffer_interface, sbuf, size); } catch (const std::exception & exc) { GGML_LOG_ERROR("ggml-hex: %s failed to allocate host buffer context: %s\n", dev_ctx->c_name(), exc.what()); @@ -1618,11 +1694,16 @@ ggml_backend_hexagon_device_context::ggml_backend_hexagon_device_context(int dev host_buffer_type.device = dev; host_buffer_type.iface = ggml_backend_hexagon_host_buffer_type_interface; host_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(config.name + "-HOST", this); + + fence_buffer_type.device = dev; + fence_buffer_type.iface = ggml_backend_hexagon_buffer_type_interface; + fence_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(config.name + "-FENCE", this); } ggml_backend_hexagon_device_context::~ggml_backend_hexagon_device_context() { delete static_cast(buffer_type.context); delete static_cast(host_buffer_type.context); + delete static_cast(fence_buffer_type.context); } static bool ggml_backend_buffer_is_hexagon(const struct ggml_backend_buffer * b) { @@ -1698,8 +1779,8 @@ struct ggml_hexagon_opbatch { if (it != b_map.end()) { return it->second; } // Add new buffer to the batch - int bi = n_bufs++; GGML_ASSERT(n_bufs < HTP_OP_MAX_BUFS); + int bi = n_bufs++; b_map.insert({sbuf->fd(), bi}); @@ -1902,6 +1983,12 @@ struct ggml_hexagon_opbatch { } } + void update_mdev_group(uint32_t mdev_idx) { + if (n_ops > 0 && h_ops[0].opcode == HTP_OP_MDEV_GROUP) { + h_ops[0].params[0] = (int32_t) mdev_idx; + } + } + bool try_fuse_allreduce_add(const htp_opnode & node) { if (n_ops == 0 || opt_ar_select != 2) return false; if (node.opcode != HTP_OP_ADD) return false; @@ -1910,15 +1997,16 @@ struct ggml_hexagon_opbatch { if (last_node.opcode != HTP_OP_ALLREDUCE) return false; auto * ar_kparams = (struct htp_allreduce_kernel_params *) last_node.kernel_params; - const uint32_t rank = (uint32_t) ar_kparams->rank; - const ggml_tensor * ar_local = (rank < last_node.inputs.size()) ? last_node.inputs[rank] : nullptr; + const uint32_t rank = (uint32_t) ar_kparams->rank; + const uint32_t n_ranks = (uint32_t) ar_kparams->n_ranks; + const ggml_tensor * ar_local = last_node.inputs[rank]; const ggml_tensor * add_src0 = node.src0(); const ggml_tensor * add_src1 = node.src1(); + const ggml_tensor * add_dst = node.dst(); - if (!add_src0 || !add_src1 || !ar_local) return false; if (!ggml_hexagon_tensor_is_fuseable(ar_local)) return false; - const ggml_tensor * res_tensor = nullptr; + const ggml_tensor * res_tensor; if (add_src0 == ar_local || add_src0->data == ar_local->data) { res_tensor = add_src1; } else if (add_src1 == ar_local || add_src1->data == ar_local->data) { @@ -1927,14 +2015,12 @@ struct ggml_hexagon_opbatch { return false; } - if (!res_tensor || !res_tensor->data) return false; - if (ar_local->type != res_tensor->type) return false; const bool is_same_shape = (ar_local->ne[0] == res_tensor->ne[0] && ar_local->ne[1] == res_tensor->ne[1] && ar_local->ne[2] == res_tensor->ne[2] && ar_local->ne[3] == res_tensor->ne[3]); - const bool is_row_bcast = (ar_local->ne[0] == res_tensor->ne[0] && - res_tensor->ne[1] == 1 && res_tensor->ne[2] == 1 && res_tensor->ne[3] == 1); + const bool is_row_bcast = !is_same_shape && (ar_local->ne[0] == res_tensor->ne[0] && res_tensor->ne[1] == 1 && + res_tensor->ne[2] == 1 && res_tensor->ne[3] == 1); if (!is_same_shape && !is_row_bcast) return false; @@ -1947,13 +2033,21 @@ struct ggml_hexagon_opbatch { return false; } } - if (ggml_is_contiguous(ar_local) != ggml_is_contiguous(node.dst())) { + if (ggml_is_contiguous(ar_local) != ggml_is_contiguous(add_dst)) { return false; } + for (uint32_t r = 0; r < n_ranks; r++) { + const ggml_tensor * ar_src = last_node.inputs[r]; + if (ggml_hexagon_tensors_overlap(add_dst, ar_src)) { + HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: dst overlaps allreduce src %u\n", sess->c_name(), r); + return false; + } + } + struct htp_allreduce_kernel_params new_kparams; if (!ggml_hexagon_precompute_allreduce_params( - sess, node.dst(), (uint32_t) ar_kparams->rank, (uint32_t) ar_kparams->n_ranks, true, is_row_bcast, &new_kparams + sess, add_dst, (uint32_t) ar_kparams->rank, (uint32_t) ar_kparams->n_ranks, true, is_row_bcast, &new_kparams )) { HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: solver failed\n", sess->c_name()); return false; @@ -1961,7 +2055,6 @@ struct ggml_hexagon_opbatch { size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; auto fit_t = [&](const ggml_tensor * t) { - if (!t) return; if (!t_map.count(t)) { extra_tens++; auto sbuf = static_cast(t->buffer->context); @@ -1972,7 +2065,7 @@ struct ggml_hexagon_opbatch { } }; fit_t(res_tensor); - fit_t(node.dst()); + fit_t(add_dst); if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) { return false; } @@ -1981,7 +2074,7 @@ struct ggml_hexagon_opbatch { last_node.name = "ALLREDUCE+ADD"; last_node.inputs.push_back(res_tensor); last_node.outputs.clear(); - last_node.outputs.push_back(node.dst()); + last_node.outputs.push_back(add_dst); last_node.fused.push_back(node.node); memcpy(last_node.kernel_params, &new_kparams, sizeof(new_kparams)); @@ -1989,9 +2082,8 @@ struct ggml_hexagon_opbatch { o.opcode = HTP_OP_ALLREDUCE_ADD; memcpy(o.kernel_params, &new_kparams, sizeof(new_kparams)); - const uint32_t n_ranks = (uint32_t) ar_kparams->n_ranks; o.src[2 * n_ranks] = add_tensor(res_tensor); - o.dst[0] = add_tensor(node.dst()); + o.dst[0] = add_tensor(add_dst); for (uint32_t d = 1; d < HTP_OP_MAX_OUTPUTS; d++) { o.dst[d] = 0xffff; } @@ -2011,10 +2103,9 @@ struct ggml_hexagon_opbatch { const ggml_tensor * mul_src1 = node.src1(); const ggml_tensor * rms_out = last_node.dst(); - if (!mul_src0 || !mul_src1 || !rms_out) return false; if (!ggml_hexagon_tensor_is_fuseable(rms_out)) return false; - const ggml_tensor * weight = nullptr; + const ggml_tensor * weight; if (mul_src0 == rms_out || mul_src0->data == rms_out->data) { weight = mul_src1; } else if (mul_src1 == rms_out || mul_src1->data == rms_out->data) { @@ -2023,10 +2114,7 @@ struct ggml_hexagon_opbatch { return false; } - if (!weight || !weight->data) return false; - const ggml_tensor * src0 = last_node.src0(); - if (!src0 || !src0->data) return false; if (src0->ne[0] != weight->ne[0] || src0->ne[0] != node.dst()->ne[0]) { return false; @@ -2057,7 +2145,6 @@ struct ggml_hexagon_opbatch { size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; auto fit_t = [&](const ggml_tensor * t) { - if (!t) return; if (!t_map.count(t)) { extra_tens++; auto sbuf = static_cast(t->buffer->context); @@ -2112,10 +2199,9 @@ struct ggml_hexagon_opbatch { const ggml_tensor * add_src1 = node.src1(); const ggml_tensor * mm_out = last_node.dst(); - if (!add_src0 || !add_src1 || !mm_out) return false; if (!ggml_hexagon_tensor_is_fuseable(mm_out)) return false; - const ggml_tensor * src2 = nullptr; + const ggml_tensor * src2; if (add_src0 == mm_out || add_src0->data == mm_out->data) { src2 = add_src1; } else if (add_src1 == mm_out || add_src1->data == mm_out->data) { @@ -2124,11 +2210,8 @@ struct ggml_hexagon_opbatch { return false; } - if (!src2 || !src2->data) return false; - const ggml_tensor * src0 = last_node.src0(); const ggml_tensor * src1 = last_node.src1(); - if (!src0 || !src1) return false; struct htp_mm_kernel_params kparams; ggml_hexagon_precompute_fused_matmul_add_params(sess, src0, src1, src2, node.dst(), &kparams); @@ -2144,7 +2227,6 @@ struct ggml_hexagon_opbatch { size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; auto fit_t = [&](const ggml_tensor * t) { - if (!t) return; if (!t_map.count(t)) { extra_tens++; auto sbuf = static_cast(t->buffer->context); @@ -2197,7 +2279,6 @@ struct ggml_hexagon_opbatch { const ggml_tensor * w_in = node.src0(); const ggml_tensor * x_in = node.src1(); const ggml_tensor * d_in = node.dst(); - if (!w_in || !x_in || !d_in) return false; htp_opnode & last_node = ops[n_ops - 1]; @@ -2231,7 +2312,6 @@ struct ggml_hexagon_opbatch { size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; auto fit_t = [&](const ggml_tensor * t) { - if (!t) return; if (!t_map.count(t)) { extra_tens++; auto sbuf = static_cast(t->buffer->context); @@ -2282,7 +2362,6 @@ struct ggml_hexagon_opbatch { const ggml_tensor * w0 = last_node.src0(); const ggml_tensor * x = last_node.src1(); const ggml_tensor * w1 = node.src0(); - if (!w0 || !x || !w1) return false; struct htp_mm_kernel_params kparams; ggml_hexagon_precompute_fused_mmnx_params(sess, w0, x, 2, &kparams); @@ -2297,7 +2376,6 @@ struct ggml_hexagon_opbatch { size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; auto fit_t = [&](const ggml_tensor * t) { - if (!t) return; if (!t_map.count(t)) { extra_tens++; auto sbuf = static_cast(t->buffer->context); @@ -2359,7 +2437,6 @@ struct ggml_hexagon_opbatch { const ggml_tensor * x_in = node.src1(); const ggml_tensor * ids_in = node.node->src[2]; const ggml_tensor * d_in = node.dst(); - if (!w_in || !x_in || !ids_in || !d_in) return false; htp_opnode & last_node = ops[n_ops - 1]; @@ -2394,7 +2471,6 @@ struct ggml_hexagon_opbatch { size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; auto fit_t = [&](const ggml_tensor * t) { - if (!t) return; if (!t_map.count(t)) { extra_tens++; auto sbuf = static_cast(t->buffer->context); @@ -2447,7 +2523,6 @@ struct ggml_hexagon_opbatch { const ggml_tensor * x = last_node.src1(); const ggml_tensor * ids = last_node.node->src[2]; const ggml_tensor * w1 = node.src0(); - if (!w0 || !x || !ids || !w1) return false; struct htp_mm_kernel_params kparams; ggml_hexagon_precompute_fused_mmidnx_params(sess, w0, x, node.dst(), 2, &kparams); @@ -2462,7 +2537,6 @@ struct ggml_hexagon_opbatch { size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; auto fit_t = [&](const ggml_tensor * t) { - if (!t) return; if (!t_map.count(t)) { extra_tens++; auto sbuf = static_cast(t->buffer->context); @@ -2540,17 +2614,14 @@ struct ggml_hexagon_opqueue { // Shared buffer for storing batches ggml_hexagon_shared_buffer *shm_buf; size_t shm_blk_size; - - uint64_t req_seq = 0; - uint64_t rsp_seq = 0; + size_t depth; using opvec = std::vector; - std::queue done; // completed batch ids std::vector op_cache; // per batch op cache std::vector start_usec; // per batch start time - ggml_hexagon_opqueue(ggml_hexagon_session *sess, size_t batch_size, size_t depth) { + ggml_hexagon_opqueue(ggml_hexagon_session *sess, size_t batch_size, size_t depth) : depth(depth) { size_t n_bufs = HTP_OP_MAX_BUFS; size_t n_ops = batch_size; size_t n_tensors = n_ops * HTP_OP_MAX_OUTPUTS + n_ops * HTP_OP_MAX_INPUTS; @@ -2571,9 +2642,6 @@ struct ggml_hexagon_opqueue { op_cache.resize(depth); start_usec.resize(depth, 0); - // init done queue - for (unsigned int i = 0; i < depth; i++) { done.push(i); } - if (opt_verbose) { GGML_LOG_INFO("ggml-hex: %s allocated opqueue : batch-size %zu depth %zu shm-size %zu shm-block-size %zu\n", sess->c_name(), batch_size, depth, shm_buf->size(), shm_blk_size); @@ -2587,7 +2655,7 @@ struct ggml_hexagon_opqueue { size_t shm_size() const { return shm_buf ? shm_buf->size() : 0; } // push new batch - bool push(htp_opbatch_req& req, dspqueue_buffer& dbuf, ggml_hexagon_opbatch* op_batch) { + bool push(htp_opbatch_req& req, dspqueue_buffer& dbuf, const ggml_hexagon_opbatch* op_batch, uint64_t seq) { static_assert(sizeof(htp_opbatch_req) % 8 == 0, "sizeof(htp_opbatch_req) must be multiple of 8"); static_assert(sizeof(htp_opbatch_rsp) % 8 == 0, "sizeof(htp_opbatch_rsp) must be multiple of 8"); static_assert(sizeof(htp_buf_desc) % 8 == 0, "sizeof(htp_buf_desc) must be multiple of 8"); @@ -2595,16 +2663,17 @@ struct ggml_hexagon_opqueue { static_assert(sizeof(htp_op_desc) % 8 == 0, "sizeof(htp_op_desc) must be multiple of 8"); static_assert(sizeof(htp_prof_desc) % 8 == 0, "sizeof(htp_prof_desc) must be multiple of 8"); - if (done.empty()) { return false; } + if (seq - shm_buf->sess->batch_rsp_seq > depth) { return false; } - req.id = done.front(); done.pop(); // batch id + const uint32_t slot = (uint32_t) ((seq - 1) % depth); + + req.seq = seq; req.n_bufs = op_batch->n_bufs; req.n_tensors = op_batch->n_tens; req.n_ops = op_batch->n_ops; - req.seq = ++req_seq; - op_cache[req.id] = std::move(op_batch->ops); - start_usec[req.id] = ggml_time_us(); + op_cache[slot] = op_batch->ops; + start_usec[slot] = ggml_time_us(); const size_t b_size = sizeof(htp_buf_desc) * req.n_bufs; const size_t t_size = sizeof(htp_tensor) * req.n_tensors; @@ -2619,7 +2688,7 @@ struct ggml_hexagon_opqueue { req.n_traces = 0; } - dbuf.ptr = shm_buf->base() + (req.id * shm_blk_size); + dbuf.ptr = shm_buf->base() + ((size_t) slot * shm_blk_size); dbuf.fd = shm_buf->fd(); dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT; dbuf.offset = (uint8_t*) dbuf.ptr - (uint8_t*) shm_buf->base(); @@ -2632,18 +2701,14 @@ struct ggml_hexagon_opqueue { uint8_t * t_ptr = m_ptr; m_ptr += t_size; uint8_t * o_ptr = m_ptr; - op_batch->sort_buffers(); - memcpy(b_ptr, (void *) op_batch->h_bufs.data(), b_size); memcpy(t_ptr, (void *) op_batch->h_tens.data(), t_size); memcpy(o_ptr, (void *) op_batch->h_ops.data(), o_size); - HEX_VERBOSE("ggml-hex: %s opqueue-push batch #%u : n-bufs %u n-tensors %u n-ops %u vmem %zu : b-size %zu t-size %zu o-size %zu m-size %zu\n", - shm_buf->sess->c_name(), req.id, req.n_bufs, req.n_tensors, req.n_ops, op_batch->b_vmem, + HEX_VERBOSE("ggml-hex: %s opqueue-push batch #%llu : n-bufs %u n-tensors %u n-ops %u vmem %zu : b-size %zu t-size %zu o-size %zu m-size %zu\n", + shm_buf->sess->c_name(), (unsigned long long) req.seq, req.n_bufs, req.n_tensors, req.n_ops, op_batch->b_vmem, b_size, t_size, o_size, (size_t) dbuf.size); - op_batch->reset(); - if (opt_verbose > 1) { htp_buf_desc *b = (htp_buf_desc*) b_ptr; for (unsigned int i=0; i < req.n_bufs; i++) { @@ -2662,9 +2727,7 @@ struct ggml_hexagon_opqueue { } void pop(htp_opbatch_rsp rsp, dspqueue_buffer dbuf) { - GGML_ASSERT(rsp.id < op_cache.size()); - - done.push(rsp.id); + const uint32_t slot = (uint32_t) ((rsp.seq - 1) % depth); const size_t b_size = sizeof(htp_buf_desc) * rsp.n_bufs; const size_t t_size = sizeof(htp_tensor) * rsp.n_tensors; @@ -2681,15 +2744,15 @@ struct ggml_hexagon_opqueue { const size_t m_size = b_size + t_size + o_size + p_size + tr_size; GGML_ASSERT(m_size <= shm_blk_size); - HEX_VERBOSE("ggml-hex: %s opqueue-pop batch #%u : n-bufs %u n-tensors %u n-ops %u : m-size %zu b-size %zu t-size %zu o-size %zu\n", - shm_buf->sess->c_name(), rsp.id, rsp.n_bufs, rsp.n_tensors, rsp.n_ops, + HEX_VERBOSE("ggml-hex: %s opqueue-pop batch #%llu : n-bufs %u n-tensors %u n-ops %u : m-size %zu b-size %zu t-size %zu o-size %zu\n", + shm_buf->sess->c_name(), (unsigned long long) rsp.seq, rsp.n_bufs, rsp.n_tensors, rsp.n_ops, (size_t) dbuf.size, b_size, t_size, o_size); uint8_t * m_ptr = (uint8_t*) dbuf.ptr; uint8_t * p_ptr = m_ptr + (b_size + t_size + o_size); if (rsp.n_ops > 0) { - auto & ops = op_cache[rsp.id]; + auto & ops = op_cache[slot]; GGML_ASSERT(rsp.n_ops <= ops.size()); const htp_prof_desc * pd = (const htp_prof_desc *) p_ptr; @@ -2712,16 +2775,41 @@ struct ggml_hexagon_opqueue { ggml_hexagon_dump_trace_events(shm_buf->sess->name, rsp, trace_events, n_traces); } } - - if (rsp.seq > rsp_seq) { - rsp_seq = rsp.seq; - } } }; -// Flush HTP response queue i.e wait for all outstanding requests to complete +void ggml_hexagon_session::flush_peers() { + auto vpeers = std::move(virt_peers); + virt_peers.clear(); + for (auto * peer : vpeers) { + peer->flush_sync(); + } + + auto ppeers = std::move(phys_peers); + phys_peers.clear(); + for (auto * peer : ppeers) { + peer->flush_async(); + } + + for (auto & sub : this->mdev.sessions) { + sub->flush_peers(); + } +} + +void ggml_hexagon_session::flush_async() { + flush_peers(); + flush_batch(); +} + void ggml_hexagon_session::flush_pending(bool all) { - while (this->op_pending) { + for (auto & sub : this->mdev.sessions) { + sub->flush_pending(all); + if (sub->last_error > HTP_STATUS_OK) { + this->last_error = sub->last_error.load(); + } + } + + while (this->batch_rsp_seq < this->batch_req_seq) { struct htp_opbatch_rsp rsp; uint32_t rsp_size; uint32_t flags; @@ -2746,32 +2834,64 @@ void ggml_hexagon_session::flush_pending(bool all) { GGML_ABORT("ggml-hex: %s dspcall : bad response : size %u dspbufs %u\n", this->c_name(), rsp_size, n_dbufs); } - if (rsp.status != HTP_STATUS_OK) { - GGML_LOG_ERROR("ggml-hex: %s dspcall : dsp-rsp: %s\n", this->c_name(), status_to_str(rsp.status)); - // TODO: handle errors + if (rsp.status > HTP_STATUS_OK) { + GGML_LOG_ERROR("ggml-hex: %s dspcall : dsp-rsp %s\n", this->c_name(), status_to_str(rsp.status)); + this->last_error = rsp.status; + for (auto & sub : this->mdev.sessions) { + sub->last_error = rsp.status; + } } op_queue->pop(rsp, dbuf); - this->op_pending--; // atomic dec + GGML_ASSERT(rsp.seq == this->batch_rsp_seq + 1); + this->batch_rsp_seq = rsp.seq; if (!all) break; } } +void ggml_hexagon_session::flush_sync(bool all) { + flush_async(); + flush_pending(all); +} + void ggml_hexagon_session::flush_batch(size_t min_ops) { if (op_batch->n_ops < min_ops) { return; } + op_batch->sort_buffers(); + htp_opbatch_req req {}; dspqueue_buffer dbuf{}; - if (!op_queue->push(req, dbuf, op_batch)) { + const uint64_t seq = ++this->batch_req_seq; + + op_batch->update_mdev_group(this->mdev.idx); + + if (!op_queue->push(req, dbuf, op_batch, seq)) { flush_pending(false); - op_queue->push(req, dbuf, op_batch); + op_queue->push(req, dbuf, op_batch, seq); } - // Bump pending flag (cleared in the session::flush once we get the response) - this->op_pending++; // atomic inc + for (auto & sub : this->mdev.sessions) { + htp_opbatch_req sub_req {}; + dspqueue_buffer sub_dbuf{}; + + sub->batch_req_seq = seq; + op_batch->update_mdev_group(sub->mdev.idx); + + if (!sub->op_queue->push(sub_req, sub_dbuf, op_batch, seq)) { + sub->flush_pending(false); + sub->op_queue->push(sub_req, sub_dbuf, op_batch, seq); + } + + HEX_VERBOSE("ggml-hex: %s queue-opbatch: %p size %u\n", sub->c_name(), sub_dbuf.ptr, sub_dbuf.size); + + int err = dspqueue_write(sub->queue, 0, 1, &sub_dbuf, sizeof(sub_req), (const uint8_t*) &sub_req, DSPQUEUE_TIMEOUT); + if (err != 0) { + GGML_ABORT("ggml-hex: %s dspqueue_write failed: 0x%08x\n", sub->c_name(), (unsigned) err); + } + } HEX_VERBOSE("ggml-hex: %s queue-opbatch: %p size %u\n", this->c_name(), dbuf.ptr, dbuf.size); @@ -2779,28 +2899,28 @@ void ggml_hexagon_session::flush_batch(size_t min_ops) { if (err != 0) { GGML_ABORT("ggml-hex: %s dspqueue_write failed: 0x%08x\n", this->c_name(), (unsigned) err); } -} -void ggml_hexagon_session::flush(bool all) { - flush_sync_peers(); - flush_batch(); - flush_pending(all); + op_batch->reset(); } void ggml_hexagon_session::enqueue_op(const htp_opnode & node) { - for (auto t : node.get_inputs()) { + auto clone_tensor_buffer = [this](const ggml_tensor * t) { if (t && t->buffer && ggml_backend_buffer_is_hexagon(t->buffer)) { + auto sbuf = static_cast(t->buffer->context); if (ggml_backend_hexagon_buffer_get_sess(t->buffer) != this) { - this->clone_buffer(static_cast(t->buffer->context)); + this->clone_buffer(sbuf); + } + for (auto & sub : this->mdev.sessions) { + sub->clone_buffer(sbuf); } } + }; + + for (auto t : node.get_inputs()) { + clone_tensor_buffer(t); } for (auto t : node.get_outputs()) { - if (t && t->buffer && ggml_backend_buffer_is_hexagon(t->buffer)) { - if (ggml_backend_hexagon_buffer_get_sess(t->buffer) != this) { - this->clone_buffer(static_cast(t->buffer->context)); - } - } + clone_tensor_buffer(t); } if (opt_opfusion && op_batch->try_fuse(node)) { @@ -2808,39 +2928,84 @@ void ggml_hexagon_session::enqueue_op(const htp_opnode & node) { } if (!op_batch->fit_op(node)) { - flush_batch(); + flush_async(); } + + if (this->mdev.count > 1 && op_batch->n_ops == 0) { + enqueue_mdev_group(); + } + op_batch->add_op(node); } +void ggml_hexagon_session::enqueue_mdev_group() { + htp_opnode group_node(HTP_OP_MDEV_GROUP); + + uint8_t * fence_slot = this->mdev_fence_slot; + + static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE }; + ggml_tensor dummy_t {}; + dummy_t.buffer = &this->fence_buf->backend_buffer; + dummy_t.extra = &fence_extra; + dummy_t.data = (void *) fence_slot; + dummy_t.type = GGML_TYPE_I8; + dummy_t.ne[0] = HTP_FENCE_SLOT_SIZE; + dummy_t.ne[1] = (int64_t) this->mdev.count; + dummy_t.ne[2] = 1; + dummy_t.ne[3] = 1; + dummy_t.nb[0] = 1; + dummy_t.nb[1] = HTP_FENCE_SLOT_SIZE; + dummy_t.nb[2] = dummy_t.nb[1] * dummy_t.ne[1]; + dummy_t.nb[3] = dummy_t.nb[2]; + dummy_t.op = GGML_OP_NONE; + dummy_t.op_params[0] = (int32_t) this->mdev.idx; + + ggml_tensor * node = group_node.add_dummy(dummy_t); + node->src[0] = node; + group_node.init(node); + group_node.outputs.clear(); + group_node.name = "MDEV_GROUP"; + + if (this->fence_buf->sess != this) { + this->clone_buffer(this->fence_buf); + } + for (auto & sub : this->mdev.sessions) { + sub->clone_buffer(this->fence_buf); + } + + op_batch->add_op(group_node); +} + void ggml_hexagon_session::enqueue_cpy(const ggml_tensor * src, ggml_tensor * dst, const ggml_tensor * sync_tensor, uint32_t fence_seq) { - htp_opnode cpy_node(HTP_OP_CPY); + const bool with_fence = sync_tensor != nullptr; + htp_opnode cpy_node(with_fence ? HTP_OP_CPY_FENCE : HTP_OP_CPY); ggml_tensor* node = cpy_node.add_dummy(*dst); node->op = GGML_OP_CPY; node->src[0] = const_cast(src); - node->src[1] = sync_tensor ? cpy_node.add_dummy(*sync_tensor) : nullptr; - if (sync_tensor) { + node->src[1] = with_fence ? cpy_node.add_dummy(*sync_tensor) : nullptr; + if (with_fence) { node->op_params[0] = (int32_t) fence_seq; } cpy_node.init(node); - if (sync_tensor) { + if (with_fence) { cpy_node.name = "CPY+FENCE"; } this->enqueue_op(cpy_node); } -void ggml_hexagon_session::enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq) { +void ggml_hexagon_session::enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq, bool wait) { htp_opnode sync_node(HTP_OP_FENCE); ggml_tensor* node = sync_node.add_dummy(*sync_tensor); node->op = GGML_OP_NONE; node->src[0] = node; node->op_params[0] = (int32_t) fence_seq; + node->op_params[1] = wait ? 0 : 1; sync_node.init(node); - sync_node.name = "FENCE"; + sync_node.name = wait ? "FENCE_WAIT" : "FENCE_SIGNAL"; this->enqueue_op(sync_node); } @@ -2858,7 +3023,6 @@ static bool ggml_hexagon_precompute_allreduce_params( kparams->n_ranks = (int32_t) n_ranks; kparams->is_row_bcast = (has_add && is_row_bcast) ? 1 : 0; - const uint32_t n_bufs = n_ranks + 1 + (has_add ? 1 : 0); const uint32_t nelem = (uint32_t) ggml_nelements(dst); const uint32_t elem_size = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float); const bool is_contiguous = ggml_is_contiguous(dst); @@ -2902,6 +3066,7 @@ static bool ggml_hexagon_precompute_allreduce_params( const uint32_t rank_nelem = (uint32_t) kparams->rank_nelem; const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, (std::max)(1u, rank_nelem / 128)); kparams->n_threads = n_threads; + const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast); uint32_t block_elems = 65536; if (block_elems > rank_nelem / n_threads && rank_nelem / n_threads > 128) { @@ -2911,15 +3076,15 @@ static bool ggml_hexagon_precompute_allreduce_params( kparams->block_elems = block_elems; kparams->vtcm_size_per_thread = 2 * block_elems * elem_size; - kparams->vtcm_size = n_threads * n_bufs * kparams->vtcm_size_per_thread; + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; while ((size_t) kparams->vtcm_size > sess->vtcm_size && block_elems > 128) { - const size_t max_bytes_per_buf = sess->vtcm_size / (n_threads * n_bufs * 2); + const size_t max_bytes_per_buf = sess->vtcm_size / (n_vtcm_buffers * 2); block_elems = (uint32_t) hex_align_down((size_t) (max_bytes_per_buf / elem_size), 128); if (block_elems < 128) break; kparams->block_elems = block_elems; kparams->vtcm_size_per_thread = 2 * block_elems * elem_size; - kparams->vtcm_size = n_threads * n_bufs * kparams->vtcm_size_per_thread; + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; } if (sess->vtcm_size < (size_t) kparams->vtcm_size || block_elems < 128) { @@ -2935,6 +3100,7 @@ static bool ggml_hexagon_precompute_allreduce_params( const uint32_t rank_nrows = (uint32_t) kparams->rank_nelem; const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, (std::max)(1u, rank_nrows)); kparams->n_threads = n_threads; + const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast); const uint32_t row_bytes = ne0 * elem_size; const uint32_t row_size_aligned = (uint32_t) hex_align_up(row_bytes, 128); @@ -2946,14 +3112,14 @@ static bool ggml_hexagon_precompute_allreduce_params( kparams->block_elems = block_rows; kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned); - kparams->vtcm_size = n_threads * n_bufs * kparams->vtcm_size_per_thread; + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; while ((size_t) kparams->vtcm_size > sess->vtcm_size && block_rows > 1) { - const size_t max_rows_per_buf = sess->vtcm_size / (n_threads * n_bufs * 2 * row_size_aligned); + const size_t max_rows_per_buf = sess->vtcm_size / (n_vtcm_buffers * 2 * row_size_aligned); block_rows = (std::max)(1u, (uint32_t) max_rows_per_buf); kparams->block_elems = block_rows; kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned); - kparams->vtcm_size = n_threads * n_bufs * kparams->vtcm_size_per_thread; + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; if (max_rows_per_buf == 0) break; } @@ -3009,28 +3175,20 @@ void ggml_hexagon_session::enqueue_allreduce( this->enqueue_op(ar_node); } -void ggml_hexagon_session::wait_event(uint64_t seq) { - flush_sync_peers(); - HEX_VERBOSE("ggml-hex: %s opqueue-wait start: seq %llu, current rsp-seq %llu, pending %d\n", - this->name.c_str(), (unsigned long long)seq, (unsigned long long)op_queue->rsp_seq, (int)this->op_pending); - while (op_queue->rsp_seq < seq && this->op_pending > 0) { - this->flush_pending(false); - } - HEX_VERBOSE("ggml-hex: %s opqueue-wait end: seq %llu, current rsp-seq %llu, pending %d\n", - this->name.c_str(), (unsigned long long)seq, (unsigned long long)op_queue->rsp_seq, (int)this->op_pending); -} - -uint64_t ggml_hexagon_session::record_event() { - flush_batch(); - return op_queue->req_seq; -} - bool ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer *sbuf) { - if (this->cloned_buffers.find(sbuf->fd()) != this->cloned_buffers.end()) return true; + GGML_ASSERT(sbuf && sbuf->mem); + if (sbuf->sess == this) return true; + + auto mem = sbuf->mem; + int fd = mem->fd; + + GGML_ASSERT(fd >= 0); + + if (this->cloned_buffers.find(fd) != this->cloned_buffers.end()) return true; HEX_VERBOSE("ggml-hex: %s clone-buffer: %s base %p size %zu fd %d\n", this->name.c_str(), - sbuf->c_name(), sbuf->base(), sbuf->size(), sbuf->fd()); + sbuf->c_name(), sbuf->base(), sbuf->size(), fd); auto clone = std::make_unique(this, *sbuf); try { @@ -3040,10 +3198,38 @@ bool ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer *sbuf) return false; } - this->cloned_buffers[sbuf->fd()] = std::move(clone); + this->cloned_buffers[fd] = std::move(clone); + mem->mapped_clones.insert(this); return true; } +void ggml_hexagon_session::release_buffer(const ggml_hexagon_shared_buffer * sbuf) { + GGML_ASSERT(sbuf && sbuf->mem); + + auto mem = sbuf->mem; + int fd = mem->fd; + + GGML_ASSERT(fd >= 0); + + auto it = this->cloned_buffers.find(fd); + if (it != this->cloned_buffers.end()) { + auto clone = std::move(it->second); + this->cloned_buffers.erase(it); + } + mem->mapped_clones.erase(this); +} + +void ggml_hexagon_session::unclone_buffer(const ggml_hexagon_shared_buffer * sbuf) { + GGML_ASSERT(sbuf && sbuf->mem); + + auto mem = sbuf->mem; + std::vector sessions(mem->mapped_clones.begin(), mem->mapped_clones.end()); + + for (auto * sess : sessions) { + sess->release_buffer(sbuf); + } +} + static size_t ggml_hexagon_measure_max_vmem(ggml_hexagon_session *sess) { // Allocate a bunch pinned buffers till failure. // This is kind of expensive but handy for figuring out exactly how much we can mmap on a specific device. @@ -3082,14 +3268,16 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n this->valid_queue = false; this->valid_iface = false; - this->phys_idx = phys_idx; - this->virt_idx = virt_idx; - this->domain_id = config.domain_id; - this->session_id = 0; - this->name = config.name; - this->op_pending = 0; + this->name = config.name; + this->phys_idx = phys_idx; + this->virt_idx = virt_idx; + this->domain_id = config.domain_id; + this->session_id = 0; + this->batch_req_seq = 0; + this->batch_rsp_seq = 0; + this->last_error = HTP_STATUS_OK; - GGML_LOG_DEBUG("ggml-hex: %s allocating new session\n", this->name.c_str()); + GGML_LOG_DEBUG("ggml-hex: %s allocating new session : domain %u phys-idx %u virt-idx %u\n", this->name.c_str(), this->domain_id, phys_idx, virt_idx); if (config.domain_id < 0 || config.domain_name.empty()) { GGML_LOG_ERROR("ggml-hex: %s: invalid physical CDSP core %d\n", config.name.c_str(), config.physical_idx); @@ -3098,25 +3286,14 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n const std::string & dom_name = config.domain_name; - // Enable Unsigned PD for all domains - { - struct remote_rpc_control_unsigned_module u; - u.domain = -1; - u.enable = 1; - int err = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *) &u, sizeof(u)); - if (err != AEE_SUCCESS) { - GGML_LOG_ERROR("ggml-hex: %s failed to enable unsigned PD : error 0x%x\n", this->c_name(), err); - throw std::runtime_error("ggml-hex: remote_session_control(unsign) failed (see log for details)"); - } - } - // Create new session if virtual_idx > 0 if (virt_idx > 0) { - struct remote_rpc_reserve_new_session n; + struct remote_rpc_reserve_new_session n {}; n.domain_name_len = dom_name.size(); n.domain_name = const_cast(dom_name.c_str()); n.session_name = const_cast(this->name.c_str()); n.session_name_len = this->name.size(); + n.session_id = virt_idx; int err = remote_session_control(FASTRPC_RESERVE_NEW_SESSION, (void *) &n, sizeof(n)); if (err != AEE_SUCCESS) { @@ -3130,7 +3307,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n this->domain_id = n.effective_domain_id; this->valid_session = true; } else { - struct remote_rpc_effective_domain_id eff = {}; + struct remote_rpc_effective_domain_id eff {}; eff.domain_name = const_cast(dom_name.c_str()); eff.domain_name_len = dom_name.size(); eff.session_id = 0; @@ -3144,6 +3321,18 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n } } + // Enable unsigned modules + { + struct remote_rpc_control_unsigned_module u; + u.domain = this->domain_id; + u.enable = 1; + int err = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *) &u, sizeof(u)); + if (err != AEE_SUCCESS) { + GGML_LOG_ERROR("ggml-hex: %s failed to enable unsigned PD : error 0x%x\n", this->c_name(), err); + throw std::runtime_error("ggml-hex: remote_session_control(unsign) failed (see log for details)"); + } + } + char session_uri[256]; { char htp_uri[256]; @@ -3171,7 +3360,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n // Open session int err = htp_iface_open(session_uri, &this->handle); if (err != AEE_SUCCESS) { - GGML_LOG_ERROR("ggml-hex: %s failed to open session : error 0x%x\n", this->c_name(), err); + GGML_LOG_ERROR("ggml-hex: %s failed to open session : uri %s error 0x%x\n", this->c_name(), session_uri, err); throw std::runtime_error("ggml-hex: failed to open session (see log for details)"); } @@ -3186,8 +3375,9 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n unsigned long long hw_vtcm_size = 0; int hw_err = htp_iface_hwinfo(this->handle, &hw_n_threads, &hw_n_hvx, &hw_n_hmx, &hw_vtcm_size); if (hw_err == 0) { - this->n_threads = opt_nhvx > 0 ? (uint32_t)opt_nhvx : (uint32_t)hw_n_threads; - this->n_hvx = opt_nhvx > 0 ? (uint32_t)opt_nhvx : (uint32_t)hw_n_hvx; + const uint32_t max_n_threads = (std::min)((uint32_t) HTP_MAX_NTHREADS, (uint32_t) hw_n_threads); + this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) max_n_threads) : max_n_threads; + this->n_hvx = this->n_threads; this->n_hmx = (opt_nhmx != 0) ? (uint32_t)hw_n_hmx : 0; this->vtcm_size = (uint64_t)hw_vtcm_size; GGML_LOG_INFO("ggml-hex: %s hwinfo: threads %u, hvx %u, hmx %u, vtcm %llu MB\n", @@ -3195,8 +3385,9 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n (unsigned long long)(this->vtcm_size / (1024 * 1024))); } else { GGML_LOG_WARN("ggml-hex: %s failed to query hwinfo (0x%x), using defaults\n", this->c_name(), hw_err); - this->n_threads = opt_nhvx > 0 ? (uint32_t)opt_nhvx : 8; - this->n_hvx = opt_nhvx > 0 ? (uint32_t)opt_nhvx : 8; + const uint32_t default_n_threads = (std::min)(8u, (uint32_t) HTP_MAX_NTHREADS); + this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) HTP_MAX_NTHREADS) : default_n_threads; + this->n_hvx = this->n_threads; this->n_hmx = (opt_nhmx != 0) ? 1 : 0; this->vtcm_size = 8 * 1024 * 1024; } @@ -3252,6 +3443,11 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n // Allocate buffers and state for op batching this->op_queue = new ggml_hexagon_opqueue(this, opt_opbatch, opt_opqueue); + this->fence_buf = new ggml_hexagon_fence_buffer(this, &dev_ctx->fence_buffer_type, 64 * 1024); + if (this->mdev.count > 1) { + this->mdev_fence_slot = this->alloc_fence(this->mdev.count); + } + if (!opt_vmem) { opt_vmem = ggml_hexagon_measure_max_vmem(this); GGML_LOG_INFO("ggml-hex: %s measured max vmem %zu\n", this->c_name(), opt_vmem); @@ -3262,7 +3458,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem); // Start dspqueue/opbatch processing - err = htp_iface_start(this->handle, this->session_id, this->queue_id, opt_nhvx, opt_nhmx, this->max_vmem); + err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, this->max_vmem); if (err != 0) { GGML_LOG_ERROR("ggml-hex: %s failed to start session: 0x%08x\n", this->c_name(), (unsigned) err); throw std::runtime_error("ggml-hex: iface start failed (see log for details)"); @@ -3283,6 +3479,8 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n void ggml_hexagon_session::release() noexcept(true) { GGML_LOG_INFO("ggml-hex: releasing session: %s\n", this->name.c_str()); + this->mdev.sessions.clear(); + int err; if (this->valid_iface) { @@ -3295,6 +3493,19 @@ void ggml_hexagon_session::release() noexcept(true) { delete this->op_batch; delete this->op_queue; + for (auto & it : this->cpy_fence_slots) { + free_fence((void *) it.second, 1); + } + this->cpy_fence_slots.clear(); + + if (this->fence_buf) { + unclone_buffer(this->fence_buf); + delete this->fence_buf; + this->fence_buf = nullptr; + } + while (!this->cloned_buffers.empty()) { + release_buffer(this->cloned_buffers.begin()->second.get()); + } if (opt_etm) { err = htp_iface_etm(this->handle, 0); @@ -3321,23 +3532,30 @@ void ggml_hexagon_session::release() noexcept(true) { if (this->valid_handle) { htp_iface_close(this->handle); } - - this->cloned_buffers.clear(); } -ggml_hexagon_session::ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev) noexcept(false) { - op_batch = nullptr; - op_queue = nullptr; - fence_seq = ((uintptr_t)this) & 0xFFFF; +ggml_hexagon_session::ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev, uint32_t mdev_idx, uint32_t mdev_count) noexcept(false) { + this->dev = dev; + this->dev_ctx = static_cast(dev->context); + this->mdev.idx = mdev_idx; + this->mdev.count = mdev_count > 0 ? mdev_count : (uint32_t) (1 + config.mdev_group.size()); + op_batch = nullptr; + op_queue = nullptr; + fence_buf = nullptr; + fence_seq = ((uintptr_t)this) & 0xFFFF; try { allocate(config); + if (this->mdev.idx == 0 && !config.mdev_group.empty()) { + for (size_t i = 0; i < config.mdev_group.size(); i++) { + this->mdev.sessions.push_back(std::make_unique( + config.mdev_group[i], this->dev, (uint32_t) (i + 1), this->mdev.count)); + } + } } catch (const std::exception & exc) { release(); throw; } - - GGML_UNUSED(dev); } ggml_hexagon_session::~ggml_hexagon_session() noexcept(true) { @@ -3563,10 +3781,6 @@ static bool ggml_hexagon_supported_gated_delta_net(const struct ggml_hexagon_ses const struct ggml_tensor * state = op->src[5]; const struct ggml_tensor * dst = op; - if (!q || !k || !v || !g || !beta || !state) { - return false; - } - if (q->type != GGML_TYPE_F32 || k->type != GGML_TYPE_F32 || v->type != GGML_TYPE_F32 || g->type != GGML_TYPE_F32 || beta->type != GGML_TYPE_F32 || state->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { @@ -3754,6 +3968,7 @@ static void ggml_hexagon_precompute_hvx_mm_params( struct htp_mm_kernel_params * kparams ) { kparams->n_hmx = 0; + kparams->n_threads = sess->n_threads; const bool is_quant = (wtype != GGML_TYPE_F16 && wtype != GGML_TYPE_F32); const int src1_nrows = ne11 * ne12 * ne13; @@ -4193,6 +4408,7 @@ static void ggml_hexagon_precompute_fused_mmnx_params( struct htp_mm_kernel_params * kparams ) { memset(kparams, 0, sizeof(*kparams)); + kparams->n_threads = sess->n_threads; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; @@ -4921,6 +5137,14 @@ static bool ggml_hexagon_supported_pad(const struct ggml_hexagon_session * sess, return false; } + const int32_t lp0 = ((const int32_t *) op->op_params)[0]; + const int32_t rp0 = ((const int32_t *) op->op_params)[1]; + const int32_t circular = ((const int32_t *) op->op_params)[8]; + + if (circular && (lp0 > src0->ne[0] || rp0 > src0->ne[0])) { + return false; + } + return true; GGML_UNUSED(sess); @@ -4972,10 +5196,6 @@ static bool ggml_hexagon_supported_solve_tri(const struct ggml_hexagon_session * const struct ggml_tensor * src1 = op->src[1]; // B const struct ggml_tensor * dst = op; // X - if (!src0 || !src1) { - return false; - } - if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { return false; } @@ -5145,7 +5365,7 @@ static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const st } static bool is_mergeable_mul_mat(const ggml_tensor * t) { - if (!t || t->op != GGML_OP_MUL_MAT) return false; + if (t->op != GGML_OP_MUL_MAT) return false; const ggml_tensor * src0 = t->src[0]; const ggml_tensor * src1 = t->src[1]; @@ -5179,7 +5399,7 @@ static bool is_mergeable_mul_mat_pair(const ggml_tensor * n1, const ggml_tensor } static bool is_mergeable_mul_mat_id(const ggml_tensor * t) { - if (!t || t->op != GGML_OP_MUL_MAT_ID) return false; + if (t->op != GGML_OP_MUL_MAT_ID) return false; const ggml_tensor * src0 = t->src[0]; return ggml_hexagon_is_repack_type(src0->type); @@ -5213,6 +5433,10 @@ static bool is_mergeable_mul_mat_id_pair(const ggml_tensor * n1, const ggml_tens static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, ggml_cgraph * graph) { auto sess = static_cast(backend->context); + if (sess->last_error > HTP_STATUS_OK) { + return GGML_STATUS_FAILED; + } + HEX_VERBOSE("ggml-hex: %s graph-compute n_nodes %d\n", sess->c_name(), graph->n_nodes); const std::vector * nodes_ptr = nullptr; @@ -5228,6 +5452,8 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg auto * extra = (ggml_hexagon_tensor_extra *) graph->nodes[i]->extra; if (!extra) continue; + extra->flags &= ~GGML_HEXAGON_TENSOR_FUSEABLE; + if (graph->nodes[i]->op == GGML_OP_RMS_NORM && ggml_can_fuse(graph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) { extra->flags |= GGML_HEXAGON_TENSOR_FUSEABLE; } else if (graph->nodes[i]->op == GGML_OP_MUL_MAT || graph->nodes[i]->op == GGML_OP_MUL_MAT_ID) { @@ -5299,6 +5525,10 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg sess->enqueue_op(node); } + if (sess->last_error > HTP_STATUS_OK) { + return GGML_STATUS_FAILED; + } + return GGML_STATUS_SUCCESS; } @@ -5308,7 +5538,10 @@ static void ggml_backend_hexagon_synchronize(ggml_backend_t backend) { HEX_VERBOSE("ggml-hex: %s synchronize\n", sess->c_name()); // Wait until all pending ops complete - sess->flush(); + sess->flush_sync(); + if (sess->last_error > HTP_STATUS_OK) { + GGML_ABORT("ggml-hex: %s synchronize failed : dsp-error %s\n", sess->c_name(), status_to_str(sess->last_error)); + } } enum ggml_hexagon_mem_range_type { @@ -5543,27 +5776,38 @@ static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgr GGML_UNUSED(backend); } +static uint64_t ggml_hexagon_session_key(const ggml_hexagon_session * sess) { + return ((uint64_t) (uint32_t) sess->phys_idx << 32) | (uint32_t) sess->virt_idx; +} + static bool ggml_hexagon_cpy_tensor_async_phys(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) { auto sess_src = static_cast(backend_src->context); auto sess_dst = static_cast(backend_dst->context); auto sbuf_dst = (ggml_hexagon_shared_buffer *) dst->buffer->context; - if (sess_dst->fence_seq == 0) sess_dst->fence_seq = 1; - uint32_t fence_seq = sess_dst->fence_seq++; - if (sess_dst->fence_seq == 0) sess_dst->fence_seq = 1; + if (!sess_src->clone_buffer(sbuf_dst)) { return false; } - volatile uint32_t * fence = (volatile uint32_t *) sbuf_dst->alloc_fence(); + const uint64_t src_key = ggml_hexagon_session_key(sess_src); + auto & fence_slot = sess_dst->cpy_fence_slots[src_key]; + if (!fence_slot) { + fence_slot = (volatile uint32_t *) sess_dst->alloc_fence(1); + } - HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu : seq %u\n", + if (!sess_src->clone_buffer(sess_dst->fence_buf)) { return false; } + + if (++sess_dst->fence_seq == 0) sess_dst->fence_seq = 1; + uint32_t fence_seq = sess_dst->fence_seq; + + HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu : seq 0x%x\n", sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src), fence_seq); - // dummy extra (must be static) + // dummy fence extra (must be static) static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE }; ggml_tensor fence_tensor {}; - fence_tensor.buffer = dst->buffer; + fence_tensor.buffer = &sess_dst->fence_buf->backend_buffer; fence_tensor.extra = &fence_extra; - fence_tensor.data = (void *) fence; + fence_tensor.data = (void *) fence_slot; fence_tensor.type = GGML_TYPE_I32; fence_tensor.ne[0] = 1; fence_tensor.ne[1] = 1; @@ -5576,9 +5820,9 @@ static bool ggml_hexagon_cpy_tensor_async_phys(ggml_backend_t backend_src, ggml_ fence_tensor.op = GGML_OP_NONE; sess_src->enqueue_cpy(src, dst, &fence_tensor, fence_seq); - sess_dst->enqueue_fence(&fence_tensor, fence_seq); + sess_dst->enqueue_fence(&fence_tensor, fence_seq, /* wait = */ true); - sess_dst->add_sync_peer(sess_src); + sess_dst->add_peer(sess_src); return true; } @@ -5586,15 +5830,15 @@ static bool ggml_hexagon_cpy_tensor_async_phys(ggml_backend_t backend_src, ggml_ static bool ggml_hexagon_cpy_tensor_async_virt(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) { auto sess_src = static_cast(backend_src->context); auto sess_dst = static_cast(backend_dst->context); - auto sbuf_dst = (ggml_hexagon_shared_buffer *) dst->buffer->context; + auto sbuf_src = (ggml_hexagon_shared_buffer *) src->buffer->context; - if (!sess_src->clone_buffer(sbuf_dst)) { return false; } + if (!sess_dst->clone_buffer(sbuf_src)) { return false; } HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu\n", sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src)); - sess_src->enqueue_cpy(src, dst); - sess_src->flush(true); + sess_dst->enqueue_cpy(src, dst); + sess_dst->add_peer(sess_src); return true; } @@ -5604,7 +5848,14 @@ static bool ggml_backend_hexagon_cpy_tensor_async(ggml_backend_t backend_src, gg return false; } - *(ggml_hexagon_tensor_extra *) dst->extra = *(const ggml_hexagon_tensor_extra *) src->extra; + // FIXME: ggml-meta needs to call init_tensor on auxiliary tensors + if (!dst->extra) { + ggml_backend_buffer_init_tensor(dst->buffer, dst); + } + + auto * dst_extra = static_cast(dst->extra); + const auto * src_extra = static_cast(src->extra); + dst_extra->flags = src_extra->flags & ~GGML_HEXAGON_TENSOR_FUSEABLE; auto sess_src = static_cast(backend_src->context); auto sess_dst = static_cast(backend_dst->context); @@ -5612,7 +5863,6 @@ static bool ggml_backend_hexagon_cpy_tensor_async(ggml_backend_t backend_src, gg if (sess_src == sess_dst) { HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu\n", sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src)); sess_src->enqueue_cpy(src, dst); - sess_src->flush_batch(); return true; } @@ -5623,8 +5873,30 @@ static bool ggml_backend_hexagon_cpy_tensor_async(ggml_backend_t backend_src, gg } static ggml_backend_event_t ggml_backend_hexagon_device_event_new(ggml_backend_dev_t dev) { + auto dev_ctx = static_cast(dev->context); + auto sess = dev_ctx->session(); + ggml_hexagon_event * hex_event = new ggml_hexagon_event(); - HEX_VERBOSE("ggml-hex: %s event-new : event %p\n", ggml_backend_dev_name(dev), (void *)hex_event); + hex_event->fence_sess = sess; + hex_event->sess = sess; + hex_event->fence_slot = (volatile uint32_t *) sess->alloc_fence(1); + + static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE }; + hex_event->fence_tensor.buffer = &sess->fence_buf->backend_buffer; + hex_event->fence_tensor.extra = &fence_extra; + hex_event->fence_tensor.data = (void *) hex_event->fence_slot; + hex_event->fence_tensor.type = GGML_TYPE_I32; + hex_event->fence_tensor.ne[0] = 1; + hex_event->fence_tensor.ne[1] = 1; + hex_event->fence_tensor.ne[2] = 1; + hex_event->fence_tensor.ne[3] = 1; + hex_event->fence_tensor.nb[0] = sizeof(int32_t); + hex_event->fence_tensor.nb[1] = sizeof(int32_t); + hex_event->fence_tensor.nb[2] = sizeof(int32_t); + hex_event->fence_tensor.nb[3] = sizeof(int32_t); + hex_event->fence_tensor.op = GGML_OP_NONE; + + HEX_VERBOSE("ggml-hex: %s event-new : event %p fence %p\n", ggml_backend_dev_name(dev), (void *)hex_event, (void *)hex_event->fence_slot); return new ggml_backend_event { /* .device = */ dev, @@ -5632,49 +5904,83 @@ static ggml_backend_event_t ggml_backend_hexagon_device_event_new(ggml_backend_d }; } -static void ggml_backend_hexagon_device_event_free(ggml_backend_dev_t dev, ggml_backend_event_t event) { - GGML_UNUSED(dev); - - if (event == nullptr) { +static void ggml_hexagon_event_synchronize(ggml_backend_dev_t dev, ggml_hexagon_event * hex_event) { + if (hex_event->seq == 0) { return; } - ggml_hexagon_event * hex_event = (ggml_hexagon_event *)event->context; + HEX_VERBOSE("ggml-hex: %s event-synchronize : event %p seq 0x%x fence %p\n", + ggml_backend_dev_name(dev), (void *)hex_event, hex_event->seq, (void *)hex_event->fence_slot); + + auto * fence = reinterpret_cast *>(hex_event->fence_slot); + + if ((int32_t)(fence[0].load(std::memory_order_relaxed) - hex_event->seq) < 0) { + hex_event->sess->flush_async(); + } + + while (true) { + if ((int32_t)(fence[0].load(std::memory_order_acquire) - hex_event->seq) >= 0) { + uint32_t status = fence[1].load(std::memory_order_acquire); + if (status > HTP_STATUS_OK) { + GGML_ABORT("ggml-hex: %s event-synchronize failed : dsp-error %s\n", + hex_event->sess->c_name(), status_to_str(status)); + } + break; + } + std::this_thread::yield(); + } +} + +static void ggml_backend_hexagon_device_event_free(ggml_backend_dev_t dev, ggml_backend_event_t event) { + auto * hex_event = static_cast(event->context); + ggml_hexagon_event_synchronize(dev, hex_event); HEX_VERBOSE("ggml-hex: %s event-free : event %p\n", ggml_backend_dev_name(dev), (void *)hex_event); + hex_event->fence_sess->free_fence((void *) hex_event->fence_slot, 1); delete hex_event; delete event; } static void ggml_backend_hexagon_device_event_synchronize(ggml_backend_dev_t dev, ggml_backend_event_t event) { - GGML_UNUSED(dev); - - ggml_hexagon_event * hex_event = (ggml_hexagon_event *)event->context; - HEX_VERBOSE("ggml-hex: %s event-synchronize : event %p seq %llu\n", - ggml_backend_dev_name(dev), (void *)hex_event, (unsigned long long)hex_event->seq); - if (hex_event->sess != nullptr) { - hex_event->sess->wait_event(hex_event->seq); - } + auto * hex_event = static_cast(event->context); + ggml_hexagon_event_synchronize(dev, hex_event); } static void ggml_backend_hexagon_event_record(ggml_backend_t backend, ggml_backend_event_t event) { auto sess = static_cast(backend->context); - ggml_hexagon_event * hex_event = (ggml_hexagon_event *)event->context; + auto hex_event = static_cast(event->context); + if (++sess->fence_seq == 0) sess->fence_seq = 1; hex_event->sess = sess; - hex_event->seq = sess->record_event(); - HEX_VERBOSE("ggml-hex: %s event-record : event %p seq %llu\n", - sess->c_name(), (void *)hex_event, (unsigned long long)hex_event->seq); + hex_event->seq = sess->fence_seq; + + sess->enqueue_fence(&hex_event->fence_tensor, hex_event->seq, /* wait = */ false); + + HEX_VERBOSE("ggml-hex: %s event-record : event %p seq 0x%x fence %p\n", + sess->c_name(), (void *)hex_event, hex_event->seq, (void *)hex_event->fence_slot); } static void ggml_backend_hexagon_event_wait(ggml_backend_t backend, ggml_backend_event_t event) { - GGML_UNUSED(backend); + auto sess = static_cast(backend->context); + auto hex_event = static_cast(event->context); - ggml_hexagon_event * hex_event = (ggml_hexagon_event *)event->context; - if (hex_event->sess != nullptr) { - HEX_VERBOSE("ggml-hex: %s event-wait : event %p seq %llu\n", - hex_event->sess->c_name(), (void *)hex_event, (unsigned long long)hex_event->seq); - hex_event->sess->wait_event(hex_event->seq); + if (hex_event->seq == 0) { + return; } + + HEX_VERBOSE("ggml-hex: %s event-wait : event %p seq 0x%x fence %p\n", + sess->c_name(), (void *)hex_event, hex_event->seq, (void *)hex_event->fence_slot); + + // same physical NPU runs sequentially in FIFO order + if (sess->phys_idx == hex_event->sess->phys_idx) { + if (sess != hex_event->sess) { + sess->add_peer(hex_event->sess); + } + return; + } + + sess->clone_buffer(hex_event->fence_sess->fence_buf); + sess->add_peer(hex_event->sess); + sess->enqueue_fence(&hex_event->fence_tensor, hex_event->seq, /* wait = */ true); } static void ggml_backend_hexagon_set_tensor_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) { @@ -5688,7 +5994,10 @@ static void ggml_backend_hexagon_get_tensor_async(ggml_backend_t backend, const auto sess = static_cast(backend->context); HEX_VERBOSE("ggml-hex: %s get-tensor-async %s : data %p offset %zu size %zu usage %d\n", sess->c_name(), tensor->name, data, offset, size, tensor->buffer ? (int) tensor->buffer->usage : -1); - sess->flush(true); + sess->flush_sync(); + if (sess->last_error > HTP_STATUS_OK) { + GGML_ABORT("ggml-hex: %s get-tensor-async failed : dsp-error %s\n", sess->c_name(), status_to_str(sess->last_error)); + } ggml_backend_tensor_get(tensor, data, offset, size); } @@ -5717,7 +6026,10 @@ static void ggml_backend_hexagon_get_tensor_2d_async(ggml_backend_t backend, auto sess = static_cast(backend->context); HEX_VERBOSE("ggml-hex: %s get-tensor-2d-async %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d\n", sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, tensor->buffer ? (int) tensor->buffer->usage : -1); - sess->flush(true); + sess->flush_sync(); + if (sess->last_error > HTP_STATUS_OK) { + GGML_ABORT("ggml-hex: %s get-tensor-2d-async failed : dsp-error %s\n", sess->c_name(), status_to_str(sess->last_error)); + } ggml_backend_tensor_get_2d(tensor, data, offset, size, n_copies, stride_tensor, stride_data); } @@ -6083,13 +6395,7 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons static bool ggml_backend_hexagon_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { auto dev_ctx = static_cast(dev->context); - // Technically we can clone hexagon buffers from any session but for some reason the output is garbled with layer-split, - // tensor-split works correctly, so it needs mode debugging and investigation. For now accept only our own buffers. -#if 0 - bool supp = (buft->iface.get_alignment == ggml_backend_hexagon_buffer_type_get_alignment); -#else bool supp = (buft == &dev_ctx->host_buffer_type) || (buft == &dev_ctx->buffer_type); -#endif HEX_VERBOSE("ggml-hex: %s device-supports-buft %s %s\n", dev_ctx->c_name(), ggml_backend_buft_name(buft), supp ? "yes" : "no"); return supp; @@ -6122,6 +6428,19 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) { // Create devices for (size_t i = 0; i < opt_ndev; i++) { + const auto & cfg = opt_device_configs[i]; + if (cfg.mdev_group.empty()) { + GGML_LOG_INFO("ggml-hex: device %zu: %s (phys=%d, virt=%d, domain=%s:%d)\n", + i, cfg.name.c_str(), cfg.physical_idx, cfg.virtual_idx, cfg.domain_name.c_str(), cfg.domain_id); + } else { + std::string peers_str; + for (const auto & p : cfg.mdev_group) { + if (!peers_str.empty()) peers_str += ", "; + peers_str += p.name + " (phys=" + std::to_string(p.physical_idx) + ")"; + } + GGML_LOG_INFO("ggml-hex: device %zu: %s (phys=%d, virt=%d, domain=%s:%d) [mdev peers: %s]\n", + i, cfg.name.c_str(), cfg.physical_idx, cfg.virtual_idx, cfg.domain_name.c_str(), cfg.domain_id, peers_str.c_str()); + } devices[i].iface = ggml_backend_hexagon_device_i; devices[i].reg = reg; devices[i].context = new ggml_backend_hexagon_device_context(i, opt_device_configs[i], &devices[i]); @@ -6172,17 +6491,51 @@ static void * ggml_backend_hexagon_comm_init(ggml_backend_t * backends, size_t n } } + for (size_t i = 0; i < n_backends; i++) { + auto sess_i = static_cast(backends[i]->context); + for (size_t j = i + 1; j < n_backends; j++) { + auto sess_j = static_cast(backends[j]->context); + if (sess_i->phys_idx == sess_j->phys_idx) { + return nullptr; + } + } + } + auto * ctx = new ggml_backend_hexagon_comm_context(); ctx->backends.assign(backends, backends + n_backends); ctx->n_backends = n_backends; - ctx->fence_seq = (((uintptr_t) ctx) & 0xFFFF) | 1; + + static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE }; + for (size_t i = 0; i < n_backends; i++) { + auto sess_i = static_cast(backends[i]->context); + ctx->fence_slots[i] = (volatile uint32_t *) sess_i->alloc_fence(1); + ctx->fence_tensors[i] = {}; + ctx->fence_tensors[i].buffer = &sess_i->fence_buf->backend_buffer; + ctx->fence_tensors[i].extra = &fence_extra; + ctx->fence_tensors[i].data = (void *) ctx->fence_slots[i]; + ctx->fence_tensors[i].type = GGML_TYPE_I32; + ctx->fence_tensors[i].ne[0] = 4; + ctx->fence_tensors[i].ne[1] = 1; + ctx->fence_tensors[i].ne[2] = 1; + ctx->fence_tensors[i].ne[3] = 1; + ctx->fence_tensors[i].nb[0] = sizeof(int32_t); + ctx->fence_tensors[i].nb[1] = sizeof(int32_t); + ctx->fence_tensors[i].nb[2] = sizeof(int32_t); + ctx->fence_tensors[i].nb[3] = sizeof(int32_t); + ctx->fence_tensors[i].op = GGML_OP_NONE; + } return ctx; } static void ggml_backend_hexagon_comm_free(void * comm_ctx_v) { if (!comm_ctx_v) return; - delete static_cast(comm_ctx_v); + auto * ctx = static_cast(comm_ctx_v); + for (size_t i = 0; i < ctx->n_backends; i++) { + auto sess_i = static_cast(ctx->backends[i]->context); + sess_i->free_fence((void *) ctx->fence_slots[i], 1); + } + delete ctx; } static bool ggml_backend_hexagon_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tensor ** tensors) { @@ -6192,6 +6545,16 @@ static bool ggml_backend_hexagon_comm_allreduce_tensor(void * comm_ctx_v, struct if (n_backends < 2 || n_backends > 4) return false; + for (size_t i = 0; i < n_backends; i++) { + auto sess_i = static_cast(comm_ctx->backends[i]->context); + for (size_t j = i + 1; j < n_backends; j++) { + auto sess_j = static_cast(comm_ctx->backends[j]->context); + if (sess_i->phys_idx == sess_j->phys_idx) { + return false; + } + } + } + for (size_t i = 0; i < n_backends; i++) { if (!tensors[i] || !tensors[i]->buffer || !ggml_backend_buffer_is_hexagon(tensors[i]->buffer)) { return false; @@ -6219,42 +6582,28 @@ static bool ggml_backend_hexagon_comm_allreduce_tensor(void * comm_ctx_v, struct } } - if (comm_ctx->fence_seq == 0) comm_ctx->fence_seq = 1; - uint32_t fence_seq_entry = comm_ctx->fence_seq++; - if (comm_ctx->fence_seq == 0) comm_ctx->fence_seq = 1; - uint32_t fence_seq_exit = comm_ctx->fence_seq++; - if (comm_ctx->fence_seq == 0) comm_ctx->fence_seq = 1; - - volatile uint32_t * fences[GGML_HEXAGON_MAX_SESSIONS]; - for (size_t i = 0; i < n_backends; i++) { - auto sbuf = (ggml_hexagon_shared_buffer *) tensors[i]->buffer->context; - fences[i] = (volatile uint32_t *) sbuf->alloc_fence(); + uint32_t max_seq = static_cast(comm_ctx->backends[0]->context)->fence_seq; + for (size_t i = 1; i < n_backends; i++) { + auto sess_i = static_cast(comm_ctx->backends[i]->context); + if ((int32_t)(sess_i->fence_seq - max_seq) > 0) { + max_seq = sess_i->fence_seq; + } } + if (++max_seq == 0) max_seq = 1; + uint32_t fence_seq_entry = max_seq; + if (++max_seq == 0) max_seq = 1; + uint32_t fence_seq_exit = max_seq; - static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE }; - ggml_tensor fence_tensors[GGML_HEXAGON_MAX_SESSIONS]; for (size_t i = 0; i < n_backends; i++) { - fence_tensors[i] = {}; - fence_tensors[i].buffer = tensors[i]->buffer; - fence_tensors[i].extra = &fence_extra; - fence_tensors[i].data = (void *) fences[i]; - fence_tensors[i].type = GGML_TYPE_I32; - fence_tensors[i].ne[0] = 4; - fence_tensors[i].ne[1] = 1; - fence_tensors[i].ne[2] = 1; - fence_tensors[i].ne[3] = 1; - fence_tensors[i].nb[0] = sizeof(int32_t); - fence_tensors[i].nb[1] = sizeof(int32_t); - fence_tensors[i].nb[2] = sizeof(int32_t); - fence_tensors[i].nb[3] = sizeof(int32_t); - fence_tensors[i].op = GGML_OP_NONE; + auto sess_i = static_cast(comm_ctx->backends[i]->context); + sess_i->fence_seq = max_seq; } std::vector data_tensors(n_backends); std::vector sync_tensors(n_backends); for (size_t i = 0; i < n_backends; i++) { data_tensors[i] = tensors[i]; - sync_tensors[i] = &fence_tensors[i]; + sync_tensors[i] = &comm_ctx->fence_tensors[i]; } for (size_t r = 0; r < n_backends; r++) { @@ -6262,7 +6611,7 @@ static bool ggml_backend_hexagon_comm_allreduce_tensor(void * comm_ctx_v, struct sess->enqueue_allreduce(tensors[r], data_tensors, sync_tensors, (uint32_t) r, (uint32_t) n_backends, fence_seq_entry, fence_seq_exit); for (size_t j = 0; j < n_backends; j++) { if (r != j) { - sess->add_sync_peer(static_cast(comm_ctx->backends[j]->context)); + sess->add_peer(static_cast(comm_ctx->backends[j]->context)); } } } @@ -6270,8 +6619,23 @@ static bool ggml_backend_hexagon_comm_allreduce_tensor(void * comm_ctx_v, struct return true; } +static ggml_backend_buffer_type_t ggml_backend_hexagon_split_buffer_type(int main_device, const float * tensor_split) { + GGML_UNUSED(tensor_split); + auto reg = ggml_backend_hexagon_reg(); + auto dev = ggml_backend_reg_dev_get(reg, main_device); + if (!dev) { + dev = ggml_backend_reg_dev_get(reg, 0); + } + if (!dev) return nullptr; + auto dev_ctx = static_cast(dev->context); + return &dev_ctx->buffer_type; +} + static void * ggml_backend_hexagon_get_proc_address(ggml_backend_reg_t reg, const char * name) { GGML_UNUSED(reg); + if (strcmp(name, "ggml_backend_split_buffer_type") == 0) { + return (void *) ggml_backend_hexagon_split_buffer_type; + } if (strcmp(name, "ggml_backend_comm_init") == 0) { return (void *) ggml_backend_hexagon_comm_init; } @@ -6304,6 +6668,41 @@ template std::string vec_to_str(std::vector v) { return str; } +static void ggml_hexagon_resolve_device_domain(ggml_hexagon_device_config & cfg, bool discovery_supported, const std::unordered_map & cdsp_map) { + if (discovery_supported) { + auto it = cdsp_map.find(cfg.physical_idx); + if (it != cdsp_map.end()) { + cfg.domain_id = it->second.id; + cfg.domain_name = it->second.name; + } else { + GGML_LOG_ERROR("ggml-hex: physical CDSP core %d not found on device (%zu CDSP core(s) available)\n", + cfg.physical_idx, cdsp_map.size()); + cfg.domain_id = -1; + cfg.domain_name = ""; + } + } else { + switch (cfg.physical_idx) { + case 0: + cfg.domain_id = 3; + cfg.domain_name = CDSP_DOMAIN_NAME; + break; + case 1: + cfg.domain_id = 4; + cfg.domain_name = "cdsp1"; + break; + default: + GGML_LOG_ERROR("ggml-hex: physical CDSP core %d not supported without dynamic discovery\n", + cfg.physical_idx); + cfg.domain_id = -1; + cfg.domain_name = ""; + break; + } + } + for (auto & sub_cfg : cfg.mdev_group) { + ggml_hexagon_resolve_device_domain(sub_cfg, discovery_supported, cdsp_map); + } +} + // Enumerate NPU (aka CDSP) domains via FASTRPC_GET_DOMAINS if supported, // and populate domain_id and domain_name for all configured devices. static void ggml_hexagon_discover_devices() { @@ -6350,36 +6749,7 @@ static void ggml_hexagon_discover_devices() { // Populate domain IDs and names for all configured devices for (size_t i = 0; i < opt_ndev; i++) { - auto & cfg = opt_device_configs[i]; - if (discovery_supported) { - auto it = cdsp_map.find(cfg.physical_idx); - if (it != cdsp_map.end()) { - cfg.domain_id = it->second.id; - cfg.domain_name = it->second.name; - } else { - GGML_LOG_ERROR("ggml-hex: physical CDSP core %d not found on device (%zu CDSP core(s) available)\n", - cfg.physical_idx, cdsp_map.size()); - cfg.domain_id = -1; - cfg.domain_name = ""; - } - } else { - switch (cfg.physical_idx) { - case 0: - cfg.domain_id = 3; - cfg.domain_name = CDSP_DOMAIN_NAME; - break; - case 1: - cfg.domain_id = 4; - cfg.domain_name = "cdsp1"; - break; - default: - GGML_LOG_ERROR("ggml-hex: physical CDSP core %d not supported without dynamic discovery\n", - cfg.physical_idx); - cfg.domain_id = -1; - cfg.domain_name = ""; - break; - } - } + ggml_hexagon_resolve_device_domain(opt_device_configs[i], discovery_supported, cdsp_map); } } @@ -6487,21 +6857,126 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { opt_device_configs[i].physical_idx = 0; opt_device_configs[i].virtual_idx = (int)i; opt_device_configs[i].name = "HTP" + std::to_string(i); + opt_device_configs[i].mdev_group.clear(); } } else { std::string s_devices(str_devices); - std::stringstream ss(s_devices); - std::string item; - opt_ndev = 0; - while (std::getline(ss, item, ',')) { - size_t start = item.find_first_not_of(" \t\r\n"); - size_t end = item.find_last_not_of(" \t\r\n"); - if (start == std::string::npos) { - continue; + std::vector items; + std::string curr_item; + int bracket_depth = 0; + for (char ch : s_devices) { + if (ch == '[') { + bracket_depth++; + curr_item += ch; + } else if (ch == ']') { + if (bracket_depth > 0) bracket_depth--; + curr_item += ch; + } else if (ch == ',' && bracket_depth == 0) { + size_t s = curr_item.find_first_not_of(" \t\r\n"); + size_t e = curr_item.find_last_not_of(" \t\r\n"); + if (s != std::string::npos) { + items.push_back(curr_item.substr(s, e - s + 1)); + } + curr_item.clear(); + } else { + curr_item += ch; } - item = item.substr(start, end - start + 1); + } + size_t s = curr_item.find_first_not_of(" \t\r\n"); + size_t e = curr_item.find_last_not_of(" \t\r\n"); + if (s != std::string::npos) { + items.push_back(curr_item.substr(s, e - s + 1)); + } - if (item.rfind("HTP", 0) == 0) { + opt_ndev = 0; + for (const auto & item : items) { + size_t b_open = item.find('['); + size_t b_close = item.rfind(']'); + + if (b_open != std::string::npos && b_close != std::string::npos && b_close > b_open) { + // Grouped / composite syntax: Name[phys_spec:virt] or Name[phys_spec] + std::string dev_name = item.substr(0, b_open); + std::string content = item.substr(b_open + 1, b_close - b_open - 1); + + int virt = 0; + std::string phys_spec = content; + size_t colon_pos = content.find(':'); + if (colon_pos != std::string::npos) { + phys_spec = content.substr(0, colon_pos); + try { + virt = std::stoi(content.substr(colon_pos + 1)); + } catch (...) { + virt = 0; + } + } else { + size_t dev_colon = dev_name.find(':'); + if (dev_colon != std::string::npos) { + try { + virt = std::stoi(dev_name.substr(dev_colon + 1)); + } catch (...) { + virt = 0; + } + } + } + + // Parse physical indices from phys_spec (e.g. 0-1, 0,1, 0-3, etc.) + std::vector phys_list; + std::stringstream pss(phys_spec); + std::string p_part; + while (std::getline(pss, p_part, ',')) { + size_t ps = p_part.find_first_not_of(" \t\r\n"); + size_t pe = p_part.find_last_not_of(" \t\r\n"); + if (ps == std::string::npos) continue; + p_part = p_part.substr(ps, pe - ps + 1); + + size_t dash_pos = p_part.find('-'); + if (dash_pos != std::string::npos) { + try { + int p_start = std::stoi(p_part.substr(0, dash_pos)); + int p_end = std::stoi(p_part.substr(dash_pos + 1)); + for (int p = p_start; p <= p_end; p++) { + if (std::find(phys_list.begin(), phys_list.end(), p) == phys_list.end()) { + phys_list.push_back(p); + } + } + } catch (...) { + GGML_LOG_WARN("ggml-hex: failed to parse physical range in '%s'\n", p_part.c_str()); + } + } else { + try { + int p = std::stoi(p_part); + if (std::find(phys_list.begin(), phys_list.end(), p) == phys_list.end()) { + phys_list.push_back(p); + } + } catch (...) { + GGML_LOG_WARN("ggml-hex: failed to parse physical index in '%s'\n", p_part.c_str()); + } + } + } + + if (phys_list.empty()) { + phys_list.push_back(0); + } + + if (opt_ndev < GGML_HEXAGON_MAX_SESSIONS) { + auto & cfg = opt_device_configs[opt_ndev]; + cfg.name = dev_name; + cfg.physical_idx = phys_list[0]; + cfg.virtual_idx = virt; + cfg.mdev_group.clear(); + + for (size_t k = 1; k < phys_list.size(); k++) { + ggml_hexagon_device_config sub_cfg; + sub_cfg.physical_idx = phys_list[k]; + sub_cfg.virtual_idx = virt; + sub_cfg.name = "HTP" + std::to_string(phys_list[k]) + ":" + std::to_string(virt); + cfg.mdev_group.push_back(sub_cfg); + } + opt_ndev++; + } else { + GGML_LOG_WARN("ggml-hex: max sessions limit reached (%d), ignoring device %s\n", GGML_HEXAGON_MAX_SESSIONS, item.c_str()); + } + } else if (item.rfind("HTP", 0) == 0) { std::string rest = item.substr(3); size_t colon_pos = rest.find(':'); int phys = 0; @@ -6525,6 +7000,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { opt_device_configs[opt_ndev].name = colon_pos == std::string::npos ? "HTP" + std::to_string(phys) : "HTP" + std::to_string(phys) + ":" + std::to_string(virt); + opt_device_configs[opt_ndev].mdev_group.clear(); opt_ndev++; } else { GGML_LOG_WARN("ggml-hex: max sessions limit reached (%d), ignoring device %s\n", GGML_HEXAGON_MAX_SESSIONS, item.c_str()); @@ -6539,6 +7015,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { opt_device_configs[0].physical_idx = 0; opt_device_configs[0].virtual_idx = 0; opt_device_configs[0].name = "HTP0"; + opt_device_configs[0].mdev_group.clear(); } #if defined(__ANDROID__) diff --git a/ggml/src/ggml-hexagon/htp-opnode.h b/ggml/src/ggml-hexagon/htp-opnode.h index b083e2671..ef7b5184f 100644 --- a/ggml/src/ggml-hexagon/htp-opnode.h +++ b/ggml/src/ggml-hexagon/htp-opnode.h @@ -344,6 +344,12 @@ struct htp_opformat { } else if (htp_op_is_unary(node.opcode)) { const auto * kparams = (const struct htp_unary_kernel_params *) node.kernel_params; snprintf(str, max_size, "%s vtcm %d", kparams->col_tile ? "wide-row" : "row-block", (int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_MDEV_GROUP && node.node) { + snprintf(str, max_size, "idx %d count %d", (int) node.node->op_params[0], (int) node.dst()->ne[1]); + } else if ((node.opcode == HTP_OP_FENCE || node.opcode == HTP_OP_CPY_FENCE) && node.node) { + snprintf(str, max_size, "seq 0x%x", (uint32_t) node.node->op_params[0]); + } else if (node.opcode == HTP_OP_ALLREDUCE && node.node) { + snprintf(str, max_size, "seq 0x%x -> 0x%x", (uint32_t) node.node->op_params[0], (uint32_t) node.node->op_params[1]); } else { snprintf(str, max_size, "----"); } diff --git a/ggml/src/ggml-hexagon/htp/act-ops.c b/ggml/src/ggml-hexagon/htp/act-ops.c index ac00b447d..5fff372f2 100644 --- a/ggml/src/ggml-hexagon/htp/act-ops.c +++ b/ggml/src/ggml-hexagon/htp/act-ops.c @@ -3,7 +3,6 @@ #pragma clang diagnostic ignored "-Wunused-but-set-variable" #include -#include #include #include @@ -15,7 +14,7 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "hex-common.h" #include "htp-tensor.h" #include "htp-vtcm.h" @@ -80,6 +79,7 @@ struct htp_act_context { uint32_t block; uint32_t src0_nrows; uint32_t src0_nrows_per_thread; + uint32_t row_start; int nc; uint8_t * vtcm_src0; @@ -329,104 +329,104 @@ static void geglu_f32(const float * restrict src0, } } -#define DEFINE_GLU_PER_THREAD(NAME, OP_STR, CORE_EXPR) \ - static void glu_##NAME##_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { \ - struct htp_act_context * actx = (struct htp_act_context *) data; \ - htp_act_preamble; \ - \ - struct htp_thread_trace * tr = actx->octx->ctx ? &actx->octx->ctx->trace[ith] : NULL; \ - \ - size_t src0_row_size = actx->src0_row_size; \ - size_t src1_row_size = actx->src1_row_size; \ - size_t dst_row_size = actx->dst_row_size; \ - \ - size_t src0_row_stride = actx->src0_row_stride; \ - size_t src1_row_stride = actx->src1_row_stride; \ - \ - const uint32_t src0_nrows = actx->src0_nrows; \ - const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; \ - \ - const uint32_t src0_start_row = src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \ - \ - /* no work for this thread */ \ - if (src0_start_row >= src0_end_row) { \ - return; \ - } \ - \ - const uint8_t * restrict data_src0 = actx->data_src0; \ - const uint8_t * restrict data_src1 = actx->data_src1; \ - uint8_t * restrict data_dst = actx->data_dst; \ - \ - const size_t src0_row_size_aligned = actx->src0_row_size_aligned; \ - const size_t src1_row_size_aligned = actx->src1_row_size_aligned; \ - const size_t dst_row_size_aligned = actx->dst_row_size_aligned; \ - \ - uint8_t * restrict src0_spad_data = actx->vtcm_src0 + (ith * actx->vtcm_src0_size_per_thread); \ - uint8_t * restrict src1_spad_data = actx->vtcm_src1 + (ith * actx->vtcm_src1_size_per_thread); \ - uint8_t * restrict dst_spad_data = actx->vtcm_dst + (ith * actx->vtcm_dst_size_per_thread); \ - \ - size_t src0_spad_half_size = actx->src0_spad_half_size; \ - size_t src1_spad_half_size = actx->src1_spad_half_size; \ - size_t dst_spad_half_size = actx->dst_spad_half_size; \ - \ - const int BLOCK = actx->block; \ - if (BLOCK == 0) { \ - FARF(ERROR, \ - OP_STR \ - " : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", \ - actx->vtcm_src0_size_per_thread, src0_row_size_aligned); \ - return; \ - } \ - \ - dma_queue * dma_queue = actx->octx->ctx->dma[ith]; \ - \ - /* See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379 */ \ - for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { \ - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \ - \ - /* Dummy DMA transation for sequencing (interleaving dst,src,dst,...) */ \ - dma_queue_push_vtcm_to_ddr(dma_queue, \ - dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), \ - dst_row_size, dst_row_size_aligned, 0); \ - \ - dma_queue_push( \ - dma_queue, \ - dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_stride)), \ - src0_row_size_aligned, src0_row_stride, src0_row_size, block_size); \ - dma_queue_push( \ - dma_queue, \ - dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_stride)), \ - src1_row_size_aligned, src1_row_stride, src1_row_size, block_size); \ - } \ - \ - for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { \ - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \ - \ - float * dst_spad = (float *) dma_queue_pop(dma_queue).src; \ - float * src0_spad = (float *) dma_queue_pop(dma_queue).dst; \ - float * src1_spad = (float *) dma_queue_pop(dma_queue).dst; \ - \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ - CORE_EXPR; \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ - \ - dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), \ - dst_row_size, dst_row_size_aligned, block_size); \ - \ - /* prefetch N+2 loop iteration if any */ \ - const uint32_t pref_block = (ir + BLOCK * 2); \ - if (pref_block < src0_end_row) { \ - const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); \ - dma_queue_push(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_stride)), \ - src0_row_size_aligned, src0_row_stride, src0_row_size, pref_block_size); \ - dma_queue_push(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_stride)), \ - src1_row_size_aligned, src1_row_stride, src1_row_size, pref_block_size); \ - } \ - } \ - \ - dma_queue_flush(dma_queue); \ - \ +#define DEFINE_GLU_PER_THREAD(NAME, OP_STR, CORE_EXPR) \ + static void glu_##NAME##_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_act_context * actx = (struct htp_act_context *) data; \ + htp_act_preamble; \ + \ + struct htp_thread_trace * tr = actx->octx->ctx ? &actx->octx->ctx->trace[ith] : NULL; \ + \ + size_t src0_row_size = actx->src0_row_size; \ + size_t src1_row_size = actx->src1_row_size; \ + size_t dst_row_size = actx->dst_row_size; \ + \ + size_t src0_row_stride = actx->src0_row_stride; \ + size_t src1_row_stride = actx->src1_row_stride; \ + \ + const uint32_t src0_nrows = actx->src0_nrows; \ + const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; \ + \ + const uint32_t src0_start_row = actx->row_start + src0_nrows_per_thread * ith; \ + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, actx->row_start + src0_nrows); \ + \ + /* no work for this thread */ \ + if (src0_start_row >= src0_end_row) { \ + return; \ + } \ + \ + const uint8_t * restrict data_src0 = actx->data_src0; \ + const uint8_t * restrict data_src1 = actx->data_src1; \ + uint8_t * restrict data_dst = actx->data_dst; \ + \ + const size_t src0_row_size_aligned = actx->src0_row_size_aligned; \ + const size_t src1_row_size_aligned = actx->src1_row_size_aligned; \ + const size_t dst_row_size_aligned = actx->dst_row_size_aligned; \ + \ + uint8_t * restrict src0_spad_data = actx->vtcm_src0 + (ith * actx->vtcm_src0_size_per_thread); \ + uint8_t * restrict src1_spad_data = actx->vtcm_src1 + (ith * actx->vtcm_src1_size_per_thread); \ + uint8_t * restrict dst_spad_data = actx->vtcm_dst + (ith * actx->vtcm_dst_size_per_thread); \ + \ + size_t src0_spad_half_size = actx->src0_spad_half_size; \ + size_t src1_spad_half_size = actx->src1_spad_half_size; \ + size_t dst_spad_half_size = actx->dst_spad_half_size; \ + \ + const int BLOCK = actx->block; \ + if (BLOCK == 0) { \ + FARF(ERROR, \ + OP_STR \ + " : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", \ + actx->vtcm_src0_size_per_thread, src0_row_size_aligned); \ + return; \ + } \ + \ + dma_queue * dma_queue = actx->octx->ctx->dma[ith]; \ + \ + /* See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379 */ \ + for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { \ + const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \ + \ + /* Dummy DMA transation for sequencing (interleaving dst,src,dst,...) */ \ + dma_queue_push_vtcm_to_ddr(dma_queue, \ + dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), \ + dst_row_size, dst_row_size_aligned, 0); \ + \ + dma_queue_push( \ + dma_queue, \ + dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_stride)), \ + src0_row_size_aligned, src0_row_stride, src0_row_size, block_size); \ + dma_queue_push( \ + dma_queue, \ + dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_stride)), \ + src1_row_size_aligned, src1_row_stride, src1_row_size, block_size); \ + } \ + \ + for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { \ + const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \ + \ + float * dst_spad = (float *) dma_queue_pop(dma_queue).src; \ + float * src0_spad = (float *) dma_queue_pop(dma_queue).dst; \ + float * src1_spad = (float *) dma_queue_pop(dma_queue).dst; \ + \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ + CORE_EXPR; \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ + \ + dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), \ + dst_row_size, dst_row_size_aligned, block_size); \ + \ + /* prefetch N+2 loop iteration if any */ \ + const uint32_t pref_block = (ir + BLOCK * 2); \ + if (pref_block < src0_end_row) { \ + const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); \ + dma_queue_push(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_stride)), \ + src0_row_size_aligned, src0_row_stride, src0_row_size, pref_block_size); \ + dma_queue_push(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_stride)), \ + src1_row_size_aligned, src1_row_stride, src1_row_size, pref_block_size); \ + } \ + } \ + \ + dma_queue_flush(dma_queue); \ + \ } DEFINE_GLU_PER_THREAD(swiglu, "swiglu-f32", swiglu_f32(src0_spad, src1_spad, dst_spad, block_size, actx)) @@ -473,14 +473,30 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { } const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); + const size_t dst_row_size = dst->ne[0] * SIZEOF_FP32; + + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; // row_size = bytes of useful data per row (what the kernel touches / what DMA copies). // row_stride = bytes between successive rows in DDR (may exceed row_size for non-contig src). - const size_t nc_bytes = dst->ne[0] * SIZEOF_FP32; - const size_t src0_row_size = nc_bytes; - const size_t src1_row_size = nc_bytes; - const size_t dst_row_size = nc_bytes; + const size_t nc_bytes = dst_row_size; + const size_t src0_row_size = nc_bytes; + const size_t src1_row_size = nc_bytes; const size_t src0_row_stride = src0->nb[1]; const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1]; @@ -518,7 +534,7 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { struct htp_act_context actx; actx.octx = octx; - actx.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; + actx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); actx.src0_row_size = src0_row_size; actx.src1_row_size = src1_row_size; @@ -545,7 +561,8 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { actx.dst_spad_half_size = L.dst_bytes_per_thread / 2; actx.block = actx.src0_spad_half_size / actx.src0_row_size_aligned; - actx.src0_nrows = src0_nrows; + actx.src0_nrows = nrows; + actx.row_start = row_start; actx.nc = dst->ne[0]; @@ -570,7 +587,7 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { actx.data_src1 = data_src1; actx.data_dst = (uint8_t *) dst->data; - worker_pool_run_func(octx->ctx->worker_pool, act_op_func, &actx, n_threads); + work_queue_run(octx->ctx->work_queue, act_op_func, &actx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.c b/ggml/src/ggml-hexagon/htp/allreduce-ops.c index d35f685a6..d6e7f0d10 100644 --- a/ggml/src/ggml-hexagon/htp/allreduce-ops.c +++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.c @@ -17,6 +17,7 @@ #include "hex-dma.h" #include "hex-profile.h" #include "allreduce-ops.h" +#include "htp-fence.h" struct htp_allreduce_context { struct htp_ops_context * octx; @@ -242,7 +243,42 @@ DEFINE_ALLREDUCE_THREAD_DMA_2D(add_f32, float, hvx_add_f32_aaa, 1, 0) DEFINE_ALLREDUCE_THREAD_DMA_2D(add_bcast_f16, __fp16, hvx_add_f16_aaa, 1, 1) DEFINE_ALLREDUCE_THREAD_DMA_2D(add_bcast_f32, float, hvx_add_f32_aaa, 1, 1) +static int validate_allreduce( + struct htp_ops_context * octx, + const struct htp_allreduce_kernel_params * kparams, + uint32_t n_ranks +) { + if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + if (kparams->vtcm_size_per_thread <= 0 || kparams->vtcm_size <= 0) { + return HTP_STATUS_INVAL_PARAMS; + } + + const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD); + const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count( + n_ranks, octx->n_threads, has_add, kparams->is_row_bcast != 0); + const size_t vtcm_size = n_vtcm_buffers * (size_t) kparams->vtcm_size_per_thread; + if (vtcm_size != (size_t) kparams->vtcm_size) { + return HTP_STATUS_INVAL_PARAMS; + } + if (vtcm_size > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + if (octx->dst->type != HTP_TYPE_F16 && octx->dst->type != HTP_TYPE_F32) { + return HTP_STATUS_NO_SUPPORT; + } + + return HTP_STATUS_OK; +} + int op_allreduce(struct htp_ops_context * octx) { + if (octx->ctx->mdev.count > 1 && octx->ctx->mdev.idx > 0) { + return HTP_STATUS_OK; + } + const struct htp_allreduce_kernel_params * kparams = (const struct htp_allreduce_kernel_params *) octx->kernel_params; const struct htp_tensor * dst = octx->dst; @@ -253,38 +289,53 @@ int op_allreduce(struct htp_ops_context * octx) { return HTP_STATUS_INVAL_PARAMS; } - if (dst->type != HTP_TYPE_F16 && dst->type != HTP_TYPE_F32) { - return HTP_STATUS_NO_SUPPORT; - } - - const uint32_t nelem = dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; const uint32_t fence_seq_entry = (uint32_t) octx->op_params[0]; const uint32_t fence_seq_exit = (uint32_t) octx->op_params[1]; + const struct htp_tensor * my_sync = octx->src[n_ranks + rank]; + atomic_uint * my_fence = (atomic_uint *) (uintptr_t) my_sync->data; + + const int status = validate_allreduce(octx, kparams, n_ranks); + if (status != HTP_STATUS_OK) { + if (status == HTP_STATUS_NO_SUPPORT) { + FARF(ERROR, "ggml-hex: allreduce unsupported type %d : rank %u\n", dst->type, rank); + } + htp_fence_write(my_fence, fence_seq_exit, status); + return status; + } + + const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD); + const uint32_t nelem = dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; + // 1. Entry Barrier: Synchronize all ranks before reading struct htp_thread_trace * tr0 = &octx->ctx->trace[0]; htp_trace_event_start(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry); - const struct htp_tensor * my_sync = octx->src[n_ranks + rank]; - atomic_uint * my_fence = (atomic_uint *) my_sync->data; - - atomic_store(&my_fence[0], fence_seq_entry); - asm volatile ("syncht" : : : "memory"); - Q6_dccleaninva_A((void *) my_fence); + htp_fence_write(my_fence, fence_seq_entry, octx->status); for (uint32_t j = 0; j < n_ranks; j++) { if (j == rank) continue; const struct htp_tensor * peer_sync = octx->src[n_ranks + j]; - atomic_uint * peer_fence = (atomic_uint *) peer_sync->data; + atomic_uint * peer_fence = (atomic_uint *) (uintptr_t) peer_sync->data; uint64_t spins = 0; while (1) { - Q6_dccleaninva_A((void *) peer_fence); - uint32_t val = atomic_load(&peer_fence[0]); - if (val == fence_seq_entry || val == fence_seq_exit) { + uint32_t peer_seq; + uint32_t peer_status; + htp_fence_read(peer_fence, &peer_seq, &peer_status); + if ((int32_t)(peer_seq - fence_seq_entry) >= 0) { + if (peer_status > HTP_STATUS_OK) { + FARF(ERROR, "ggml-hex: allreduce entry peer %u failed with status %u\n", j, peer_status); + htp_fence_write(my_fence, fence_seq_exit, peer_status); + htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry); + return peer_status; + } break; } if (++spins > HTP_FENCE_TIMEOUT) { - FARF(ERROR, "ggml-hex: allreduce entry fence-wait TIMEOUT: rank %u waiting on %u (fence %p seq %u)\n", rank, j, peer_fence, fence_seq_entry); + FARF(ERROR, "ggml-hex: allreduce entry fence-wait TIMEOUT : rank %u waiting on %u fence %p seq 0x%x peer-seq 0x%x\n", + rank, j, peer_fence, fence_seq_entry, peer_seq); + htp_fence_write(my_fence, fence_seq_exit, HTP_STATUS_INTERNAL_ERR); + htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry); return HTP_STATUS_INTERNAL_ERR; } hex_pause(); @@ -301,8 +352,6 @@ int op_allreduce(struct htp_ops_context * octx) { const uint32_t elems_per_thread = (uint32_t) kparams->elems_per_thread; const uint32_t vtcm_size_per_thread = (uint32_t) kparams->vtcm_size_per_thread; - const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD); - struct htp_allreduce_context actx; actx.octx = octx; actx.n_ranks = n_ranks; @@ -339,6 +388,8 @@ int op_allreduce(struct htp_ops_context * octx) { } break; default: + FARF(ERROR, "ggml-hex: allreduce unsupported kernel %d : rank %u\n", kparams->kernel_type, rank); + htp_fence_write(my_fence, fence_seq_exit, HTP_STATUS_NO_SUPPORT); return HTP_STATUS_NO_SUPPORT; } @@ -368,23 +419,31 @@ int op_allreduce(struct htp_ops_context * octx) { // 4. Exit Barrier: Synchronize all ranks after writing htp_trace_event_start(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit); - atomic_store(&my_fence[0], fence_seq_exit); - asm volatile ("syncht" : : : "memory"); - Q6_dccleaninva_A((void *) my_fence); + htp_fence_write(my_fence, fence_seq_exit, octx->status); for (uint32_t j = 0; j < n_ranks; j++) { if (j == rank) continue; const struct htp_tensor * peer_sync = octx->src[n_ranks + j]; - atomic_uint * peer_fence = (atomic_uint *) peer_sync->data; + atomic_uint * peer_fence = (atomic_uint *) (uintptr_t) peer_sync->data; uint64_t spins = 0; while (1) { - Q6_dccleaninva_A((void *) peer_fence); - uint32_t val = atomic_load(&peer_fence[0]); - if (val == fence_seq_exit) { + uint32_t peer_seq; + uint32_t peer_status; + htp_fence_read(peer_fence, &peer_seq, &peer_status); + if ((int32_t)(peer_seq - fence_seq_exit) >= 0) { + if (peer_status > HTP_STATUS_OK) { + FARF(ERROR, "ggml-hex: allreduce exit peer %u failed with status %u\n", j, peer_status); + htp_fence_write(my_fence, fence_seq_exit, peer_status); + htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit); + return peer_status; + } break; } if (++spins > HTP_FENCE_TIMEOUT) { - FARF(ERROR, "ggml-hex: allreduce exit fence-wait TIMEOUT: rank %u waiting on %u (fence %p seq %u)\n", rank, j, peer_fence, fence_seq_exit); + FARF(ERROR, "ggml-hex: allreduce exit fence-wait TIMEOUT : rank %u waiting on %u fence %p seq 0x%x peer-seq 0x%x\n", + rank, j, peer_fence, fence_seq_exit, peer_seq); + htp_fence_write(my_fence, fence_seq_exit, HTP_STATUS_INTERNAL_ERR); + htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit); return HTP_STATUS_INTERNAL_ERR; } hex_pause(); @@ -394,5 +453,5 @@ int op_allreduce(struct htp_ops_context * octx) { htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit); - return HTP_STATUS_OK; + return octx->status; } diff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.h b/ggml/src/ggml-hexagon/htp/allreduce-ops.h index de447d87e..0aed2b8b7 100644 --- a/ggml/src/ggml-hexagon/htp/allreduce-ops.h +++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.h @@ -2,6 +2,8 @@ #define ALLREDUCE_OPS_H #include +#include +#include #define HTP_ALLREDUCE_MAX_RANKS 4 @@ -15,6 +17,15 @@ enum htp_allreduce_kernel_type { HTP_ALLREDUCE_KERNEL_DMA_2D, }; +static inline size_t htp_allreduce_vtcm_buffer_count( + uint32_t n_ranks, + uint32_t n_threads, + bool has_add, + bool is_row_bcast +) { + return (size_t) (n_ranks + 1) * n_threads + (has_add ? (is_row_bcast ? 1 : n_threads) : 0); +} + struct htp_allreduce_kernel_params { int32_t rank; int32_t n_ranks; diff --git a/ggml/src/ggml-hexagon/htp/argsort-ops.c b/ggml/src/ggml-hexagon/htp/argsort-ops.c index 774faef5f..e3c49e763 100644 --- a/ggml/src/ggml-hexagon/htp/argsort-ops.c +++ b/ggml/src/ggml-hexagon/htp/argsort-ops.c @@ -11,9 +11,10 @@ #include "hvx-utils.h" #include "hex-dma.h" +#include "hex-common.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" #ifndef MIN #define MIN(a, b) ((a) < (b) ? (a) : (b)) @@ -22,6 +23,9 @@ struct htp_argsort_context { struct htp_ops_context * octx; uint32_t nrows_per_thread; + uint32_t total_rows; + uint32_t row_start; + uint32_t row_end; uint8_t * vtcm_base; size_t vtcm_per_thread; }; @@ -336,10 +340,9 @@ static void htp_argsort_f32_##ne00##_##order_name(unsigned int n, unsigned int i const struct htp_tensor * src0 = octx->src[0]; \ const struct htp_tensor * dst = octx->dst; \ uint8_t * spad = actx->vtcm_base + actx->vtcm_per_thread * i; \ - uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; \ uint32_t rows_per_thread = actx->nrows_per_thread; \ - uint32_t start_row = rows_per_thread * i; \ - uint32_t end_row = MIN(start_row + rows_per_thread, total_rows); \ + uint32_t start_row = actx->row_start + rows_per_thread * i; \ + uint32_t end_row = MIN(start_row + rows_per_thread, actx->row_end); \ size_t values_size = hex_round_up(ne00 * sizeof(float), 128); \ float * values_buf = (float *) spad; \ int32_t * indices_buf = (int32_t *) (spad + values_size); \ @@ -386,9 +389,6 @@ static void htp_argsort_f32_fallback(unsigned int n, unsigned int i, void * data // Dimensions uint32_t ne00 = src0->ne[0]; - uint32_t ne01 = src0->ne[1]; - uint32_t ne02 = src0->ne[2]; - uint32_t ne03 = src0->ne[3]; uint32_t nb01 = src0->nb[1]; @@ -398,10 +398,9 @@ static void htp_argsort_f32_fallback(unsigned int n, unsigned int i, void * data enum ggml_sort_order order = (enum ggml_sort_order) octx->op_params[0]; // Rows to process - uint32_t total_rows = ne01 * ne02 * ne03; uint32_t rows_per_thread = actx->nrows_per_thread; - uint32_t start_row = rows_per_thread * i; - uint32_t end_row = MIN(start_row + rows_per_thread, total_rows); + uint32_t start_row = actx->row_start + rows_per_thread * i; + uint32_t end_row = MIN(start_row + rows_per_thread, actx->row_end); size_t values_size = hex_round_up(ne00 * sizeof(float), 128); uint32_t num_vec_ind_values = hmx_ceil_div(ne00, VLEN/(sizeof(int32_t))); @@ -451,8 +450,28 @@ int op_argsort(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - const uint32_t total_rows = octx->src[0]->ne[1] * octx->src[0]->ne[2] * octx->src[0]->ne[3]; - const uint32_t n_threads = MIN(total_rows, octx->n_threads); + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + + const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const size_t dst_row_size = dst->ne[0] * sizeof(int32_t); + + uint32_t row_start = 0; + uint32_t row_end = total_rows; + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(int32_t), (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + row_end = range.start + range.count; + } + + const uint32_t nrows = row_end - row_start; + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; // Allocate scratchpad // We need 1 row of float + 1 row of int32 per thread. @@ -478,7 +497,10 @@ int op_argsort(struct htp_ops_context * octx) { struct htp_argsort_context actx; actx.octx = octx; - actx.nrows_per_thread = (total_rows + n_threads - 1) / n_threads; + actx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + actx.total_rows = nrows; + actx.row_start = row_start; + actx.row_end = row_end; actx.vtcm_base = (uint8_t *) octx->ctx->vtcm_base; actx.vtcm_per_thread = spad_per_thread; @@ -508,7 +530,7 @@ int op_argsort(struct htp_ops_context * octx) { } // Run jobs - worker_pool_run_func(octx->ctx->worker_pool, job_func, &actx, n_threads); + work_queue_run(octx->ctx->work_queue, job_func, &actx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.c b/ggml/src/ggml-hexagon/htp/binary-ops.c index db6177963..bfa849e0e 100644 --- a/ggml/src/ggml-hexagon/htp/binary-ops.c +++ b/ggml/src/ggml-hexagon/htp/binary-ops.c @@ -13,9 +13,10 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" #include "htp-tensor.h" #ifndef MIN @@ -36,6 +37,8 @@ struct htp_binary_context { uint32_t block_max; uint32_t nrows_per_thread; + uint32_t total_rows; + uint32_t row_start; size_t src0_row_size_aligned; size_t src1_row_size_aligned; size_t dst_row_size_aligned; @@ -48,27 +51,27 @@ struct htp_binary_context { const struct htp_tensor * src0 = octx->src[0]; \ const struct htp_tensor * src1 = octx->src[1]; \ const struct htp_tensor * dst = octx->dst; \ - \ - const uint32_t ne00 = src0->ne[0]; \ - const uint32_t ne01 = src0->ne[1]; \ - const uint32_t ne02 = src0->ne[2]; \ - const uint32_t ne03 = src0->ne[3]; \ - \ - const uint32_t ne10 = src1->ne[0]; \ - const uint32_t ne11 = src1->ne[1]; \ - const uint32_t ne12 = src1->ne[2]; \ - const uint32_t ne13 = src1->ne[3]; \ - \ - const uint32_t nb01 = src0->nb[1]; \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t nb11 = src1->nb[1]; \ - const uint32_t nb12 = src1->nb[2]; \ - const uint32_t nb13 = src1->nb[3]; \ - \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ + \ + const uint32_t ne00 = src0->ne[0]; \ + const uint32_t ne01 = src0->ne[1]; \ + const uint32_t ne02 = src0->ne[2]; \ + const uint32_t ne03 = src0->ne[3]; \ + \ + const uint32_t ne10 = src1->ne[0]; \ + const uint32_t ne11 = src1->ne[1]; \ + const uint32_t ne12 = src1->ne[2]; \ + const uint32_t ne13 = src1->ne[3]; \ + \ + const uint32_t nb01 = src0->nb[1]; \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t nb11 = src1->nb[1]; \ + const uint32_t nb12 = src1->nb[2]; \ + const uint32_t nb13 = src1->nb[3]; \ + \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ const uint32_t nb3 = dst->nb[3]; static inline uint32_t calc_block_size(struct htp_binary_context * bctx, uint32_t ir, uint32_t end_row, uint32_t ne01, uint32_t ne02) { @@ -93,87 +96,87 @@ static inline uint32_t calc_block_size(struct htp_binary_context * bctx, uint32_ } // Macro for scalar op switch -#define COMPUTE_SCALAR_OP(DST, SRC, VAL, TYPE, N) \ - if(TYPE == HTP_TYPE_F32) { \ - switch (octx->op) { \ - case HTP_OP_ADD: hvx_add_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \ - case HTP_OP_SUB: hvx_sub_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \ - case HTP_OP_MUL: hvx_mul_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \ +#define COMPUTE_SCALAR_OP(DST, SRC, VAL, TYPE, N) \ + if(TYPE == HTP_TYPE_F32) { \ + switch (octx->op) { \ + case HTP_OP_ADD: hvx_add_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \ + case HTP_OP_SUB: hvx_sub_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \ + case HTP_OP_MUL: hvx_mul_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \ case HTP_OP_DIV: hvx_mul_scalar_f32_aa(DST, SRC, 1.0f / (*(float *)VAL), N); break; \ - default: break; \ - } \ - } \ - else { \ - switch (octx->op) { \ - case HTP_OP_ADD: hvx_add_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ - case HTP_OP_SUB: hvx_sub_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ - case HTP_OP_MUL: hvx_mul_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ - case HTP_OP_DIV: hvx_div_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ - default: break; \ - } \ + default: break; \ + } \ + } \ + else { \ + switch (octx->op) { \ + case HTP_OP_ADD: hvx_add_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ + case HTP_OP_SUB: hvx_sub_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ + case HTP_OP_MUL: hvx_mul_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ + case HTP_OP_DIV: hvx_div_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ + default: break; \ + } \ } // Macro for vector op switch (All Aligned) -#define COMPUTE_VECTOR_OP_AAA(DST, SRC0, SRC1, TYPE, N) \ - if(TYPE == HTP_TYPE_F32) { \ - switch (octx->op) { \ +#define COMPUTE_VECTOR_OP_AAA(DST, SRC0, SRC1, TYPE, N) \ + if(TYPE == HTP_TYPE_F32) { \ + switch (octx->op) { \ case HTP_OP_ADD: hvx_add_f32_aaa(DST, SRC0, SRC1, N); break; \ case HTP_OP_SUB: hvx_sub_f32_aaa(DST, SRC0, SRC1, N); break; \ case HTP_OP_MUL: hvx_mul_f32_aaa(DST, SRC0, SRC1, N); break; \ case HTP_OP_DIV: hvx_div_f32_aaa(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ - } \ - else { \ - switch (octx->op) { \ + default: break; \ + } \ + } \ + else { \ + switch (octx->op) { \ case HTP_OP_ADD: hvx_add_f16_aaa(DST, SRC0, SRC1, N); break; \ case HTP_OP_SUB: hvx_sub_f16_aaa(DST, SRC0, SRC1, N); break; \ case HTP_OP_MUL: hvx_mul_f16_aaa(DST, SRC0, SRC1, N); break; \ case HTP_OP_DIV: hvx_div_f16_aaa(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ + default: break; \ + } \ } // Macro for vector op switch (Dst Aligned, Src0 Aligned, Src1 Unaligned) -#define COMPUTE_VECTOR_OP_AAU(DST, SRC0, SRC1, TYPE, N) \ - if(TYPE == HTP_TYPE_F32) { \ - switch (octx->op) { \ +#define COMPUTE_VECTOR_OP_AAU(DST, SRC0, SRC1, TYPE, N) \ + if(TYPE == HTP_TYPE_F32) { \ + switch (octx->op) { \ case HTP_OP_ADD: hvx_add_f32_aau(DST, SRC0, SRC1, N); break; \ case HTP_OP_SUB: hvx_sub_f32_aau(DST, SRC0, SRC1, N); break; \ case HTP_OP_MUL: hvx_mul_f32_aau(DST, SRC0, SRC1, N); break; \ case HTP_OP_DIV: hvx_div_f32_aau(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ - } \ - else { \ - switch (octx->op) { \ + default: break; \ + } \ + } \ + else { \ + switch (octx->op) { \ case HTP_OP_ADD: hvx_add_f16_aau(DST, SRC0, SRC1, N); break; \ case HTP_OP_SUB: hvx_sub_f16_aau(DST, SRC0, SRC1, N); break; \ case HTP_OP_MUL: hvx_mul_f16_aau(DST, SRC0, SRC1, N); break; \ case HTP_OP_DIV: hvx_div_f16_aau(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ + default: break; \ + } \ } // Macro for vector op switch (All Unaligned - generic loop used in element repeat) -#define COMPUTE_VECTOR_OP_UUU(DST, SRC0, SRC1, TYPE, N) \ - if(TYPE == HTP_TYPE_F32) { \ - switch (octx->op) { \ +#define COMPUTE_VECTOR_OP_UUU(DST, SRC0, SRC1, TYPE, N) \ + if(TYPE == HTP_TYPE_F32) { \ + switch (octx->op) { \ case HTP_OP_ADD: hvx_add_f32_uuu(DST, SRC0, SRC1, N); break; \ case HTP_OP_SUB: hvx_sub_f32_uuu(DST, SRC0, SRC1, N); break; \ case HTP_OP_MUL: hvx_mul_f32_uuu(DST, SRC0, SRC1, N); break; \ case HTP_OP_DIV: hvx_div_f32_uuu(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ - } \ - else { \ - switch (octx->op) { \ + default: break; \ + } \ + } \ + else { \ + switch (octx->op) { \ case HTP_OP_ADD: hvx_add_f16_uuu(DST, SRC0, SRC1, N); break; \ case HTP_OP_SUB: hvx_sub_f16_uuu(DST, SRC0, SRC1, N); break; \ case HTP_OP_MUL: hvx_mul_f16_uuu(DST, SRC0, SRC1, N); break; \ case HTP_OP_DIV: hvx_div_f16_uuu(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ + default: break; \ + } \ } // 1. Scalar src1 (ne10 == 1) @@ -184,9 +187,8 @@ static void binary_job_scalar(unsigned int nth, unsigned int ith, void * data) { const uint32_t src0_type = octx->src[0]->type; const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16); - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; FARF(HIGH, "binary-scalar: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); @@ -222,6 +224,8 @@ static void binary_job_scalar(unsigned int nth, unsigned int ith, void * data) { } // Main loop + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); @@ -242,12 +246,14 @@ static void binary_job_scalar(unsigned int nth, unsigned int ith, void * data) { uint8_t * src1_ptr = (uint8_t *)src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11; uint32_t s1_stride = (ne11 == 1) ? 0 : nb11; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); for (uint32_t r = 0; r < current_block_size; r++) { uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned; uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned; COMPUTE_SCALAR_OP(r_dst, r_src0, src1_ptr, src0_type, ne00); src1_ptr += s1_stride; } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); @@ -266,6 +272,7 @@ static void binary_job_scalar(unsigned int nth, unsigned int ith, void * data) { } ir += current_block_size; } + dma_queue_flush(q); } @@ -277,9 +284,8 @@ static void binary_job_vector_same_shape(unsigned int nth, unsigned int ith, voi const uint32_t src0_type = octx->src[0]->type; const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16); - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; FARF(HIGH, "binary-same-shape: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); @@ -323,18 +329,22 @@ static void binary_job_vector_same_shape(unsigned int nth, unsigned int ith, voi spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst; uint8_t * s1_spad = (uint8_t *) dma_queue_pop(q).dst; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); for (uint32_t r = 0; r < current_block_size; r++) { uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned; uint8_t * r_src1 = s1_spad + r * bctx->src1_row_size_aligned; uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned; COMPUTE_VECTOR_OP_AAA(r_dst, r_src0, r_src1, src0_type, ne00); } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); uint32_t i03, i02, i01, rem; i03 = fastdiv(ir, &bctx->src0_dim12_div); @@ -366,6 +376,7 @@ static void binary_job_vector_same_shape(unsigned int nth, unsigned int ith, voi } ir += current_block_size; } + dma_queue_flush(q); } @@ -377,9 +388,8 @@ static void binary_job_vector_row_broadcast(unsigned int nth, unsigned int ith, const uint32_t src0_type = octx->src[0]->type; const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16); - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; FARF(HIGH, "binary-row-bcast: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); @@ -416,17 +426,21 @@ static void binary_job_vector_row_broadcast(unsigned int nth, unsigned int ith, spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); for (uint32_t r = 0; r < current_block_size; r++) { uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned; uint8_t * r_src1 = (uint8_t *)s1_ptr; // Constant uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned; COMPUTE_VECTOR_OP_AAA(r_dst, r_src0, r_src1, src0_type, ne00); } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); uint32_t i03 = fastdiv(ir, &bctx->src0_dim12_div); uint32_t rem = ir - i03 * (ne02 * ne01); @@ -447,6 +461,7 @@ static void binary_job_vector_row_broadcast(unsigned int nth, unsigned int ith, } ir += current_block_size; } + dma_queue_flush(q); } @@ -458,9 +473,8 @@ static void binary_job_vector_complex(unsigned int nth, unsigned int ith, void * const uint32_t src0_type = octx->src[0]->type; const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16); - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; FARF(HIGH, "binary-complex: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); @@ -493,6 +507,8 @@ static void binary_job_vector_complex(unsigned int nth, unsigned int ith, void * spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; @@ -503,6 +519,7 @@ static void binary_job_vector_complex(unsigned int nth, unsigned int ith, void * uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); for (uint32_t r = 0; r < current_block_size; r++) { uint32_t r_i01 = i01 + r; uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div); @@ -516,6 +533,7 @@ static void binary_job_vector_complex(unsigned int nth, unsigned int ith, void * // Read src1 from DDR (unaligned) COMPUTE_VECTOR_OP_AAU(r_dst, r_src0, r_src1, src0_type, ne00); } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); @@ -532,6 +550,7 @@ static void binary_job_vector_complex(unsigned int nth, unsigned int ith, void * } ir += current_block_size; } + dma_queue_flush(q); } @@ -544,9 +563,8 @@ static void binary_job_element_repeat(unsigned int nth, unsigned int ith, void * const uint32_t src0_type = octx->src[0]->type; const uint32_t elem_size_bytes = (src0_type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16); const uint32_t row_size_bytes = ne00 * elem_size_bytes;; - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); @@ -579,6 +597,8 @@ static void binary_job_element_repeat(unsigned int nth, unsigned int ith, void * spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; @@ -589,6 +609,7 @@ static void binary_job_element_repeat(unsigned int nth, unsigned int ith, void * uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); for (uint32_t r = 0; r < current_block_size; r++) { uint32_t r_i01 = i01 + r; uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div); @@ -606,6 +627,7 @@ static void binary_job_element_repeat(unsigned int nth, unsigned int ith, void * COMPUTE_VECTOR_OP_UUU(r_dst + c * elem_size_bytes, r_src0 + c * elem_size_bytes, r_src1_row, src0_type, len); } } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); @@ -622,6 +644,7 @@ static void binary_job_element_repeat(unsigned int nth, unsigned int ith, void * } ir += current_block_size; } + dma_queue_flush(q); } @@ -650,9 +673,8 @@ static void binary_job_add_id(unsigned int nth, unsigned int ith, void * data) { const uint32_t nb2 = dst->nb[2]; const uint32_t nb3 = dst->nb[3]; - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); @@ -683,6 +705,8 @@ static void binary_job_add_id(unsigned int nth, unsigned int ith, void * data) { spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; @@ -693,6 +717,7 @@ static void binary_job_add_id(unsigned int nth, unsigned int ith, void * data) { uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); for (uint32_t r = 0; r < current_block_size; r++) { uint32_t r_i01 = i01 + r; // linear within block since we split at ne01 @@ -704,6 +729,7 @@ static void binary_job_add_id(unsigned int nth, unsigned int ith, void * data) { hvx_add_f32_aau(r_dst, r_src0, r_src1, ne00); } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, ne00 * sizeof(float), current_block_size); @@ -720,6 +746,7 @@ static void binary_job_add_id(unsigned int nth, unsigned int ith, void * data) { } ir += current_block_size; } + dma_queue_flush(q); } @@ -729,15 +756,31 @@ static int execute_op_binary(struct htp_ops_context * octx) { const struct htp_tensor * dst = octx->dst; const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); - // Use packed row sizes for VTCM allocation + // Use packed row sizes for VTCM allocation and alignment const uint32_t src0_type = octx->src[0]->type; const size_t elem_size = (src0_type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16); const size_t src0_row_size = src0->ne[0] * elem_size; const size_t src1_row_size = src1->ne[0] * elem_size; const size_t dst_row_size = dst->ne[0] * elem_size; + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) elem_size, (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN); size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN); size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN); @@ -815,7 +858,9 @@ static int execute_op_binary(struct htp_ops_context * octx) { struct htp_binary_context bctx; bctx.octx = octx; - bctx.nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; + bctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + bctx.total_rows = nrows; + bctx.row_start = row_start; bctx.block_max = rows_per_buffer; bctx.src0_row_size_aligned = src0_row_size_aligned; bctx.src1_row_size_aligned = src1_row_size_aligned; @@ -850,7 +895,7 @@ static int execute_op_binary(struct htp_ops_context * octx) { dma_queue_pop(q); } - worker_pool_run_func(octx->ctx->worker_pool, worker_func, &bctx, n_threads); + work_queue_run(octx->ctx->work_queue, worker_func, &bctx, n_threads); return HTP_STATUS_OK; } @@ -870,4 +915,3 @@ int op_binary(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.c b/ggml/src/ggml-hexagon/htp/concat-ops.c index 51d39e8d9..966e867b3 100644 --- a/ggml/src/ggml-hexagon/htp/concat-ops.c +++ b/ggml/src/ggml-hexagon/htp/concat-ops.c @@ -1,5 +1,8 @@ +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" #include "hexagon_types.h" #include "hexagon_protos.h" #include "hvx_hexagon_protos.h" @@ -13,6 +16,10 @@ struct htp_concat_context { struct htp_ops_context * octx; uint32_t dim; uint32_t nrows_per_thread; + uint32_t row_start; + uint32_t nrows; + uint32_t elem_start; + uint32_t nelems; struct fastdiv_values div_ne0; struct fastdiv_values div_ne1; struct fastdiv_values div_ne2; @@ -28,10 +35,10 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * const uint32_t src0_ne0 = src0->ne[0]; const uint32_t src1_ne0 = src1->ne[0]; - const uint32_t ne1 = dst->ne[1]; - const uint32_t start_i = ith * cctx->nrows_per_thread; - const uint32_t end_i = (start_i + cctx->nrows_per_thread < ne1) ? (start_i + cctx->nrows_per_thread) : ne1; + const uint32_t row_end = cctx->row_start + cctx->nrows; + const uint32_t start_i = cctx->row_start + ith * cctx->nrows_per_thread; + const uint32_t end_i = (start_i + cctx->nrows_per_thread < row_end) ? (start_i + cctx->nrows_per_thread) : row_end; if (start_i >= end_i) return; dma_queue * q = octx->ctx->dma[ith]; @@ -51,6 +58,8 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * const uint32_t spad0_row_bytes = hex_round_up((src0_ne0 + src1_ne0_padded) * sizeof(float), VLEN); uint32_t mu = src1_ne0_padded * spad1_stride; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t i = start_i; i < end_i; i += block_i) { uint32_t current_block_i = (end_i - i < block_i) ? (end_i - i) : block_i; @@ -66,6 +75,7 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * HVX_Vector * vtcm_tmp = (HVX_Vector *)(spad1_base + src1_ne0_padded * spad1_stride); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); for (uint32_t j = 0; j < src1_ne0_padded; j += 32) { #pragma unroll(4) for (uint32_t ii = 0; ii < current_block_i; ii++) { @@ -75,6 +85,7 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * hvx_vmemu(dst_ptr) = vtcm_tmp[ii]; } } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); dma_queue_pop(q); // src0 @@ -95,10 +106,10 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * const uint32_t src0_ne0 = src0->ne[0]; const uint32_t src1_ne0 = src1->ne[0]; - const uint32_t ne1 = dst->ne[1]; - const uint32_t start_i = ith * cctx->nrows_per_thread; - const uint32_t end_i = (start_i + cctx->nrows_per_thread < ne1) ? (start_i + cctx->nrows_per_thread) : ne1; + const uint32_t row_end = cctx->row_start + cctx->nrows; + const uint32_t start_i = cctx->row_start + ith * cctx->nrows_per_thread; + const uint32_t end_i = (start_i + cctx->nrows_per_thread < row_end) ? (start_i + cctx->nrows_per_thread) : row_end; if (start_i >= end_i) return; dma_queue * q = octx->ctx->dma[ith]; @@ -118,6 +129,8 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * const uint32_t spad0_row_bytes = hex_round_up((src0_ne0 + src1_ne0_padded) * sizeof(__fp16), VLEN); uint32_t mu = src1_ne0_padded * spad1_stride; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t i = start_i; i < end_i; i += block_i) { uint32_t current_block_i = (end_i - i < block_i) ? (end_i - i) : block_i; @@ -133,6 +146,7 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * HVX_Vector * vtcm_tmp = (HVX_Vector *)(spad1_base + src1_ne0_padded * spad1_stride); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); for (uint32_t j = 0; j < src1_ne0_padded; j += 64) { #pragma unroll(4) for (uint32_t ii = 0; ii < current_block_i; ii++) { @@ -142,6 +156,7 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * hvx_vmemu(dst_ptr) = vtcm_tmp[ii]; } } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); dma_queue_pop(q); // src0 @@ -164,11 +179,14 @@ static void concat_generic(unsigned int nth, unsigned int ith, void * data) { const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2; const uint32_t ne[4] = {dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]}; - const uint32_t total_elements = ne[0] * ne[1] * ne[2] * ne[3]; - const uint32_t chunk_size = (total_elements + nth - 1) / nth; - const uint32_t start_idx = MIN(ith * chunk_size, total_elements); - const uint32_t end_idx = MIN(start_idx + chunk_size, total_elements); + // Per-device element range aligned to prevent false sharing + const uint32_t elem_start = cctx->elem_start; + const uint32_t nelems = cctx->nelems; + const uint32_t chunk_size = (nelems + nth - 1) / nth; + + const uint32_t start_idx = MIN(elem_start + ith * chunk_size, elem_start + nelems); + const uint32_t end_idx = MIN(start_idx + chunk_size, elem_start + nelems); // Naive scalar element-wise copy for (uint32_t idx = start_idx; idx < end_idx; idx++) { @@ -236,13 +254,28 @@ int op_concat(struct htp_ops_context * octx) { void (*worker_func)(unsigned int, unsigned int, void *) = concat_generic; if (dim == 0 && is_2d && is_src1_transposed && !is_src0_transposed) { - n_threads = MIN(dst->ne[1], n_threads); - if (n_threads < 1) { - n_threads = 1; + const uint32_t total_rows = dst->ne[1]; + const size_t dst_data_row_size = dst->ne[0] * type_size; + uint32_t row_start = 0; + uint32_t nrows = total_rows; + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, type_size, (uint32_t) dst_data_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + cctx.row_start = row_start; + cctx.nrows = nrows; + uint32_t block_i = (type_size == 4) ? 32 : 64; - cctx.nrows_per_thread = hmx_ceil_div(dst->ne[1], n_threads); + cctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); // Allocate VTCM uint32_t spad1_stride = block_i * type_size; @@ -270,8 +303,26 @@ int op_concat(struct htp_ops_context * octx) { } else { worker_func = concat_2d_f16_transposed; } + } else { + const uint32_t total_elements = dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; + uint32_t elem_start = 0; + uint32_t nelems = total_elements; + if (octx->ctx->mdev.count > 1) { + const uint32_t elems_per_chunk = HEX_L2_LINE_SIZE / type_size; + const bool can_split = htp_tensor_mdev_data_aligned(dst) && htp_tensor_is_contiguous(dst, type_size) && !htp_tensor_is_permuted(dst); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_elements, can_split ? elems_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + elem_start = range.start; + nelems = range.count; + } + + if (nelems == 0) { + return HTP_STATUS_OK; + } + + cctx.elem_start = elem_start; + cctx.nelems = nelems; } - worker_pool_run_func(octx->ctx->worker_pool, worker_func, &cctx, n_threads); + work_queue_run(octx->ctx->work_queue, worker_func, &cctx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.c index b151b757f..7f01a8c1e 100644 --- a/ggml/src/ggml-hexagon/htp/cpy-ops.c +++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c @@ -16,6 +16,7 @@ #include "htp-ops.h" #include "hvx-utils.h" #include "htp-tensor.h" +#include "htp-fence.h" struct htp_copy_context { struct htp_ops_context * octx; @@ -29,7 +30,23 @@ struct htp_copy_context { uint32_t src0_blocks_per_row; uint32_t dst_blocks_per_row; + uint32_t elem_start; + uint32_t nelem; + uint32_t elem_per_thread; + uint32_t src0_nrows_per_thread; + uint32_t row_start; + uint32_t nrows; + + struct fastdiv_values div_ne01; + struct fastdiv_values div_ne02_ne01; + + struct fastdiv_values div_ne0; + struct fastdiv_values div_ne1_ne0; + struct fastdiv_values div_ne2_ne1_ne0; + struct fastdiv_values div_ne00; + struct fastdiv_values div_ne01_ne00; + struct fastdiv_values div_ne02_ne01_ne00; }; #define cpy_preamble \ @@ -54,131 +71,113 @@ struct htp_copy_context { const uint32_t nb0 = dst->nb[0]; \ const uint32_t nb1 = dst->nb[1]; \ const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; \ - \ - const uint32_t nr = ne01; + const uint32_t nb3 = dst->nb[3]; -#define DEFINE_CPY_SAMESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \ -static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \ - struct htp_copy_context * ct = (struct htp_copy_context *) data; \ - struct htp_ops_context * octx = ct->octx; \ - cpy_preamble; \ - const uint32_t dr = ct->src0_nrows_per_thread; \ - const uint32_t ir0 = dr * ith; \ - const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr; \ - if (ir0 >= nr) return; \ - for (uint32_t i03 = 0; i03 < ne03; i03++) { \ - for (uint32_t i02 = 0; i02 < ne02; i02++) { \ - _Pragma("unroll(4)") \ - for (uint32_t i01 = ir0; i01 < ir1; i01++) { \ - uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; \ - uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; \ - hex_l2fetch(src0_ptr, ne00 * ELEM_SIZE, nb01, 2); \ - hvx_copy_uu(dst_ptr, src0_ptr, ne00, ELEM_SIZE); \ - } \ - } \ - } \ +#define DEFINE_CPY_SAMESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \ +static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_copy_context * ct = (struct htp_copy_context *) data; \ + struct htp_ops_context * octx = ct->octx; \ + cpy_preamble; \ + const uint32_t dr = ct->src0_nrows_per_thread; \ + const uint32_t ir0 = ct->row_start + dr * ith; \ + const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); \ + if (ir0 >= ir1) return; \ + const bool contiguous = (nb01 == ne00 * ELEM_SIZE) && (nb1 == nb01) && \ + (nb02 == ne01 * nb01) && (nb2 == nb02) && \ + (nb03 == ne02 * nb02) && (nb3 == nb03); \ + const uint32_t ne02_ne01 = ne02 * ne01; \ + uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01); \ + uint32_t rem = ir0 - i03 * ne02_ne01; \ + uint32_t i02 = fastdiv(rem, &ct->div_ne01); \ + uint32_t i01 = rem - i02 * ne01; \ + uint8_t * dst_ptr = (uint8_t *) dst->data + i01*nb1 + i02*nb2 + i03*nb3; \ + uint8_t * src0_ptr = (uint8_t *) src0->data + i01*nb01 + i02*nb02 + i03*nb03; \ + if (contiguous) { \ + hvx_copy_uu(dst_ptr, src0_ptr, (ir1 - ir0) * ne00, ELEM_SIZE); \ + return; \ + } \ + for (uint32_t r = ir0; r < ir1; r++) { \ + hex_l2fetch(src0_ptr, ne00 * ELEM_SIZE, nb01, 2); \ + hvx_copy_uu(dst_ptr, src0_ptr, ne00, ELEM_SIZE); \ + dst_ptr += nb1; \ + src0_ptr += nb01; \ + if (++i01 == ne01) { \ + i01 = 0; \ + if (++i02 == ne02) { \ + i02 = 0; \ + i03++; \ + } \ + dst_ptr = (uint8_t *) dst->data + i02*nb2 + i03*nb3; \ + src0_ptr = (uint8_t *) src0->data + i02*nb02 + i03*nb03; \ + } \ + } \ } DEFINE_CPY_SAMESHAPE(f32, float, 4) DEFINE_CPY_SAMESHAPE(f16, __fp16, 2) -#define DEFINE_CPY_RESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \ -static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void * data) { \ - struct htp_copy_context * ct = (struct htp_copy_context *) data; \ - struct htp_ops_context * octx = ct->octx; \ - cpy_preamble; \ - const uint32_t dr = ct->src0_nrows_per_thread; \ - const uint32_t ir0 = dr * ith; \ - const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr; \ - if (ir0 >= nr) return; \ - const bool src0_contig = (nb00 == ELEM_SIZE) && \ - (nb01 == ne00 * nb00) && \ - (nb02 == ne01 * nb01) && \ - (nb03 == ne02 * nb02); \ - const bool dst_contig = (nb0 == ELEM_SIZE) && \ - (nb1 == ne0 * nb0) && \ - (nb2 == ne1 * nb1) && \ - (nb3 == ne2 * nb2); \ - if (src0_contig && dst_contig) { \ - for (int64_t i03 = 0; i03 < ne03; i03++) { \ - for (int64_t i02 = 0; i02 < ne02; i02++) { \ - uint8_t * src_ptr = (uint8_t *) src0->data + i03*nb03 + i02*nb02 + ir0*nb01; \ - uint32_t flat = ((i03*ne02 + i02)*ne01 + ir0) * ne00; \ - uint8_t * dst_ptr = (uint8_t *) dst->data + flat * ELEM_SIZE; \ - hvx_copy_uu(dst_ptr, src_ptr, (ir1 - ir0) * ne00, ELEM_SIZE); \ - } \ - } \ - return; \ - } \ - const bool reshape_flat_fast = (ne03 == 1 && ne2 == 1 && ne3 == 1) && \ - (ne0 == ne00 * ne01) && (ne1 == ne02) && \ - (nb00 == ELEM_SIZE) && (nb0 == ELEM_SIZE); \ - if (reshape_flat_fast) { \ - for (uint32_t i02 = 0; i02 < ne02; i02++) { \ - for (uint32_t i01 = ir0; i01 < ir1; i01++) { \ - uint8_t * src0_ptr = (uint8_t *) src0->data + i01 * nb01 + i02 * nb02; \ - uint8_t * dst_ptr = (uint8_t *) dst->data + i01 * ne00 * ELEM_SIZE + i02 * nb1; \ - hvx_copy_uu(dst_ptr, src0_ptr, ne00, ELEM_SIZE); \ - } \ - } \ - return; \ - } \ - int64_t k10 = 0; \ - int64_t i11 = 0; \ - int64_t i12 = 0; \ - int64_t i13 = 0; \ - const int64_t nk00 = ct->src0_blocks_per_row; \ - const int64_t nk0 = ct->dst_blocks_per_row; \ - for (int64_t i03 = 0; i03 < ne03; i03++) { \ - for (int64_t i02 = 0; i02 < ne02; i02++) { \ - k10 += nk00 * ir0; \ - while (k10 >= nk0) { \ - k10 -= nk0; \ - if (++i11 == ne1) { \ - i11 = 0; \ - if (++i12 == ne2) { \ - i12 = 0; \ - if (++i13 == ne3) { \ - i13 = 0; \ - } \ - } \ - } \ - } \ - for (int64_t i01 = ir0; i01 < ir1; i01++) { \ - for (int64_t k00 = 0; k00 < nk00; k00++) { \ - const char * src0_ptr = ((char *) src0->data + k00*nb00 + i01*nb01 + i02*nb02 + i03*nb03); \ - char * dst_ptr = ((char *) dst->data + k10*nb0 + i11*nb1 + i12*nb2 + i13*nb3); \ - memcpy(dst_ptr, src0_ptr, ELEM_SIZE); \ - if (++k10 == nk0) { \ - k10 = 0; \ - if (++i11 == ne1) { \ - i11 = 0; \ - if (++i12 == ne2) { \ - i12 = 0; \ - if (++i13 == ne3) { \ - i13 = 0; \ - } \ - } \ - } \ - } \ - } \ - } \ - k10 += nk00 * (ne01 - ir1); \ - while (k10 >= nk0) { \ - k10 -= nk0; \ - if (++i11 == ne1) { \ - i11 = 0; \ - if (++i12 == ne2) { \ - i12 = 0; \ - if (++i13 == ne3) { \ - i13 = 0; \ - } \ - } \ - } \ - } \ - } \ - } \ +#define DEFINE_CPY_RESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \ +static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_copy_context * ct = (struct htp_copy_context *) data; \ + struct htp_ops_context * octx = ct->octx; \ + cpy_preamble; \ + const uint32_t th_nelem = ct->elem_per_thread; \ + const uint32_t th_start = ct->elem_start + ith * th_nelem; \ + const uint32_t th_end = MIN(th_start + th_nelem, ct->elem_start + ct->nelem); \ + if (th_start >= th_end) return; \ + \ + const uint32_t ne01_ne00 = ne01 * ne00; \ + const uint32_t ne02_ne01_ne00 = ne02 * ne01_ne00; \ + const uint32_t ne1_ne0 = ne1 * ne0; \ + const uint32_t ne2_ne1_ne0 = ne2 * ne1_ne0; \ + \ + uint32_t e = th_start; \ + uint32_t i13 = fastdiv(e, &ct->div_ne2_ne1_ne0); \ + uint32_t rem = e - i13 * ne2_ne1_ne0; \ + uint32_t i12 = fastdiv(rem, &ct->div_ne1_ne0); \ + uint32_t rem2 = rem - i12 * ne1_ne0; \ + uint32_t i11 = fastdiv(rem2, &ct->div_ne0); \ + uint32_t i10 = rem2 - i11 * ne0; \ + \ + uint32_t i03 = fastdiv(e, &ct->div_ne02_ne01_ne00); \ + uint32_t rem_s = e - i03 * ne02_ne01_ne00; \ + uint32_t i02 = fastdiv(rem_s, &ct->div_ne01_ne00); \ + uint32_t rem2_s = rem_s - i02 * ne01_ne00; \ + uint32_t i01 = fastdiv(rem2_s, &ct->div_ne00); \ + uint32_t i00 = rem2_s - i01 * ne00; \ + \ + char * dst_ptr = (char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3; \ + const char * src0_ptr = (const char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03; \ + \ + for (; e < th_end; e++) { \ + *((ELEM_TYPE *) dst_ptr) = *((const ELEM_TYPE *) src0_ptr); \ + \ + dst_ptr += nb0; \ + if (++i10 == ne0) { \ + i10 = 0; \ + if (++i11 == ne1) { \ + i11 = 0; \ + if (++i12 == ne2) { \ + i12 = 0; \ + i13++; \ + } \ + } \ + dst_ptr = (char *) dst->data + i11*nb1 + i12*nb2 + i13*nb3; \ + } \ + \ + src0_ptr += nb00; \ + if (++i00 == ne00) { \ + i00 = 0; \ + if (++i01 == ne01) { \ + i01 = 0; \ + if (++i02 == ne02) { \ + i02 = 0; \ + i03++; \ + } \ + } \ + src0_ptr = (const char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03; \ + } \ + } \ } DEFINE_CPY_RESHAPE(f32, float, 4) @@ -189,22 +188,33 @@ static void cpy_thread_f16_f32_sameshape(unsigned int nth, unsigned int ith, voi struct htp_ops_context * octx = ct->octx; cpy_preamble; - // parallelize by src0 rows const uint32_t dr = ct->src0_nrows_per_thread; - const uint32_t ir0 = dr * ith; - const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr; - if (ir0 >= nr) return; + const uint32_t ir0 = ct->row_start + dr * ith; + const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); + if (ir0 >= ir1) return; - // copy by rows - for (uint32_t i03 = 0; i03 < ne03; i03++) { - for (uint32_t i02 = 0; i02 < ne02; i02++) { - #pragma unroll(2) - for (uint32_t i01 = ir0; i01 < ir1; i01++) { - uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; - uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; - hex_l2fetch(src0_ptr, ne00 * sizeof(float), nb01, 2); - hvx_copy_f16_f32_uu(dst_ptr, src0_ptr, ne00); + const uint32_t ne02_ne01 = ne02 * ne01; + uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01); + uint32_t rem = ir0 - i03 * ne02_ne01; + uint32_t i02 = fastdiv(rem, &ct->div_ne01); + uint32_t i01 = rem - i02 * ne01; + + uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; + uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; + + for (uint32_t r = ir0; r < ir1; r++) { + hex_l2fetch(src0_ptr, ne00 * sizeof(float), nb01, 2); + hvx_copy_f16_f32_uu(dst_ptr, src0_ptr, ne00); + dst_ptr += nb1; + src0_ptr += nb01; + if (++i01 == ne01) { + i01 = 0; + if (++i02 == ne02) { + i02 = 0; + i03++; } + dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3; + src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03; } } } @@ -214,22 +224,33 @@ static void cpy_thread_f32_f16_sameshape(unsigned int nth, unsigned int ith, voi struct htp_ops_context * octx = ct->octx; cpy_preamble; - // parallelize by src0 rows const uint32_t dr = ct->src0_nrows_per_thread; - const uint32_t ir0 = dr * ith; - const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr; - if (ir0 >= nr) return; + const uint32_t ir0 = ct->row_start + dr * ith; + const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); + if (ir0 >= ir1) return; - // copy by rows - for (uint32_t i03 = 0; i03 < ne03; i03++) { - for (uint32_t i02 = 0; i02 < ne02; i02++) { - #pragma unroll(2) - for (uint32_t i01 = ir0; i01 < ir1; i01++) { - uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; - uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; - hex_l2fetch(src0_ptr, ne00 * sizeof(__fp16), nb01, 2); - hvx_copy_f32_f16_uu(dst_ptr, src0_ptr, ne00); + const uint32_t ne02_ne01 = ne02 * ne01; + uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01); + uint32_t rem = ir0 - i03 * ne02_ne01; + uint32_t i02 = fastdiv(rem, &ct->div_ne01); + uint32_t i01 = rem - i02 * ne01; + + uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; + uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; + + for (uint32_t r = ir0; r < ir1; r++) { + hex_l2fetch(src0_ptr, ne00 * sizeof(__fp16), nb01, 2); + hvx_copy_f32_f16_uu(dst_ptr, src0_ptr, ne00); + dst_ptr += nb1; + src0_ptr += nb01; + if (++i01 == ne01) { + i01 = 0; + if (++i02 == ne02) { + i02 = 0; + i03++; } + dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3; + src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03; } } } @@ -250,15 +271,19 @@ static inline void cpy_dma_sametype_sameshape( dma_queue * q = octx->ctx->dma[0]; if (contiguous_outer) { - dma_queue_push(q, dma_make_ptr((void *) dst->data, (const void *) src0->data), nb1, nb01, ne00 * elem_size, ne01 * ne02 * ne03); - dma_queue_pop(q); + if (!dma_queue_push(q, dma_make_ptr((void *) dst->data, (const void *) src0->data), nb1, nb01, ne00 * elem_size, ne01 * ne02 * ne03)) { + dma_queue_flush(q); + dma_queue_push(q, dma_make_ptr((void *) dst->data, (const void *) src0->data), nb1, nb01, ne00 * elem_size, ne01 * ne02 * ne03); + } + dma_queue_flush(q); return; } for (uint32_t i03 = 0; i03 < ne03; i03++) { for (uint32_t i02 = 0; i02 < ne02; i02++) { - uint8_t* dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3; - uint8_t* src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03; + uint8_t * dst_ptr = (uint8_t *) dst->data + i02 * nb2 + i03 * nb3; + uint8_t * src0_ptr = (uint8_t *) src0->data + i02 * nb02 + i03 * nb03; + if (!dma_queue_push(q, dma_make_ptr(dst_ptr, src0_ptr), nb1, nb01, ne00 * elem_size, ne01)) { dma_queue_flush(q); dma_queue_push(q, dma_make_ptr(dst_ptr, src0_ptr), nb1, nb01, ne00 * elem_size, ne01); @@ -269,10 +294,9 @@ static inline void cpy_dma_sametype_sameshape( dma_queue_flush(q); } -int op_cpy(struct htp_ops_context * octx) { +static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) { cpy_preamble; - - const uint32_t n_threads = MIN(nr, octx->n_threads); + *use_dma = false; struct htp_copy_context ct; ct.octx = octx; @@ -296,59 +320,117 @@ int op_cpy(struct htp_ops_context * octx) { } const bool sametype = (src0->type == dst->type); - const bool transposed = (nb00 > nb01) || (nb0 > nb1); + const bool transposed = (nb00 > nb01) || (nb0 > nb1) || + (nb00 != ct.src0_type_size) || (nb0 != ct.dst_type_size) || + (nb01 < ne00 * ct.src0_type_size) || (nb1 < ne0 * ct.dst_type_size); const bool sameshape = !transposed && (ne00 == ne0 && ne01 == ne1 && ne02 == ne2 && ne03 == ne3); - ct.src0_nrows_per_thread = (nr + n_threads - 1) / n_threads; + const uint32_t n_threads = octx->n_threads; - worker_callback_t copy_fun = NULL; - bool use_dma = false; + const bool dst_is_contiguous = htp_tensor_is_contiguous(dst, ct.dst_type_size); - if (sametype && sameshape) { - use_dma = true; - } else if (sameshape) { - /**/ if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32) - copy_fun = cpy_thread_f16_f32_sameshape; - else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_F16) - copy_fun = cpy_thread_f32_f16_sameshape; - else - return HTP_STATUS_NO_SUPPORT; - } else if (sametype) { - if (src0->type == HTP_TYPE_F32) { - copy_fun = cpy_thread_f32_reshape; - } else { - copy_fun = cpy_thread_f16_reshape; + if (sameshape) { + const uint32_t total_rows = ne01 * ne02 * ne03; + const uint32_t row_size = ne00 * ct.dst_type_size; + + ct.div_ne01 = init_fastdiv_values(ne01); + ct.div_ne02_ne01 = init_fastdiv_values(ne02 * ne01); + + uint32_t row_start = 0; + uint32_t nrows = total_rows; + + if (octx->ctx->mdev.count > 1) { + const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1; + const bool can_split = htp_tensor_mdev_data_aligned(dst) && dst_is_contiguous; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, can_split ? rows_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + ct.row_start = row_start; + ct.nrows = nrows; + ct.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + + if (sametype && octx->ctx->mdev.count <= 1) { + *use_dma = true; + cpy_dma_sametype_sameshape(octx, dst, src0, ct.src0_type_size, ne00, ne01, ne02, ne03, nb01, nb02, nb03, nb1, nb2, nb3); + } else { + work_queue_func_t copy_fun = NULL; + if (sametype) { + copy_fun = (src0->type == HTP_TYPE_F32) ? cpy_thread_f32_sameshape : cpy_thread_f16_sameshape; + } else if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32) { + copy_fun = cpy_thread_f16_f32_sameshape; + } else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_F16) { + copy_fun = cpy_thread_f32_f16_sameshape; + } else { + return HTP_STATUS_NO_SUPPORT; + } + work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads); + } + } else if (sametype) { + const uint32_t total_elems = ne0 * ne1 * ne2 * ne3; + const uint32_t elems_per_line = (ct.dst_type_size == 4) ? 32 : 64; + + ct.div_ne0 = init_fastdiv_values(ne0); + ct.div_ne1_ne0 = init_fastdiv_values(ne1 * ne0); + ct.div_ne2_ne1_ne0 = init_fastdiv_values(ne2 * ne1 * ne0); + ct.div_ne00 = init_fastdiv_values(ne00); + ct.div_ne01_ne00 = init_fastdiv_values(ne01 * ne00); + ct.div_ne02_ne01_ne00 = init_fastdiv_values(ne02 * ne01 * ne00); + + uint32_t elem_start = 0; + uint32_t nelem = total_elems; + + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && dst_is_contiguous; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_elems, can_split ? elems_per_line : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + elem_start = range.start; + nelem = range.count; + } + + if (nelem == 0) { + return HTP_STATUS_OK; + } + + ct.elem_start = elem_start; + ct.nelem = nelem; + ct.elem_per_thread = fastdiv(nelem + n_threads - 1, &octx->n_threads_div); + + work_queue_func_t copy_fun = (src0->type == HTP_TYPE_F32) ? cpy_thread_f32_reshape : cpy_thread_f16_reshape; + work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads); } else { return HTP_STATUS_NO_SUPPORT; } - FARF(HIGH, "cpy-%s-%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) : use_dma=%d n_threads %u\n", - src0->type == HTP_TYPE_F32 ? "f32" : "f16", dst->type == HTP_TYPE_F32 ? "f32" : "f16", - ne00, ne01, ne02, ne03, ne0, ne1, ne2, ne3, use_dma, n_threads); - - if (use_dma) { - cpy_dma_sametype_sameshape(octx, dst, src0, ct.src0_type_size, ne00, ne01, ne02, ne03, nb01, nb02, nb03, nb1, nb2, nb3); - } else { - worker_pool_run_func(octx->ctx->worker_pool, copy_fun, &ct, n_threads); - } - - const struct htp_tensor *sync = octx->src[1]; - if (sync && (sync->flags & HTP_TENSOR_FENCE)) { - if (!use_dma) { - // htp_tensor_flush_all(octx->ctx, octx->dsts, 1); - qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE); - } - - atomic_uint * sync_fence = (atomic_uint *) sync->data; - const uint32_t seq = (uint32_t) octx->op_params[0]; - - atomic_store(&sync_fence[0], seq); - asm volatile ("syncht" : : : "memory"); - Q6_dccleaninva_A((void *) sync_fence); - - FARF(HIGH, "ggml-hex: sync-release : fence %p seq %u\n", sync_fence, seq); - } - return HTP_STATUS_OK; } + +int op_cpy(struct htp_ops_context * octx) { + bool use_dma = false; + int status = exec_cpy(octx, &use_dma); + + htp_ops_context_set_status(octx, status); + + if (octx->op == HTP_OP_CPY_FENCE) { + if (!use_dma) { + htp_flush_dirty_ranges(octx->ctx); + } + + htp_mdev_group_barrier(octx); + + if (octx->ctx->mdev.idx == 0) { + const struct htp_tensor * sync = octx->src[1]; + const uint32_t seq = (uint32_t) octx->op_params[0]; + atomic_uint * sync_fence = (atomic_uint *) (uintptr_t) sync->data; + htp_fence_write(sync_fence, seq, octx->status); + + FARF(HIGH, "ggml-hex: sync-release : fence %p seq 0x%x status %d\n", sync_fence, seq, octx->status); + } + } + + return octx->status; +} diff --git a/ggml/src/ggml-hexagon/htp/cumsum-ops.c b/ggml/src/ggml-hexagon/htp/cumsum-ops.c index 2d45c39f2..971fa3bcc 100644 --- a/ggml/src/ggml-hexagon/htp/cumsum-ops.c +++ b/ggml/src/ggml-hexagon/htp/cumsum-ops.c @@ -7,6 +7,8 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -17,25 +19,25 @@ #define htp_cumsum_tensors_preamble \ const struct htp_tensor * restrict src0 = octx->src[0]; \ const struct htp_tensor * restrict dst = octx->dst; \ - \ - const uint32_t ne00 = src0->ne[0]; \ - const uint32_t ne01 = src0->ne[1]; \ - const uint32_t ne02 = src0->ne[2]; \ - const uint32_t ne03 = src0->ne[3]; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb00 = src0->nb[0]; \ - const uint32_t nb01 = src0->nb[1]; \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t nb0 = dst->nb[0]; \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ + \ + const uint32_t ne00 = src0->ne[0]; \ + const uint32_t ne01 = src0->ne[1]; \ + const uint32_t ne02 = src0->ne[2]; \ + const uint32_t ne03 = src0->ne[3]; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb00 = src0->nb[0]; \ + const uint32_t nb01 = src0->nb[1]; \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t nb0 = dst->nb[0]; \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ const uint32_t nb3 = dst->nb[3]; struct htp_cumsum_context { @@ -46,6 +48,7 @@ struct htp_cumsum_context { size_t dst_row_size_aligned; uint32_t rows_per_thread; uint32_t total_rows; + uint32_t row_start; }; #define htp_cumsum_preamble \ @@ -116,11 +119,8 @@ static inline void hvx_cumsum_row_f32(const float * restrict src, float * restri static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) { htp_cumsum_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - - const uint32_t ir0 = cctx->rows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->total_rows); + const uint32_t ir0 = cctx->row_start + cctx->rows_per_thread * ith; + const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->row_start + cctx->total_rows); if (ir0 >= ir1) { return; @@ -149,11 +149,15 @@ static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * dat src_row_size_aligned, src_row_size, 1); } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = ir0; ir < ir1; ir++) { float * dst_spad_row = (float *) dma_queue_pop(dma_queue).src; float * src_spad_row = (float *) dma_queue_pop(dma_queue).dst; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); hvx_cumsum_row_f32(src_spad_row, dst_spad_row, ne00); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(dst_data + (ir * dst_row_size), (uint8_t *) dst_spad_row), @@ -168,12 +172,10 @@ static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * dat } dma_queue_flush(dma_queue); - t2 = HAP_perf_get_qtimer_count(); - FARF(HIGH, "cumsum-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", + FARF(HIGH, "cumsum-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n", ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1, - dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); } // --------------------------------------------------------------------------- @@ -183,14 +185,14 @@ static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * dat static void cumsum_thread_f32(unsigned int nth, unsigned int ith, void * data) { htp_cumsum_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - const uint8_t * src_data = (const uint8_t *) src0->data; uint8_t * dst_data = (uint8_t *) dst->data; - const uint32_t ir0 = cctx->rows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->total_rows); + const uint32_t ir0 = cctx->row_start + cctx->rows_per_thread * ith; + const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->row_start + cctx->total_rows); + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); for (uint32_t ir = ir0; ir < ir1; ir++) { const float * restrict src_row = (const float *) (src_data + ir * cctx->src_row_size); @@ -198,12 +200,11 @@ static void cumsum_thread_f32(unsigned int nth, unsigned int ith, void * data) { hvx_cumsum_row_f32(src_row, dst_row, ne00); } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); - FARF(HIGH, "cumsum-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", + FARF(HIGH, "cumsum-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n", ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1, - dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); } int op_cumsum_f32(struct htp_ops_context * octx) { @@ -214,8 +215,25 @@ int op_cumsum_f32(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, total_rows); + const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const size_t dst_data_row_size = dst->ne[0] * sizeof(float); + + uint32_t row_start = 0; + uint32_t nrows = total_rows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_data_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; const size_t src_row_size = src0->nb[1]; const size_t dst_row_size = dst->nb[1]; @@ -240,14 +258,15 @@ int op_cumsum_f32(struct htp_ops_context * octx) { .dst_row_size = dst_row_size, .src_row_size_aligned = src_row_size_aligned, .dst_row_size_aligned = dst_row_size_aligned, - .rows_per_thread = (total_rows + n_threads - 1) / n_threads, - .total_rows = total_rows, + .rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .total_rows = nrows, + .row_start = row_start, }; if (octx->ctx->vtcm_size < spad_per_thread * n_threads) { - worker_pool_run_func(octx->ctx->worker_pool, cumsum_thread_f32, &cctx, n_threads); + work_queue_run(octx->ctx->work_queue, cumsum_thread_f32, &cctx, n_threads); } else { - worker_pool_run_func(octx->ctx->worker_pool, cumsum_thread_f32_dma, &cctx, n_threads); + work_queue_run(octx->ctx->work_queue, cumsum_thread_f32_dma, &cctx, n_threads); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/diag-ops.c b/ggml/src/ggml-hexagon/htp/diag-ops.c index 9b3194d90..a69fd89d3 100644 --- a/ggml/src/ggml-hexagon/htp/diag-ops.c +++ b/ggml/src/ggml-hexagon/htp/diag-ops.c @@ -5,8 +5,11 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" #include "hvx-types.h" #include "hex-utils.h" #include "hvx-copy.h" @@ -15,17 +18,17 @@ #define htp_diag_tensors_preamble \ const struct htp_tensor * restrict src0 = octx->src[0]; \ const struct htp_tensor * restrict dst = octx->dst; \ - \ - const uint32_t ne02 = src0->ne[2]; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ + \ + const uint32_t ne02 = src0->ne[2]; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ const uint32_t nb3 = dst->nb[3]; struct htp_diag_context { @@ -36,6 +39,7 @@ struct htp_diag_context { size_t dst_row_size_aligned; uint32_t batches_per_thread; uint32_t total_batches; + uint32_t batch_start; }; #define htp_diag_preamble \ @@ -57,11 +61,8 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) htp_diag_preamble; dma_queue * dma_queue = octx->ctx->dma[ith]; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - - const uint32_t ib0 = dctx->batches_per_thread * ith; - const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->total_batches); + const uint32_t ib0 = dctx->batch_start + dctx->batches_per_thread * ith; + const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->batch_start + dctx->total_batches); if (ib0 >= ib1) { return; @@ -79,6 +80,8 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) uint8_t * src_spad = octx->src0_spad.data + (ith * src_batch_size_aligned); uint8_t * dst_spad = octx->dst_spad.data + (ith * dst_row_size_aligned); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ib = ib0; ib < ib1; ib++) { const uint32_t i3 = ib / ne02; const uint32_t i2 = ib % ne02; @@ -96,7 +99,9 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) for (uint32_t i1 = 0; i1 < ne1; i1++) { // Compute row in VTCM + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (ib * ne1 + i1)); hvx_diag_row_f32(src_spad_f32, dst_spad_f32, i1, ne0); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (ib * ne1 + i1)); // Write completed row back to DDR uint8_t * dst_row = dst_data + i3 * nb3 + i2 * nb2 + i1 * nb1; @@ -107,12 +112,9 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) } } - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "diag-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", + FARF(HIGH, "diag-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n", ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ib0, ib1, - dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); } // --------------------------------------------------------------------------- @@ -122,14 +124,14 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) static void diag_thread_f32(unsigned int nth, unsigned int ith, void * data) { htp_diag_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - const uint8_t * src_data = (const uint8_t *) src0->data; uint8_t * dst_data = (uint8_t *) dst->data; - const uint32_t ib0 = dctx->batches_per_thread * ith; - const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->total_batches); + const uint32_t ib0 = dctx->batch_start + dctx->batches_per_thread * ith; + const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->batch_start + dctx->total_batches); + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ib0); for (uint32_t ib = ib0; ib < ib1; ib++) { const uint32_t i3 = ib / ne02; @@ -143,12 +145,11 @@ static void diag_thread_f32(unsigned int nth, unsigned int ith, void * data) { } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ib0); - FARF(HIGH, "diag-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", + FARF(HIGH, "diag-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n", ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ib0, ib1, - dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); } int op_diag_f32(struct htp_ops_context * octx) { @@ -160,7 +161,36 @@ int op_diag_f32(struct htp_ops_context * octx) { } const uint32_t total_batches = src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, total_batches); + const size_t dst_batch_size = dst->ne[1] * dst->nb[1]; + + uint32_t batch_start = 0; + uint32_t nbatches = total_batches; + + if (octx->ctx->mdev.count > 1) { + bool can_split = htp_tensor_mdev_data_aligned(dst) && (dst->ne[0] == 1 || dst->nb[0] == sizeof(float)) && !htp_tensor_is_permuted(dst); + uint32_t batches_per_chunk = 1; + if (can_split) { + if (dst->ne[2] > 1 && (dst->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0 && + (dst->ne[3] <= 1 || (dst->nb[3] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0)) { + batches_per_chunk = 1; + } else if (dst->nb[2] == dst_batch_size && + (dst->ne[3] <= 1 || dst->nb[3] == dst->nb[2] * dst->ne[2])) { + batches_per_chunk = (dst_batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(dst_batch_size, HEX_L2_LINE_SIZE)) : 1; + } else { + can_split = false; + } + } + + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, can_split ? batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + batch_start = range.start; + nbatches = range.count; + } + + if (nbatches == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; const size_t src_batch_size = src0->ne[0] * sizeof(float); const size_t dst_row_size = dst->ne[0] * sizeof(float); @@ -185,14 +215,15 @@ int op_diag_f32(struct htp_ops_context * octx) { .dst_row_size = dst_row_size, .src_batch_size_aligned = src_batch_size_aligned, .dst_row_size_aligned = dst_row_size_aligned, - .batches_per_thread = (total_batches + n_threads - 1) / n_threads, - .total_batches = total_batches, + .batches_per_thread = fastdiv(nbatches + n_threads - 1, &octx->n_threads_div), + .total_batches = nbatches, + .batch_start = batch_start, }; if (octx->ctx->vtcm_size < spad_per_thread * n_threads) { - worker_pool_run_func(octx->ctx->worker_pool, diag_thread_f32, &dctx, n_threads); + work_queue_run(octx->ctx->work_queue, diag_thread_f32, &dctx, n_threads); } else { - worker_pool_run_func(octx->ctx->worker_pool, diag_thread_f32_dma, &dctx, n_threads); + work_queue_run(octx->ctx->work_queue, diag_thread_f32_dma, &dctx, n_threads); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/fill-ops.c b/ggml/src/ggml-hexagon/htp/fill-ops.c index 3ccfbe74e..1f6eaafad 100644 --- a/ggml/src/ggml-hexagon/htp/fill-ops.c +++ b/ggml/src/ggml-hexagon/htp/fill-ops.c @@ -3,10 +3,11 @@ #pragma clang diagnostic ignored "-Wunused-but-set-variable" #include -#include - #include +#include "hex-common.h" +#include "hex-profile.h" + #include "hvx-copy.h" #include "hvx-utils.h" @@ -14,28 +15,30 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" // ggml op_params layout for FILL: // op_params[0] (as float) - the scalar fill value -#define fill_preamble \ +#define fill_preamble \ const struct htp_tensor * dst = octx->dst; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; \ - \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ + const uint32_t nb3 = dst->nb[3]; \ + \ const uint32_t nr = ne1 * ne2 * ne3; struct htp_fill_context { struct htp_ops_context * octx; uint32_t nrows_per_thread; uint32_t total_rows; // ne1 * ne2 * ne3 + uint32_t row_start; bool opt_path; HVX_Vector splat_vec; uint32_t elem_size; @@ -47,10 +50,15 @@ static void fill_thread(unsigned int nth, unsigned int ith, void * data) { fill_preamble; // Parallelise over the flat row index spanning ne1*ne2*ne3 - const uint32_t ir0 = fctx->nrows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_thread, fctx->total_rows); + const uint32_t ir0 = fctx->row_start + fctx->nrows_per_thread * ith; + const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_thread, fctx->row_start + fctx->total_rows); - uint64_t t1 = HAP_perf_get_qtimer_count(); + if (ir0 >= ir1) { + return; + } + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); if (fctx->opt_path) { // Opt path: tensor is fully contiguous, treat as flat array @@ -69,9 +77,8 @@ static void fill_thread(unsigned int nth, unsigned int ith, void * data) { } } - uint64_t t2 = HAP_perf_get_qtimer_count(); - FARF(HIGH, "fill %u/%u: rows %u:%u usec %u\n", - ith, nth, ir0, ir1, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir1); + FARF(HIGH, "fill %u/%u: rows %u:%u\n", ith, nth, ir0, ir1); } int op_fill(struct htp_ops_context * octx) { @@ -85,8 +92,23 @@ int op_fill(struct htp_ops_context * octx) { return HTP_STATUS_OK; } + uint32_t row_start = 0; + uint32_t nrows = nr; + + if (octx->ctx->mdev.count > 1) { + const uint32_t row_size = nb1; + const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(nr, htp_tensor_mdev_data_aligned(dst) ? rows_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + // nr = ne1*ne2*ne3 (flat row count across all outer dims); parallelise over it. - const uint32_t n_threads = MIN(nr, octx->n_threads); + const uint32_t n_threads = octx->n_threads; // Optimize if fully contiguous: skip stride arithmetic, treat as flat array const bool opt_path = (nb2 == nb1 * ne1) && (nb3 == nb2 * ne2); @@ -99,8 +121,9 @@ int op_fill(struct htp_ops_context * octx) { struct htp_fill_context fctx = { .octx = octx, - .nrows_per_thread = (nr + n_threads - 1) / n_threads, - .total_rows = nr, + .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .total_rows = nrows, + .row_start = row_start, .opt_path = opt_path, }; @@ -117,7 +140,7 @@ int op_fill(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - worker_pool_run_func(octx->ctx->worker_pool, fill_thread, &fctx, n_threads); + work_queue_run(octx->ctx->work_queue, fill_thread, &fctx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c index c76b4d3a3..8a1caba22 100644 --- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -75,6 +74,7 @@ struct htp_fa_context { uint32_t qrows; uint32_t qrows_per_thread; + uint32_t qrow_start; bool is_q_fp32; @@ -89,8 +89,6 @@ struct htp_fa_context { const struct htp_tensor * k; const struct htp_tensor * v; - - uint64_t t_start; }; struct hmx_fa_context { @@ -206,10 +204,9 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * const uint32_t nb3 = dst->nb[3]; // total rows in q - const uint32_t nr = factx->qrows; - const uint32_t dr = factx->qrows_per_thread; - const uint32_t ir0 = dr * ith; - const uint32_t ir1 = MIN(ir0 + dr, nr); + const uint32_t dr = factx->qrows_per_thread; + const uint32_t ir0 = factx->qrow_start + dr * ith; + const uint32_t ir1 = MIN(ir0 + dr, factx->qrow_start + factx->qrows); if (ir0 >= ir1) return; @@ -1888,6 +1885,24 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { const uint32_t n_threads = factx.n_threads; const uint32_t G = factx.G; + // Multi-device: split Q blocks across devices + const uint32_t n_q_blocks = (neq1 + Br - 1) / Br; + uint32_t q_start_min = 0; + uint32_t q_start_max = neq1; + + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(n_q_blocks, htp_tensor_mdev_data_aligned(dst) ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + const uint32_t block_start = range.start; + const uint32_t block_end = range.start + range.count; + + if (block_start >= block_end) { + return HTP_STATUS_OK; + } + + q_start_min = block_start * Br; + q_start_max = MIN(block_end * Br, neq1); + } + // ======== VTCM allocation (GQA-aware) ======== // K/V row sizes drive the DMA descriptors (not the VTCM layout) and are used // throughout the KV loop below. @@ -1977,7 +1992,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { // ======== Main loop ======== for (uint32_t ib3 = 0; ib3 < neq3; ++ib3) { const uint32_t im3 = mask ? fastmodulo(ib3, mask->ne[3], &factx.src3_div3) : 0; - for (uint32_t q_start = 0; q_start < neq1; q_start += Br) { + for (uint32_t q_start = q_start_min; q_start < q_start_max; q_start += Br) { const uint32_t n_rows_q = hex_smin(Br, neq1 - q_start); const size_t n_rows_g = n_rows_q * G; const size_t g_br_actual = hex_align_up(n_rows_g, HMX_FP16_TILE_N_ROWS); @@ -1991,8 +2006,9 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { // 1. Push Q and KV DMAs for the very first iteration. // Subsequent iterations are enqueued early at the end of the previous iteration. - if (ib3 == 0 && q_start == 0 && kv_head == 0) { - const uint8_t * q_ptr = (const uint8_t *) q->data; + if (ib3 == 0 && q_start == q_start_min && kv_head == 0) { + const uint8_t * q_ptr = (const uint8_t *) q->data + q_start * q->nb[1] + + (kv_head * factx.G) * q->nb[2] + ib3 * q->nb[3]; const size_t q_row_bytes = q_transposed ? n_rows_q * q_row_bytes_trans_factor : q_row_bytes_untransposed; const size_t n_rows = q_transposed ? factx.G : n_rows_q; dma_queue_push(dma, dma_make_ptr(factx.vtcm_q_dma, q_ptr), q_row_bytes, hex_smax(q_src_stride, q_row_bytes), q_row_bytes, n_rows); @@ -2311,8 +2327,8 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { if (next_kv_head >= n_kv_heads) { next_kv_head = 0; next_q_start = q_start + Br; - if (next_q_start >= neq1) { - next_q_start = 0; + if (next_q_start >= q_start_max) { + next_q_start = q_start_min; next_ib3 = ib3 + 1; } } @@ -2398,6 +2414,10 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + if (kparams->kernel_type == HTP_FA_KERNEL_HMX) { return hmx_flash_attn_ext(octx); } @@ -2407,8 +2427,6 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { factx.k = k; factx.v = v; - factx.t_start = HAP_perf_get_qtimer_count(); - factx.src0_div21 = kparams->u.hvx.src0_div21; factx.src0_div1 = kparams->u.hvx.src0_div1; @@ -2451,8 +2469,30 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { } // total rows in q - factx.qrows = kparams->qrows; - factx.qrows_per_thread = kparams->qrows_per_thread; + const uint32_t neq1 = q->ne[1]; + const uint32_t neq2 = q->ne[2]; + const uint32_t neq3 = q->ne[3]; + const uint32_t total_qrows = neq1 * neq2 * neq3; + + uint32_t qrow_start = 0; + uint32_t qrows = total_qrows; + + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_qrows, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + qrow_start = range.start; + qrows = range.count; + } + + if (qrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + + factx.qrows = qrows; + factx.qrow_start = qrow_start; + factx.qrows_per_thread = fastdiv(qrows + n_threads - 1, &octx->n_threads_div); size_t size_vkq_acc = hex_round_up(v->ne[0] * sizeof(float), 128); // VKQ32 @@ -2461,18 +2501,18 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { uint8_t * vtcm_cur = octx->ctx->vtcm_base; - factx.spad_q = vtcm_seq_alloc(&vtcm_cur, size_q_block * octx->n_threads); - factx.spad_k = vtcm_seq_alloc(&vtcm_cur, factx.size_k_block * 2 * octx->n_threads); - factx.spad_v = vtcm_seq_alloc(&vtcm_cur, factx.size_v_block * 2 * octx->n_threads); - factx.spad_m = vtcm_seq_alloc(&vtcm_cur, (mask ? factx.size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) * octx->n_threads); - factx.spad_a = vtcm_seq_alloc(&vtcm_cur, size_vkq_acc * octx->n_threads); + factx.spad_q = vtcm_seq_alloc(&vtcm_cur, size_q_block * n_threads); + factx.spad_k = vtcm_seq_alloc(&vtcm_cur, factx.size_k_block * 2 * n_threads); + factx.spad_v = vtcm_seq_alloc(&vtcm_cur, factx.size_v_block * 2 * n_threads); + factx.spad_m = vtcm_seq_alloc(&vtcm_cur, (mask ? factx.size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) * n_threads); + factx.spad_a = vtcm_seq_alloc(&vtcm_cur, size_vkq_acc * n_threads); if ((size_t) (vtcm_cur - octx->ctx->vtcm_base) > octx->ctx->vtcm_size) { return HTP_STATUS_VTCM_TOO_SMALL; } if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { - work_queue_run(octx->ctx->work_queue, flash_attn_ext_f16_thread, &factx, octx->n_threads); + work_queue_run(octx->ctx->work_queue, flash_attn_ext_f16_thread, &factx, n_threads); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h index c4d190631..027845411 100644 --- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h @@ -51,6 +51,7 @@ struct htp_fa_kernel_params { uint32_t qrows; uint32_t qrows_per_thread; + uint32_t qrow_start; float m0; float m1; uint32_t n_head_log2; diff --git a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c index 966552152..0b6529571 100644 --- a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +++ b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c @@ -4,10 +4,13 @@ #include "hvx-utils.h" #include "hex-fastdiv.h" +#include "hex-common.h" +#include "hex-profile.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "htp-ctx.h" +#include "htp-tensor.h" #ifndef MIN #define MIN(a, b) ((a) < (b) ? (a) : (b)) @@ -22,6 +25,8 @@ struct htp_gdn_context { size_t state_bytes; uint8_t * vtcm_base; size_t vtcm_per_thread; + uint32_t row_start; + uint32_t nrows; }; static inline HVX_Vector gdn_mul_dot_f32(float * restrict dst, const float * restrict mul, const float * restrict dot, uint32_t n) { @@ -586,8 +591,9 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo const uint32_t n_seqs = v->ne[3]; const uint32_t K = octx->op_params[0]; - const uint32_t total_rows = H * n_seqs; - if (ith >= total_rows) { + const uint32_t row_end = gctx->row_start + gctx->nrows; + + if (ith >= gctx->nrows) { return; } @@ -621,11 +627,11 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo const uint64_t state_seq_stride = state->nb[3] / sizeof(float); const uint64_t state_size_per_snap = (uint64_t) S_v * S_v * H * n_seqs; - uint32_t ir_prefetch = ith; + uint32_t ir_prefetch = gctx->row_start + ith; int spad_idx = 0; // Prefetch preamble (up to 2 steps) - for (int k = 0; k < 2 && ir_prefetch < total_rows; k++) { + for (int k = 0; k < 2 && ir_prefetch < row_end; k++) { const uint32_t piv1 = fastmodulo(ir_prefetch, H, &fd_H); const uint32_t piv3 = fastdiv(ir_prefetch, &fd_H); const float * ps_in = state_in_base + (uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v; @@ -646,8 +652,11 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (gctx->row_start + ith)); + int curr_spad_idx = 0; - for (uint32_t ir = ith; ir < total_rows; ir += nth) { + for (uint32_t ir = gctx->row_start + ith; ir < row_end; ir += nth) { dma_queue_pop(dma); dma_queue_pop(dma); @@ -812,7 +821,7 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo S_v * sizeof(float), S_v); // Prefetch next block (if any) - if (ir_prefetch < total_rows) { + if (ir_prefetch < row_end) { const uint32_t piv1 = fastmodulo(ir_prefetch, H, &fd_H); const uint32_t piv3 = fastdiv(ir_prefetch, &fd_H); const float * ps_in = state_in_base + (uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v; @@ -828,6 +837,7 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo curr_spad_idx ^= 1; } dma_queue_flush(dma); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) row_end); } @@ -847,8 +857,9 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo const uint32_t H = v->ne[1]; const uint32_t n_seqs = v->ne[3]; - const uint32_t total_rows = H * n_seqs; - if (ith >= total_rows) { + const uint32_t row_end = gctx->row_start + gctx->nrows; + + if (ith >= gctx->nrows) { return; } @@ -881,11 +892,11 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo const uint64_t state_seq_stride = state->nb[3] / sizeof(float); - uint32_t ir_prefetch = ith; + uint32_t ir_prefetch = gctx->row_start + ith; int spad_idx = 0; // Prefetch preamble (up to 2 steps) - for (int k = 0; k < 2 && ir_prefetch < total_rows; k++) { + for (int k = 0; k < 2 && ir_prefetch < row_end; k++) { const uint32_t piv1 = fastmodulo(ir_prefetch, H, &fd_H); const uint32_t piv3 = fastdiv(ir_prefetch, &fd_H); const float * ps_in = state_in_base + (uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v; @@ -906,8 +917,11 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (gctx->row_start + ith)); + int curr_spad_idx = 0; - for (uint32_t ir = ith; ir < total_rows; ir += nth) { + for (uint32_t ir = gctx->row_start + ith; ir < row_end; ir += nth) { dma_queue_pop(dma); dma_queue_pop(dma); @@ -1057,7 +1071,7 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo S_v * sizeof(float), S_v); // Prefetch next block (if any) - if (ir_prefetch < total_rows) { + if (ir_prefetch < row_end) { const uint32_t piv1 = fastmodulo(ir_prefetch, H, &fd_H); const uint32_t piv3 = fastdiv(ir_prefetch, &fd_H); const float * ps_in = state_in_base + (uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v; @@ -1073,6 +1087,7 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo curr_spad_idx ^= 1; } dma_queue_flush(dma); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) row_end); } @@ -1085,10 +1100,6 @@ int op_gated_delta_net(struct htp_ops_context * octx) { const struct htp_tensor * state = octx->src[5]; const struct htp_tensor * dst = octx->dst; - if (!q || !k || !v || !g || !beta || !state || !dst) { - return HTP_STATUS_INVAL_PARAMS; - } - if (q->type != HTP_TYPE_F32 || k->type != HTP_TYPE_F32 || v->type != HTP_TYPE_F32 || g->type != HTP_TYPE_F32 || beta->type != HTP_TYPE_F32 || state->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_F32) { @@ -1124,16 +1135,37 @@ int op_gated_delta_net(struct htp_ops_context * octx) { return HTP_STATUS_OK; } + const uint32_t total_rows = H * n_seqs; + + uint32_t row_start = 0; + uint32_t nrows = total_rows; + + if (octx->ctx->mdev.count > 1) { + const uint32_t head_bytes = S_v * sizeof(float); + const uint32_t rows_per_chunk = (head_bytes > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(head_bytes, HEX_L2_LINE_SIZE)) : 1; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, htp_tensor_mdev_data_aligned(dst) ? rows_per_chunk : 0, + octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + struct htp_gdn_context gctx; gctx.octx = octx; - gctx.rows_per_thread = (H * n_seqs + octx->n_threads - 1) / octx->n_threads; + gctx.row_start = row_start; + gctx.nrows = nrows; + gctx.rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); gctx.state_bytes = (size_t) S_v * S_v * sizeof(float); size_t state_aligned = (size_t) S_v * S_v * sizeof(float); state_aligned = (state_aligned + 127) & ~(size_t)127; - assert(octx->ctx->vtcm_base != NULL); - assert(octx->ctx->vtcm_size >= 2 * state_aligned * octx->n_threads); + assert(octx->ctx->vtcm_size >= 2 * state_aligned * n_threads); gctx.vtcm_base = octx->ctx->vtcm_base; gctx.vtcm_per_thread = 2 * state_aligned; @@ -1148,9 +1180,9 @@ int op_gated_delta_net(struct htp_ops_context * octx) { gctx.vtcm_per_thread * octx->n_threads, octx->n_threads); if (n_tokens == 1) { - worker_pool_run_func(octx->ctx->worker_pool, gated_delta_net_f32_tg_thread, &gctx, octx->n_threads); + work_queue_run(octx->ctx->work_queue, gated_delta_net_f32_tg_thread, &gctx, n_threads); } else { - worker_pool_run_func(octx->ctx->worker_pool, gated_delta_net_f32_pp_thread, &gctx, octx->n_threads); + work_queue_run(octx->ctx->work_queue, gated_delta_net_f32_pp_thread, &gctx, n_threads); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.c b/ggml/src/ggml-hexagon/htp/get-rows-ops.c index a87962d22..d294ba57a 100644 --- a/ggml/src/ggml-hexagon/htp/get-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.c @@ -10,6 +10,7 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -23,9 +24,12 @@ struct get_rows_context { const struct htp_get_rows_kernel_params * kparams; struct htp_get_rows_vtcm_layout vtcm_layout; uint8_t * vtcm_base; + uint32_t task_start; + uint32_t tasks; + uint32_t tasks_per_thread; }; -#define get_rows_preamble \ +#define get_rows_preamble \ const uint32_t ne00 = octx->src[0]->ne[0]; \ const uint32_t ne01 = octx->src[0]->ne[1]; \ const uint32_t ne02 = octx->src[0]->ne[2]; \ @@ -61,12 +65,12 @@ static void get_rows_thread_st_##IDX_TYPE(unsigned int nth, unsigned int ith, vo struct htp_ops_context * octx = grctx->octx; \ const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \ get_rows_preamble; \ - const uint32_t dr = kparams->tasks_per_thread; \ - const uint32_t ir0 = dr * ith; \ - if (ir0 >= kparams->total_tasks) { \ + const uint32_t dr = grctx->tasks_per_thread; \ + const uint32_t ir0 = grctx->task_start + dr * ith; \ + if (ir0 >= grctx->task_start + grctx->tasks) { \ return; \ } \ - const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \ + const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); \ const uint32_t row_size_bytes = htp_tensor_get_row_size(octx->src[0]->type, ne00); \ dma_queue * dma_queue = octx->ctx->dma[ith]; \ for (uint32_t i = ir0; i < ir1; ++i) { \ @@ -101,12 +105,12 @@ static void get_rows_thread_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \ get_rows_preamble; \ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - const uint32_t dr = kparams->tasks_per_thread; \ - const uint32_t ir0 = dr * ith; \ - if (ir0 >= kparams->total_tasks) { \ + const uint32_t dr = grctx->tasks_per_thread; \ + const uint32_t ir0 = grctx->task_start + dr * ith; \ + if (ir0 >= grctx->task_start + grctx->tasks) { \ return; \ } \ - const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \ + const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); \ const uint32_t chunks_per_row = kparams->chunks_per_row; \ const uint32_t chunk_size = kparams->chunk_size; \ dma_queue * dma_queue = octx->ctx->dma[ith]; \ @@ -225,13 +229,41 @@ int op_get_rows(struct htp_ops_context * octx) { return HTP_STATUS_OK; } + const struct htp_tensor * dst = octx->dst; + const uint32_t total_tasks = kparams->total_tasks; + const size_t dst_row_size = htp_tensor_get_row_size(dst->type, dst->ne[0]); + + uint32_t task_start = 0; + uint32_t tasks = total_tasks; + + if (octx->ctx->mdev.count > 1) { + uint32_t tasks_per_chunk = 1; + htp_tensor_mdev_rows_per_chunk(dst, dst_row_size / dst->ne[0], (uint32_t) dst_row_size, &tasks_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_tasks, tasks_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + task_start = range.start; + tasks = range.count; + } + + if (tasks == 0) { + return HTP_STATUS_OK; + } + + if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t n_threads = octx->n_threads; + struct get_rows_context grctx; grctx.octx = octx; grctx.kparams = kparams; grctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base; + grctx.task_start = task_start; + grctx.tasks = tasks; + grctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div); const uint32_t ne00 = octx->src[0]->ne[0]; - htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, octx->src[0]->type, ne00, kparams->n_threads); + htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, octx->src[0]->type, ne00, n_threads); const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32); @@ -247,14 +279,14 @@ int op_get_rows(struct htp_ops_context * octx) { } } - FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu use_dma=%d n_threads %d\n", + FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu use-dma %d n-threads %d\n", octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3], octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3], octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3], - grctx.vtcm_layout.src0_bytes_per_thread * kparams->n_threads, - grctx.vtcm_layout.dst_bytes_per_thread * kparams->n_threads, - kparams->use_dma, kparams->n_threads); + grctx.vtcm_layout.src0_bytes_per_thread * n_threads, + grctx.vtcm_layout.dst_bytes_per_thread * n_threads, + kparams->use_dma, n_threads); - work_queue_run(octx->ctx->work_queue, q_func, &grctx, kparams->n_threads); + work_queue_run(octx->ctx->work_queue, q_func, &grctx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/hex-common.h b/ggml/src/ggml-hexagon/htp/hex-common.h index 4714486a0..e6a52540d 100644 --- a/ggml/src/ggml-hexagon/htp/hex-common.h +++ b/ggml/src/ggml-hexagon/htp/hex-common.h @@ -77,4 +77,13 @@ static inline bool hex_add_overflow(size_t a, size_t b, size_t *out) { return false; } +static inline uint32_t hex_gcd_u32(uint32_t a, uint32_t b) { + while (b != 0) { + uint32_t t = b; + b = a % b; + a = t; + } + return a; +} + #endif // HEX_COMMON_H diff --git a/ggml/src/ggml-hexagon/htp/hex-utils.h b/ggml/src/ggml-hexagon/htp/hex-utils.h index 1b3965030..853f1c1b2 100644 --- a/ggml/src/ggml-hexagon/htp/hex-utils.h +++ b/ggml/src/ggml-hexagon/htp/hex-utils.h @@ -39,7 +39,6 @@ static inline void hex_l2fetch_block(const void * addr, size_t size) { #define HEX_L2_LINE_SIZE 128 #define HEX_L2_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (lines per loop iteration) -#define HEX_L2_FLUSH_IL_THRESHOLD 1024 // inline flush threshold #define HEX_L2_FLUSH_WQ_THRESHOLD (4 * 1024) #define HEX_L2_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024) diff --git a/ggml/src/ggml-hexagon/htp/hmx-utils.h b/ggml/src/ggml-hexagon/htp/hmx-utils.h index 2a61ca734..ad295cb7d 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-utils.h +++ b/ggml/src/ggml-hexagon/htp/hmx-utils.h @@ -27,7 +27,7 @@ static inline void hmx_init_column_scales(void *out_scales, HVX_Vector v_scale) // vscatter offsets for fused dequant+transpose: write K-values directly to [K][N] tile. // word[i] = i*128 maps K-row-pair i to byte offset i*128. // Column offset (n*4) is added at runtime. Entries 0..15 cover one tile (region 2047); -// entries 16..31 cover the next adjacent tile (region 4095) — pick region size at the +// entries 16..31 cover the next adjacent tile (region 4095) - pick region size at the // call site to scatter into one tile (masked) or two contiguous tiles (unmasked). static const int32_t hmx_transpose_scatter_offsets[32] __attribute__((aligned(VLEN))) = { 0 * 128, 1 * 128, 2 * 128, 3 * 128, 4 * 128, 5 * 128, 6 * 128, 7 * 128, 8 * 128, 9 * 128, 10 * 128, @@ -198,16 +198,16 @@ static inline void hmx_interleave_cols_to_tiles(__fp16 * restrict tiles_out, } // --- HMX inline asm macros for load-store packetization --- -#define HMX_LOAD_MPY_F16(act, wt, range) \ - "{\n" \ +#define HMX_LOAD_MPY_F16(act, wt, range) \ + "{\n" \ " activation.hf = mxmem(" act ", " range ")\n" \ - " weight.hf = mxmem(" wt ", " range ")\n" \ + " weight.hf = mxmem(" wt ", " range ")\n" \ "}\n" -#define HMX_LOAD_MPY_DEEP_F16(act, wt, range) \ - "{\n" \ +#define HMX_LOAD_MPY_DEEP_F16(act, wt, range) \ + "{\n" \ " activation.hf = mxmem(" act ", " range "):deep\n" \ - " weight.hf = mxmem(" wt ", " range ")\n" \ + " weight.hf = mxmem(" wt ", " range ")\n" \ "}\n" #define HMX_STORE_AFTER_F16(out, scale_reg) \ diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.h index c8a909d61..3b60c8bdb 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ctx.h +++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h @@ -19,7 +19,7 @@ #endif #define HTP_MAX_MMAPS 16 -#define HTP_MAX_DIRTY_RANGES 16 +#define HTP_MAX_DIRTY_RANGES 32 // Memory mapping struct htp_mmap { @@ -29,6 +29,11 @@ struct htp_mmap { uint32_t reserved; }; +struct htp_dirty_range { + uint32_t start; + uint32_t end; +}; + // Scratchpad state struct htp_spad { const struct htp_tensor * src; // original src of the data (for reuse) @@ -38,6 +43,14 @@ struct htp_spad { uint32_t size_per_thread; // size per thread }; +struct htp_mdev_group { + uint16_t idx; + uint16_t count; + struct fastdiv_values count_div; + uint8_t * fence_base; + uint32_t fence_seq; +}; + struct htp_context; // Context while processing an Op @@ -65,8 +78,10 @@ struct htp_ops_context { struct htp_spad src3_spad; struct htp_spad dst_spad; - uint32_t n_threads; - uint32_t flags; + uint32_t flags; + uint32_t n_threads; + struct fastdiv_values n_threads_div; + int status; }; // Main context for htp DSP backend @@ -76,6 +91,7 @@ struct htp_context { struct htp_mmap mmap[HTP_MAX_MMAPS]; dma_queue_t dma[HTP_MAX_NTHREADS]; dma_queue_t dma_cached[HTP_MAX_NTHREADS]; + struct htp_thread_trace trace[HTP_MAX_NTHREADS + 1]; work_queue_t work_queue; hmx_queue_t hmx_queue; @@ -88,7 +104,6 @@ struct htp_context { bool hmx_enabled; bool etm; uint32_t profiler; - struct htp_thread_trace trace[HTP_MAX_NTHREADS + 1]; uint8_t * vtcm_base; size_t vtcm_size; @@ -97,16 +112,13 @@ struct htp_context { atomic_bool vtcm_needs_release; uint64_t max_vmem; - struct htp_dirty_range { - uint32_t start; - uint32_t end; - uint32_t bi; - } dirty_ranges[HTP_MAX_DIRTY_RANGES]; + struct htp_dirty_range dirty_ranges[HTP_MAX_DIRTY_RANGES]; // Persistent DDR scratchpad for MUL_MAT_ID mappings void * ddr_spad_base; size_t ddr_spad_size; + struct htp_mdev_group mdev; struct htp_ops_context octx; qurt_thread_t main_thread; @@ -115,6 +127,27 @@ struct htp_context { size_t footprint; }; +static inline bool htp_ops_context_set_n_threads(struct htp_ops_context * octx, uint32_t n_threads) { + if (n_threads == 0 || n_threads > octx->ctx->n_threads) { + return false; + } + + if (n_threads != octx->n_threads) { + octx->n_threads = n_threads; + octx->n_threads_div = n_threads == octx->ctx->n_threads + ? octx->ctx->n_threads_div + : init_fastdiv_values(n_threads); + } + + return true; +} + +static inline void htp_ops_context_set_status(struct htp_ops_context * octx, int status) { + if (status > HTP_STATUS_OK && octx->status == HTP_STATUS_OK) { + octx->status = status; + } +} + int op_matmul(struct htp_ops_context * octx); int op_matmul_id(struct htp_ops_context * octx); int op_matmul_nx(struct htp_ops_context * octx); diff --git a/ggml/src/ggml-hexagon/htp/htp-fence.h b/ggml/src/ggml-hexagon/htp/htp-fence.h new file mode 100644 index 000000000..7450b5de5 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/htp-fence.h @@ -0,0 +1,89 @@ +#ifndef HTP_FENCE_H +#define HTP_FENCE_H + +#include +#include + +#include + +#include "hex-utils.h" +#include "htp-ops.h" +#include "htp-ctx.h" + +static inline atomic_uint * htp_mdev_fence_slot(const void * fence_base, uint32_t idx) { + return (atomic_uint *) ((const uint8_t *) fence_base + (size_t) idx * HTP_FENCE_SLOT_SIZE); +} + +static inline void htp_fence_write(void * fence_ptr, uint32_t seq, uint32_t status) { + atomic_uint * fence = (atomic_uint *) fence_ptr; + atomic_store(&fence[1], status); + atomic_store(&fence[0], seq); + asm volatile ("syncht" : : : "memory"); + Q6_dccleaninva_A((void *) fence); +} + +static inline void htp_fence_read(const void * fence_ptr, uint32_t * seq, uint32_t * status) { + const atomic_uint * fence = (const atomic_uint *) fence_ptr; + Q6_dccleaninva_A((void *) fence); + asm volatile ("syncht" : : : "memory"); + *seq = atomic_load(&fence[0]); + *status = atomic_load(&fence[1]); +} + +static inline void htp_mdev_group_barrier(struct htp_ops_context * octx) { + struct htp_context * ctx = octx->ctx; + if (ctx->mdev.count <= 1) { + return; + } + + const uint32_t seq = ++ctx->mdev.fence_seq; + + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); + + const uint32_t mdev_idx = ctx->mdev.idx; + const uint32_t mdev_count = ctx->mdev.count; + + uint8_t * fence_base = ctx->mdev.fence_base; + atomic_uint * my_fence = htp_mdev_fence_slot(fence_base, mdev_idx); + htp_fence_write(my_fence, seq, octx->status); + + for (uint32_t d = 0; d < mdev_count; d++) { + if (d == mdev_idx) continue; + atomic_uint * peer_fence = htp_mdev_fence_slot(fence_base, d); + uint64_t spins = 0; + while (1) { + uint32_t peer_seq; + uint32_t peer_status; + htp_fence_read(peer_fence, &peer_seq, &peer_status); + if ((int32_t)(peer_seq - seq) >= 0) { + if (peer_status > HTP_STATUS_OK) { + FARF(ERROR, "ggml-hex: mdev %u peer %u failed with status %u : seq 0x%08x\n", + mdev_idx, d, peer_status, seq); + htp_ops_context_set_status(octx, peer_status); + } + break; + } + if (++spins == 10000) { + FARF(ALWAYS, "ggml-hex: mdev %u waiting for mdev %u : seq 0x%08x (b %u op %u) my-fence %p peer-fence %p peer-seq 0x%08x (diff %d)\n", + mdev_idx, d, seq, seq >> 12, seq & 0xfff, my_fence, peer_fence, peer_seq, (int32_t)(peer_seq - seq)); + } + if (spins > HTP_FENCE_TIMEOUT) { + FARF(ERROR, "ggml-hex: mdev %u timeout waiting for mdev %u : seq 0x%08x (b %u op %u) peer-fence %p peer-seq 0x%08x\n", + mdev_idx, d, seq, seq >> 12, seq & 0xfff, peer_fence, peer_seq); + htp_ops_context_set_status(octx, HTP_STATUS_INTERNAL_ERR); + break; + } + hex_pause(); + } + } + asm volatile ("syncht" : : : "memory"); + + if (octx->status > HTP_STATUS_OK) { + htp_fence_write(my_fence, seq, octx->status); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); +} + +#endif // HTP_FENCE_H diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h index 12a61b67f..869b19b8c 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ops.h +++ b/ggml/src/ggml-hexagon/htp/htp-ops.h @@ -77,6 +77,7 @@ enum htp_op_code { HTP_OP_GET_ROWS, HTP_OP_SCALE, HTP_OP_CPY, + HTP_OP_CPY_FENCE, HTP_OP_ARGSORT, HTP_OP_SQR, HTP_OP_SQRT, @@ -100,6 +101,7 @@ enum htp_op_code { HTP_OP_ALLREDUCE, HTP_OP_ALLREDUCE_ADD, HTP_OP_GLU_SWIGLU_CLAMP, + HTP_OP_MDEV_GROUP, HTP_OP_INVALID }; @@ -114,6 +116,7 @@ enum htp_op_code { #define HTP_OP_MAX_TENSORS 8192 // must stay under 64K (uint16) #define HTP_FENCE_TIMEOUT (1000000000ULL) +#define HTP_FENCE_SLOT_SIZE 128 #define HTP_OP_MAX_VMEM_DEFAULT (3355443200u) @@ -214,30 +217,26 @@ struct htp_prof_desc { }; struct htp_opbatch_req { - uint32_t id; // Batch id + uint64_t seq; // Sequence number uint32_t n_bufs; // Number of buffers uint32_t n_tensors; // Number of tensors uint32_t n_ops; // Number of ops uint32_t n_traces; // Number of trace descriptors per thread - uint32_t pad; // unused - uint64_t seq; // Sequence number // struct htp_buf_desc bufs[]; -- dspqueue buf 0 // struct htp_tensor tensors[]; -- dspqueue buf 0 // struct htp_op_desc ops[]; -- dspqueue buf 0 }; struct htp_opbatch_rsp { - uint32_t id; // Batch id - uint32_t status; // HTP_STATUS_... - uint32_t n_bufs; // Number of buffers - uint32_t n_tensors; // Number of tensors - uint32_t n_ops; // Number of op profile descriptors - uint32_t n_traces[HTP_MAX_NTHREADS + 1]; - uint32_t usecs; // Number of usec - uint32_t pad; // align to 8 bytes + uint64_t seq; // Sequence number uint64_t cycles_start; // Start cycle counter uint64_t cycles_stop; // Stop cycle counter - uint64_t seq; // Sequence number + uint32_t status; // HTP_STATUS_... + uint32_t n_bufs; // Number of buffers + uint32_t n_tensors; // Number of tensors + uint32_t n_ops; // Number of op profile descriptors + uint32_t usecs; // Number of usec + uint32_t n_traces[HTP_MAX_NTHREADS + 1]; // struct htp_prof_desc profs[]; -- dspqueue buf 0 }; diff --git a/ggml/src/ggml-hexagon/htp/htp-tensor.c b/ggml/src/ggml-hexagon/htp/htp-tensor.c index ae377c922..760ccd831 100644 --- a/ggml/src/ggml-hexagon/htp/htp-tensor.c +++ b/ggml/src/ggml-hexagon/htp/htp-tensor.c @@ -20,7 +20,7 @@ struct l2flush_range { struct l2flush_multi_task { struct htp_thread_trace * trace; - struct l2flush_range ranges[HTP_OP_MAX_INPUTS]; + struct l2flush_range ranges[HTP_MAX_DIRTY_RANGES]; uint32_t n_ranges; uint32_t total_blocks; uint32_t blocks_per_thread; @@ -73,6 +73,27 @@ static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, gb_first); } +static void merge_dirty_ranges(struct htp_context * ctx) { + for (uint32_t i = 0; i < HTP_MAX_DIRTY_RANGES; i++) { + struct htp_dirty_range * r = &ctx->dirty_ranges[i]; + if (!r->start) continue; + + for (uint32_t j = 0; j < HTP_MAX_DIRTY_RANGES;) { + struct htp_dirty_range * s = &ctx->dirty_ranges[j]; + if (i == j || !s->start || r->end < s->start || s->end < r->start) { + j++; + continue; + } + + r->start = MIN(r->start, s->start); + r->end = MAX(r->end, s->end); + s->start = 0; + s->end = 0; + j = 0; + } + } +} + void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) { const struct htp_tensor * pending[HTP_OP_MAX_OUTPUTS]; uint32_t n_pending = 0; @@ -83,11 +104,6 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co continue; } - if (t->size <= HEX_L2_FLUSH_IL_THRESHOLD) { - hex_l2flush((void *) (uintptr_t) t->data, t->size); - continue; - } - uint32_t t_start = t->data; uint32_t t_end = t_start + t->size; @@ -110,6 +126,8 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co } } + merge_dirty_ranges(ctx); + if (n_pending == 0) { return; } @@ -132,8 +150,8 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co struct htp_dirty_range * r = &ctx->dirty_ranges[idx]; r->start = pending[i]->data; r->end = pending[i]->data + pending[i]->size; - r->bi = pending[i]->bi; } + merge_dirty_ranges(ctx); return; } @@ -151,12 +169,12 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co struct htp_dirty_range * r = &ctx->dirty_ranges[i]; r->start = pending[i]->data; r->end = pending[i]->data + pending[i]->size; - r->bi = pending[i]->bi; } + merge_dirty_ranges(ctx); return; } - if (total_evict_size > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1 && n_evict <= HTP_OP_MAX_INPUTS) { + if (total_evict_size > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1 && n_evict <= HTP_MAX_DIRTY_RANGES) { struct l2flush_multi_task task; task.trace = ctx->trace; task.n_ranges = n_evict; @@ -195,7 +213,6 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co struct htp_dirty_range * r = &ctx->dirty_ranges[idx]; r->start = pending[i]->data; r->end = pending[i]->data + pending[i]->size; - r->bi = pending[i]->bi; } for (uint32_t i = 0; i < n_empty; i++) { @@ -203,8 +220,9 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co struct htp_dirty_range * r = &ctx->dirty_ranges[idx]; r->start = pending[n_evict + i]->data; r->end = pending[n_evict + i]->data + pending[n_evict + i]->size; - r->bi = pending[n_evict + i]->bi; } + + merge_dirty_ranges(ctx); } static void make_tensor_clean(struct htp_context * ctx, const struct htp_tensor * t) { @@ -242,15 +260,77 @@ static inline bool is_tensor_dirty(struct htp_context * ctx, const struct htp_te return false; } +static void flush_dirty_ranges(struct htp_context * ctx, const struct htp_dirty_range * ranges, uint32_t n_ranges, uint64_t total_dirty) { + if (total_dirty >= HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) { + struct l2flush_multi_task task; + task.trace = ctx->trace; + task.n_ranges = n_ranges; + + uint32_t block_acc = 0; + for (uint32_t i = 0; i < n_ranges; i++) { + const struct htp_dirty_range * r = &ranges[i]; + struct l2flush_range * rg = &task.ranges[i]; + rg->start = hex_align_down((size_t) r->start, HEX_L2_LINE_SIZE); + rg->end = hex_align_up((size_t) r->end, HEX_L2_LINE_SIZE); + rg->block_first = block_acc; + rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE; + block_acc += rg->n_blocks; + } + + task.total_blocks = block_acc; + task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div); + + work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads); + } else { + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0); + for (uint32_t i = 0; i < n_ranges; i++) { + const struct htp_dirty_range * r = &ranges[i]; + hex_l2flush((void *) (uintptr_t) r->start, r->end - r->start); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, 0); + } +} + +void htp_flush_dirty_ranges(struct htp_context * ctx) { + struct htp_dirty_range ranges[HTP_MAX_DIRTY_RANGES]; + uint32_t n_ranges = 0; + uint64_t total_dirty = 0; + + for (uint32_t i = 0; i < HTP_MAX_DIRTY_RANGES; i++) { + const struct htp_dirty_range * r = &ctx->dirty_ranges[i]; + if (!r->start) { + continue; + } + ranges[n_ranges++] = *r; + total_dirty += r->end - r->start; + } + + if (total_dirty == 0) { + return; + } + + if (total_dirty > HEX_L2_FLUSH_ALL_THRESHOLD) { + flush_all_dcache(ctx); + return; + } + + flush_dirty_ranges(ctx, ranges, n_ranges, total_dirty); + memset(ctx->dirty_ranges, 0, sizeof(ctx->dirty_ranges)); +} + void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) { const struct htp_tensor * dirty_tensors[HTP_OP_MAX_INPUTS]; + struct htp_dirty_range ranges[HTP_OP_MAX_INPUTS]; uint32_t n_dirty = 0; uint64_t total_dirty = 0; for (uint32_t i = 0; i < n; i++) { const struct htp_tensor * t = tensors[i]; - if (t && !(t->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_FENCE)) && is_tensor_dirty(ctx, t)) { + if (t && is_tensor_dirty(ctx, t)) { dirty_tensors[n_dirty++] = t; + ranges[n_dirty - 1].start = t->data; + ranges[n_dirty - 1].end = t->data + t->size; total_dirty += t->size; } } @@ -264,37 +344,8 @@ void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * co return; } - if (total_dirty >= HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) { - struct l2flush_multi_task task; - task.trace = ctx->trace; - task.n_ranges = 0; - - uint32_t block_acc = 0; - for (uint32_t i = 0; i < n_dirty; i++) { - const struct htp_tensor * t = dirty_tensors[i]; - make_tensor_clean(ctx, t); - - struct l2flush_range * rg = &task.ranges[task.n_ranges++]; - rg->start = hex_align_down((size_t) t->data, HEX_L2_LINE_SIZE); - rg->end = hex_align_up((size_t) t->data + t->size, HEX_L2_LINE_SIZE); - rg->block_first = block_acc; - rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE; - block_acc += rg->n_blocks; - } - - task.total_blocks = block_acc; - task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div); - - work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads); - return; - } - - struct htp_thread_trace * tr = &ctx->trace[0]; + flush_dirty_ranges(ctx, ranges, n_dirty, total_dirty); for (uint32_t i = 0; i < n_dirty; i++) { - const struct htp_tensor * t = dirty_tensors[i]; - htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, t->ti); - hex_l2flush((void *) (uintptr_t) t->data, t->size); - htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, t->ti); - make_tensor_clean(ctx, t); + make_tensor_clean(ctx, dirty_tensors[i]); } } diff --git a/ggml/src/ggml-hexagon/htp/htp-tensor.h b/ggml/src/ggml-hexagon/htp/htp-tensor.h index c9cadbae3..3afff6917 100644 --- a/ggml/src/ggml-hexagon/htp/htp-tensor.h +++ b/ggml/src/ggml-hexagon/htp/htp-tensor.h @@ -2,8 +2,20 @@ #define HTP_TENSOR_H #include +#include #include "htp-ops.h" #include "hex-bitmap.h" +#include "hex-common.h" +#include "hex-fastdiv.h" + +enum { + HTP_TENSOR_MDEV_LINE_SIZE = 128, +}; + +struct htp_tensor_mdev_range { + uint32_t start; + uint32_t count; +}; static inline void * htp_tensor_data(const struct htp_tensor * t) { return (void *) (uintptr_t) t->data; @@ -13,6 +25,102 @@ static inline uint32_t * htp_tensor_flags(const struct htp_tensor * t) { return (uint32_t *) &t->flags; } +static inline bool htp_tensor_is_contiguous(const struct htp_tensor * t, uint32_t type_size) { + uint32_t next_nb = type_size; + if (t->ne[0] != 1 && t->nb[0] != next_nb) { + return false; + } + next_nb *= t->ne[0]; + for (int i = 1; i < HTP_OP_MAX_DIMS; i++) { + if (t->ne[i] != 1 && t->nb[i] != next_nb) { + return false; + } + next_nb *= t->ne[i]; + } + return true; +} + +static inline bool htp_tensor_is_permuted(const struct htp_tensor * t) { + return t->nb[0] > t->nb[1] || t->nb[1] > t->nb[2] || t->nb[2] > t->nb[3]; +} + +static inline bool htp_tensor_mdev_data_aligned(const struct htp_tensor * t) { + return ((uintptr_t) t->data & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0; +} + +static inline bool htp_tensor_can_row_partition(const struct htp_tensor * t, uint32_t elem_size) { + if (!htp_tensor_mdev_data_aligned(t)) { + return false; + } + if (t->ne[0] != 1 && t->nb[0] != elem_size) { + return false; + } + if (htp_tensor_is_permuted(t)) { + return false; + } + if (t->ne[1] > 1 && (t->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) != 0) return false; + if (t->ne[2] > 1 && (t->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) != 0) return false; + if (t->ne[3] > 1 && (t->nb[3] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) != 0) return false; + return true; +} + +static inline bool htp_tensor_mdev_rows_per_chunk(const struct htp_tensor * t, uint32_t elem_size, uint32_t row_size, uint32_t * rows_per_chunk) { + *rows_per_chunk = 0; + + if (!htp_tensor_mdev_data_aligned(t)) { + return false; + } + if (t->ne[0] != 1 && t->nb[0] != elem_size) { + return false; + } + if (htp_tensor_is_permuted(t)) { + return false; + } + if (t->ne[1] > 1 && (t->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0 && + (t->ne[2] <= 1 || (t->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0) && + (t->ne[3] <= 1 || (t->nb[3] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0)) { + *rows_per_chunk = 1; + return true; + } + if (t->nb[1] == row_size && + (t->ne[2] <= 1 || t->nb[2] == t->nb[1] * t->ne[1]) && + (t->ne[3] <= 1 || t->nb[3] == t->nb[2] * t->ne[2])) { + *rows_per_chunk = (row_size > 0) ? (HTP_TENSOR_MDEV_LINE_SIZE / hex_gcd_u32(row_size, HTP_TENSOR_MDEV_LINE_SIZE)) : 1; + return true; + } + return false; +} + +static inline struct htp_tensor_mdev_range htp_tensor_mdev_partition(uint32_t total_units, uint32_t units_per_chunk, uint32_t mdev_idx, uint32_t mdev_count, const struct fastdiv_values * mdev_count_div) { + struct htp_tensor_mdev_range range = { 0, total_units }; + + if (mdev_count <= 1) { + return range; + } + + if (units_per_chunk == 0) { + range.start = (mdev_idx == 0) ? 0 : total_units; + range.count = (mdev_idx == 0) ? total_units : 0; + return range; + } + + const uint32_t total_chunks = total_units / units_per_chunk; + if (total_chunks < mdev_count) { + range.start = (mdev_idx == 0) ? 0 : total_units; + range.count = (mdev_idx == 0) ? total_units : 0; + return range; + } + + const uint32_t chunks_per_mdev = fastdiv(total_chunks + mdev_count - 1, mdev_count_div); + range.start = MIN(mdev_idx * chunks_per_mdev * units_per_chunk, total_units); + if (mdev_idx == mdev_count - 1) { + range.count = total_units - range.start; + } else { + range.count = MIN(chunks_per_mdev * units_per_chunk, total_units - range.start); + } + return range; +} + static inline uint32_t htp_tensor_get_row_size(int type, uint32_t ne00) { switch (type) { case HTP_TYPE_F32: return ne00 * 4; @@ -23,6 +131,7 @@ static inline uint32_t htp_tensor_get_row_size(int type, uint32_t ne00) { } struct htp_context; +void htp_flush_dirty_ranges(struct htp_context * ctx); void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n); void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n); diff --git a/ggml/src/ggml-hexagon/htp/hvx-arith.h b/ggml/src/ggml-hexagon/htp/hvx-arith.h index fe5477c1b..6cbead74c 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-arith.h +++ b/ggml/src/ggml-hexagon/htp/hvx-arith.h @@ -16,25 +16,25 @@ #define UNUSED(x) (void)(x) #define hvx_arith_loop_body(dst_type, src0_type, src1_type, elem_size, vec_store, vec_op) \ - do { \ - dst_type * vdst = (dst_type *) dst; \ - src0_type * vsrc0 = (src0_type *) src0; \ - src1_type * vsrc1 = (src1_type *) src1; \ - \ - const uint32_t epv = 128 / (elem_size); \ - const uint32_t nvec = n / epv; \ - const uint32_t nloe = n % epv; \ - \ - uint32_t i = 0; \ - \ - _Pragma("unroll(4)") \ - for (; i < nvec; i++) { \ - vdst[i] = vec_op(vsrc0[i], vsrc1[i]); \ - } \ - if (nloe) { \ - HVX_Vector v = vec_op(vsrc0[i], vsrc1[i]); \ - vec_store((void *) &vdst[i], nloe * (elem_size), v); \ - } \ + do { \ + dst_type * vdst = (dst_type *) dst; \ + src0_type * vsrc0 = (src0_type *) src0; \ + src1_type * vsrc1 = (src1_type *) src1; \ + \ + const uint32_t epv = 128 / (elem_size); \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = vec_op(vsrc0[i], vsrc1[i]); \ + } \ + if (nloe) { \ + HVX_Vector v = vec_op(vsrc0[i], vsrc1[i]); \ + vec_store((void *) &vdst[i], nloe * (elem_size), v); \ + } \ } while(0) #if __HVX_ARCH__ < 79 @@ -56,43 +56,43 @@ #define HVX_OP_MUL_F16(a, b) hvx_vec_mul_f16_f16(a, b) // Generic macro to define alignment permutations for an op -#define DEFINE_HVX_BINARY_OP_VARIANTS(OP_NAME, OP_MACRO, ELEM_TYPE) \ -static inline void OP_NAME##_aaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src0 % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ -static inline void OP_NAME##_aau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src0 % 128 == 0); \ - hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ -static inline void OP_NAME##_aua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ -static inline void OP_NAME##_auu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ -static inline void OP_NAME##_uaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ - assert((uintptr_t) src0 % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ -static inline void OP_NAME##_uau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ - assert((uintptr_t) src0 % 128 == 0); \ - hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ -static inline void OP_NAME##_uua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ - assert((uintptr_t) src1 % 128 == 0); \ - hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ -static inline void OP_NAME##_uuu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ +#define DEFINE_HVX_BINARY_OP_VARIANTS(OP_NAME, OP_MACRO, ELEM_TYPE) \ +static inline void OP_NAME##_aaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src0 % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ +static inline void OP_NAME##_aau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src0 % 128 == 0); \ + hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ +static inline void OP_NAME##_aua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ +static inline void OP_NAME##_auu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) dst % 128 == 0); \ + hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ +static inline void OP_NAME##_uaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) src0 % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ +} \ +static inline void OP_NAME##_uau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) src0 % 128 == 0); \ + hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ +} \ +static inline void OP_NAME##_uua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) src1 % 128 == 0); \ + hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ +} \ +static inline void OP_NAME##_uuu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ +} \ DEFINE_HVX_BINARY_OP_VARIANTS(hvx_add_f32, HVX_OP_ADD_F32, float) DEFINE_HVX_BINARY_OP_VARIANTS(hvx_sub_f32, HVX_OP_SUB_F32, float) @@ -103,25 +103,25 @@ DEFINE_HVX_BINARY_OP_VARIANTS(hvx_sub_f16, HVX_OP_SUB_F16, _Float16) DEFINE_HVX_BINARY_OP_VARIANTS(hvx_mul_f16, HVX_OP_MUL_F16, _Float16) // Dispatcher logic -#define HVX_BINARY_DISPATCHER(OP_NAME) \ +#define HVX_BINARY_DISPATCHER(OP_NAME) \ static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint32_t num_elems) { \ - if (hex_is_aligned((void *) dst, 128)) { \ - if (hex_is_aligned((void *) src0, 128)) { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \ - else OP_NAME##_aau(dst, src0, src1, num_elems); \ - } else { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \ - else OP_NAME##_auu(dst, src0, src1, num_elems); \ - } \ - } else { \ - if (hex_is_aligned((void *) src0, 128)) { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \ - else OP_NAME##_uau(dst, src0, src1, num_elems); \ - } else { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \ - else OP_NAME##_uuu(dst, src0, src1, num_elems); \ - } \ - } \ + if (hex_is_aligned((void *) dst, 128)) { \ + if (hex_is_aligned((void *) src0, 128)) { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \ + else OP_NAME##_aau(dst, src0, src1, num_elems); \ + } else { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \ + else OP_NAME##_auu(dst, src0, src1, num_elems); \ + } \ + } else { \ + if (hex_is_aligned((void *) src0, 128)) { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \ + else OP_NAME##_uau(dst, src0, src1, num_elems); \ + } else { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \ + else OP_NAME##_uuu(dst, src0, src1, num_elems); \ + } \ + } \ } HVX_BINARY_DISPATCHER(hvx_add_f32) @@ -166,44 +166,44 @@ static inline void hvx_mul_mul_f32_aa(uint8_t * restrict dst, const uint8_t * re // Scalar Operations -#define hvx_scalar_loop_body(dst_type, src_type, elem_size, vec_store, scalar_op_macro) \ - do { \ - dst_type * restrict vdst = (dst_type *) dst; \ - src_type * restrict vsrc = (src_type *) src; \ - \ - const uint32_t epv = 128 / (elem_size); \ - const uint32_t nvec = n / epv; \ - const uint32_t nloe = n % epv; \ - \ - uint32_t i = 0; \ - \ - _Pragma("unroll(4)") \ - for (; i < nvec; i++) { \ - HVX_Vector v = vsrc[i]; \ - vdst[i] = scalar_op_macro(v); \ - } \ - if (nloe) { \ - HVX_Vector v = vsrc[i]; \ - v = scalar_op_macro(v); \ - vec_store((void *) &vdst[i], nloe * (elem_size), v); \ - } \ +#define hvx_scalar_loop_body(dst_type, src_type, elem_size, vec_store, scalar_op_macro) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t epv = 128 / (elem_size); \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + HVX_Vector v = vsrc[i]; \ + vdst[i] = scalar_op_macro(v); \ + } \ + if (nloe) { \ + HVX_Vector v = vsrc[i]; \ + v = scalar_op_macro(v); \ + vec_store((void *) &vdst[i], nloe * (elem_size), v); \ + } \ } while(0) -#define HVX_OP_ADD_SCALAR_F32(v) \ - ({ \ +#define HVX_OP_ADD_SCALAR_F32(v) \ + ({ \ const HVX_VectorPred pred_inf = Q6_Q_vcmp_eq_VwVw(inf, v); \ - HVX_Vector out = HVX_OP_ADD_F32(v, val_vec); \ - Q6_V_vmux_QVV(pred_inf, inf, out); \ + HVX_Vector out = HVX_OP_ADD_F32(v, val_vec); \ + Q6_V_vmux_QVV(pred_inf, inf, out); \ }) #define HVX_OP_MUL_SCALAR_F32(v) HVX_OP_MUL_F32(v, val_vec) #define HVX_OP_SUB_SCALAR_F32(v) HVX_OP_SUB_F32(v, val_vec) -#define HVX_OP_ADD_SCALAR_F16(v) \ - ({ \ +#define HVX_OP_ADD_SCALAR_F16(v) \ + ({ \ const HVX_VectorPred pred_inf = Q6_Q_vcmp_eq_VhVh(inf, v); \ - HVX_Vector out = HVX_OP_ADD_F16(v, val_vec); \ - Q6_V_vmux_QVV(pred_inf, inf, out); \ + HVX_Vector out = HVX_OP_ADD_F16(v, val_vec); \ + Q6_V_vmux_QVV(pred_inf, inf, out); \ }) #define HVX_OP_MUL_SCALAR_F16(v) HVX_OP_MUL_F16(v, val_vec) @@ -212,31 +212,31 @@ static inline void hvx_mul_mul_f32_aa(uint8_t * restrict dst, const uint8_t * re // Scalar Variants // Generic macro to define alignment permutations for an op -#define DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(OP_NAME, OP_MACRO, SPLAT_MACRO, ELEM_TYPE) \ +#define DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(OP_NAME, OP_MACRO, SPLAT_MACRO, ELEM_TYPE) \ static inline void OP_NAME##_aa(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \ - const HVX_Vector val_vec = SPLAT_MACRO(val); \ - const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src % 128 == 0); \ - hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ + const HVX_Vector val_vec = SPLAT_MACRO(val); \ + const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src % 128 == 0); \ + hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ static inline void OP_NAME##_au(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \ - const HVX_Vector val_vec = SPLAT_MACRO(val); \ - const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ - assert((uintptr_t) dst % 128 == 0); \ - hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ + const HVX_Vector val_vec = SPLAT_MACRO(val); \ + const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ + assert((uintptr_t) dst % 128 == 0); \ + hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ static inline void OP_NAME##_ua(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \ - const HVX_Vector val_vec = SPLAT_MACRO(val); \ - const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ - assert((uintptr_t) src % 128 == 0); \ - hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ + const HVX_Vector val_vec = SPLAT_MACRO(val); \ + const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ + assert((uintptr_t) src % 128 == 0); \ + hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ +} \ static inline void OP_NAME##_uu(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \ - const HVX_Vector val_vec = SPLAT_MACRO(val); \ - const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ - hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ + const HVX_Vector val_vec = SPLAT_MACRO(val); \ + const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ + hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ +} \ DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_add_scalar_f32, HVX_OP_ADD_SCALAR_F32, hvx_vec_splat_f32, float) DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_sub_scalar_f32, HVX_OP_SUB_SCALAR_F32, hvx_vec_splat_f32, float) @@ -247,17 +247,17 @@ DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_sub_scalar_f16, HVX_OP_SUB_SCALAR_F16, DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_mul_scalar_f16, HVX_OP_MUL_SCALAR_F16, hvx_vec_splat_f16, _Float16) // Dispatcher logic -#define HVX_BINARY_SCALAR_DISPATCHER(OP_NAME, ELEM_TYPE) \ +#define HVX_BINARY_SCALAR_DISPATCHER(OP_NAME, ELEM_TYPE) \ static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, const uint32_t num_elems) { \ - if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \ - OP_NAME##_aa(dst, src, val, num_elems); \ - } else if (hex_is_aligned((void *) dst, 128)) { \ - OP_NAME##_au(dst, src, val, num_elems); \ - } else if (hex_is_aligned((void *) src, 128)) { \ - OP_NAME##_ua(dst, src, val, num_elems); \ - } else { \ - OP_NAME##_uu(dst, src, val, num_elems); \ - } \ + if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \ + OP_NAME##_aa(dst, src, val, num_elems); \ + } else if (hex_is_aligned((void *) dst, 128)) { \ + OP_NAME##_au(dst, src, val, num_elems); \ + } else if (hex_is_aligned((void *) src, 128)) { \ + OP_NAME##_ua(dst, src, val, num_elems); \ + } else { \ + OP_NAME##_uu(dst, src, val, num_elems); \ + } \ } HVX_BINARY_SCALAR_DISPATCHER(hvx_add_scalar_f32, float) @@ -350,12 +350,12 @@ static inline void hvx_max_scalar_f32(uint8_t * restrict dst, const uint8_t * re // CLAMP Scalar variants -#define HVX_OP_CLAMP_SCALAR(v) \ - ({ \ +#define HVX_OP_CLAMP_SCALAR(v) \ + ({ \ HVX_VectorPred pred_cap_right = Q6_Q_vcmp_gt_VsfVsf(v, max_vec); \ HVX_VectorPred pred_cap_left = Q6_Q_vcmp_gt_VsfVsf(min_vec, v); \ - HVX_Vector tmp = Q6_V_vmux_QVV(pred_cap_right, max_vec, v); \ - Q6_V_vmux_QVV(pred_cap_left, min_vec, tmp); \ + HVX_Vector tmp = Q6_V_vmux_QVV(pred_cap_right, max_vec, v); \ + Q6_V_vmux_QVV(pred_cap_left, min_vec, tmp); \ }) static inline void hvx_clamp_scalar_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const float min, const float max, uint32_t n) { diff --git a/ggml/src/ggml-hexagon/htp/hvx-div.h b/ggml/src/ggml-hexagon/htp/hvx-div.h index 53ee304e7..bb7ab0519 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-div.h +++ b/ggml/src/ggml-hexagon/htp/hvx-div.h @@ -219,64 +219,64 @@ static inline HVX_Vector hvx_vec_hybrid_div_f16(HVX_Vector vec1, HVX_Vector vec2 } while(0) // Generic macro to define alignment permutations for an op -#define DEFINE_HVX_DIV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \ +#define DEFINE_HVX_DIV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \ static inline void OP_NAME##_aaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src0 % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_Vector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src0 % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_Vector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_aau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src0 % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_UVector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src0 % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_UVector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_aua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_Vector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_Vector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_auu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_UVector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_UVector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_uaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) src0 % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_Vector, hvx_vec_store_u); \ -} \ + assert((uintptr_t) src0 % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_Vector, hvx_vec_store_u); \ +} \ static inline void OP_NAME##_uau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) src0 % 128 == 0); \ - OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_UVector, hvx_vec_store_u); \ -} \ + assert((uintptr_t) src0 % 128 == 0); \ + OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_UVector, hvx_vec_store_u); \ +} \ static inline void OP_NAME##_uua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) src1 % 128 == 0); \ - OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_Vector, hvx_vec_store_u); \ -} \ + assert((uintptr_t) src1 % 128 == 0); \ + OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_Vector, hvx_vec_store_u); \ +} \ static inline void OP_NAME##_uuu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_UVector, hvx_vec_store_u); \ -} \ + OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_UVector, hvx_vec_store_u); \ +} \ // Dispatcher logic -#define HVX_DIV_DISPATCHER(OP_NAME) \ +#define HVX_DIV_DISPATCHER(OP_NAME) \ static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint32_t num_elems) { \ - if (hex_is_aligned((void *) dst, 128)) { \ - if (hex_is_aligned((void *) src0, 128)) { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \ - else OP_NAME##_aau(dst, src0, src1, num_elems); \ - } else { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \ - else OP_NAME##_auu(dst, src0, src1, num_elems); \ - } \ - } else { \ - if (hex_is_aligned((void *) src0, 128)) { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \ - else OP_NAME##_uau(dst, src0, src1, num_elems); \ - } else { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \ - else OP_NAME##_uuu(dst, src0, src1, num_elems); \ - } \ - } \ + if (hex_is_aligned((void *) dst, 128)) { \ + if (hex_is_aligned((void *) src0, 128)) { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \ + else OP_NAME##_aau(dst, src0, src1, num_elems); \ + } else { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \ + else OP_NAME##_auu(dst, src0, src1, num_elems); \ + } \ + } else { \ + if (hex_is_aligned((void *) src0, 128)) { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \ + else OP_NAME##_uau(dst, src0, src1, num_elems); \ + } else { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \ + else OP_NAME##_uuu(dst, src0, src1, num_elems); \ + } \ + } \ } DEFINE_HVX_DIV_OP_VARIANTS(hvx_div_f32, hvx_div_f32_loop_body) diff --git a/ggml/src/ggml-hexagon/htp/hvx-inverse.h b/ggml/src/ggml-hexagon/htp/hvx-inverse.h index f2054f45b..256a8843b 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-inverse.h +++ b/ggml/src/ggml-hexagon/htp/hvx-inverse.h @@ -169,36 +169,36 @@ static inline HVX_Vector hvx_vec_inverse_f16_guard(HVX_Vector v_sf, HVX_Vector n } while(0) // Generic macro to define alignment permutations for an op -#define DEFINE_HVX_INV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \ +#define DEFINE_HVX_INV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \ static inline void OP_NAME##_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_Vector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_Vector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_UVector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_UVector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \ - assert((uintptr_t) src % 128 == 0); \ - OP_LOOP_BODY(HVX_UVector, HVX_Vector, hvx_vec_store_u); \ -} \ + assert((uintptr_t) src % 128 == 0); \ + OP_LOOP_BODY(HVX_UVector, HVX_Vector, hvx_vec_store_u); \ +} \ static inline void OP_NAME##_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \ - OP_LOOP_BODY(HVX_UVector, HVX_UVector, hvx_vec_store_u); \ -} \ + OP_LOOP_BODY(HVX_UVector, HVX_UVector, hvx_vec_store_u); \ +} \ // Dispatcher logic -#define HVX_INV_DISPATCHER(OP_NAME) \ +#define HVX_INV_DISPATCHER(OP_NAME) \ static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t num_elems) { \ - if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \ - OP_NAME##_aa(dst, src, num_elems); \ - } else if (hex_is_aligned((void *) dst, 128)) { \ - OP_NAME##_au(dst, src, num_elems); \ - } else if (hex_is_aligned((void *) src, 128)) { \ - OP_NAME##_ua(dst, src, num_elems); \ - } else { \ - OP_NAME##_uu(dst, src, num_elems); \ - } \ + if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \ + OP_NAME##_aa(dst, src, num_elems); \ + } else if (hex_is_aligned((void *) dst, 128)) { \ + OP_NAME##_au(dst, src, num_elems); \ + } else if (hex_is_aligned((void *) src, 128)) { \ + OP_NAME##_ua(dst, src, num_elems); \ + } else { \ + OP_NAME##_uu(dst, src, num_elems); \ + } \ } DEFINE_HVX_INV_OP_VARIANTS(hvx_inverse_f32, hvx_inverse_f32_loop_body) diff --git a/ggml/src/ggml-hexagon/htp/hvx-scale.h b/ggml/src/ggml-hexagon/htp/hvx-scale.h index 9b1a28f52..5d0650307 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-scale.h +++ b/ggml/src/ggml-hexagon/htp/hvx-scale.h @@ -68,30 +68,30 @@ static inline void hvx_scale_f32(uint8_t * restrict dst, const uint8_t * restric } } -#define hvx_scale_offset_f32_loop_body(dst_type, src_type, vec_store) \ - do { \ - dst_type * restrict vdst = (dst_type *) dst; \ - src_type * restrict vsrc = (src_type *) src; \ - \ - HVX_Vector vs = hvx_vec_splat_f32(scale); \ - HVX_Vector vo = hvx_vec_splat_f32(offset); \ - \ - const uint32_t elem_size = sizeof(float); \ - const uint32_t epv = 128 / elem_size; \ - const uint32_t nvec = n / epv; \ - const uint32_t nloe = n % epv; \ - \ - uint32_t i = 0; \ - \ - _Pragma("unroll(4)") \ - for (; i < nvec; ++i) { \ +#define hvx_scale_offset_f32_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + HVX_Vector vs = hvx_vec_splat_f32(scale); \ + HVX_Vector vo = hvx_vec_splat_f32(offset); \ + \ + const uint32_t elem_size = sizeof(float); \ + const uint32_t epv = 128 / elem_size; \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; ++i) { \ HVX_Vector v = Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs), vo); \ - vdst[i] = Q6_Vsf_equals_Vqf32(v); \ - } \ - if (nloe) { \ + vdst[i] = Q6_Vsf_equals_Vqf32(v); \ + } \ + if (nloe) { \ HVX_Vector v = Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs), vo); \ - vec_store((void *) &vdst[i], nloe * elem_size, Q6_Vsf_equals_Vqf32(v)); \ - } \ + vec_store((void *) &vdst[i], nloe * elem_size, Q6_Vsf_equals_Vqf32(v)); \ + } \ } while(0) static inline void hvx_scale_offset_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) { diff --git a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h index dd66dd84c..552017309 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +++ b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h @@ -68,50 +68,50 @@ static inline HVX_Vector hvx_vec_tanh_f32(HVX_Vector x) { return Q6_Vsf_equals_Vqf32(res); } -#define hvx_sigmoid_loop_body(dst_type, src_type, vec_store) \ - do { \ - dst_type * restrict vdst = (dst_type *) dst; \ - src_type * restrict vsrc = (src_type *) src; \ - \ - const HVX_Vector one = hvx_vec_splat_f32(1.f); \ - const HVX_Vector max_exp = hvx_vec_splat_f32(87.f); \ - const HVX_Vector min_exp = hvx_vec_splat_f32(-87.f); \ - \ - const uint32_t epv = 128 / sizeof(float); \ - const uint32_t nvec = n / epv; \ - const uint32_t nloe = n % epv; \ - \ - uint32_t i = 0; \ - \ - _Pragma("unroll(4)") \ - for (; i < nvec; i++) { \ - vdst[i] = hvx_vec_fast_sigmoid_f32_guard(vsrc[i], one, max_exp, min_exp); \ - } \ - if (nloe) { \ +#define hvx_sigmoid_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const HVX_Vector one = hvx_vec_splat_f32(1.f); \ + const HVX_Vector max_exp = hvx_vec_splat_f32(87.f); \ + const HVX_Vector min_exp = hvx_vec_splat_f32(-87.f); \ + \ + const uint32_t epv = 128 / sizeof(float); \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = hvx_vec_fast_sigmoid_f32_guard(vsrc[i], one, max_exp, min_exp); \ + } \ + if (nloe) { \ HVX_Vector tmp = hvx_vec_fast_sigmoid_f32_guard(vsrc[i], one, max_exp, min_exp); \ - vec_store((void *) &vdst[i], nloe * sizeof(float), tmp); \ - } \ + vec_store((void *) &vdst[i], nloe * sizeof(float), tmp); \ + } \ } while(0) -#define hvx_tanh_loop_body(dst_type, src_type, vec_store) \ - do { \ - dst_type * restrict vdst = (dst_type *) dst; \ - src_type * restrict vsrc = (src_type *) src; \ - \ - const uint32_t epv = 128 / sizeof(float); \ - const uint32_t nvec = n / epv; \ - const uint32_t nloe = n % epv; \ - \ - uint32_t i = 0; \ - \ - _Pragma("unroll(4)") \ - for (; i < nvec; i++) { \ - vdst[i] = hvx_vec_tanh_f32(vsrc[i]); \ - } \ - if (nloe) { \ - HVX_Vector tmp = hvx_vec_tanh_f32(vsrc[i]); \ +#define hvx_tanh_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t epv = 128 / sizeof(float); \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = hvx_vec_tanh_f32(vsrc[i]); \ + } \ + if (nloe) { \ + HVX_Vector tmp = hvx_vec_tanh_f32(vsrc[i]); \ vec_store((void *) &vdst[i], nloe * sizeof(float), tmp); \ - } \ + } \ } while(0) static inline void hvx_sigmoid_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { diff --git a/ggml/src/ggml-hexagon/htp/im2col-ops.c b/ggml/src/ggml-hexagon/htp/im2col-ops.c index 35fc103df..52bbc37d1 100644 --- a/ggml/src/ggml-hexagon/htp/im2col-ops.c +++ b/ggml/src/ggml-hexagon/htp/im2col-ops.c @@ -3,11 +3,12 @@ #pragma clang diagnostic ignored "-Wunused-but-set-variable" #include -#include #include #include #include +#include "hex-common.h" + #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "htp-ctx.h" @@ -16,14 +17,19 @@ #include "hex-dma.h" #include "hex-profile.h" #include "htp-vtcm.h" +#include "htp-tensor.h" struct htp_im2col_context { struct htp_ops_context * octx; + uint32_t patch_base; // first patch index assigned to this dev + uint32_t npatches; // number of patches assigned to this dev uint32_t npatches_per_thread; // patches = N*OH*OW (pure-DDR kernel) - uint32_t pe_rows_per_thread; // N*OH rows per worker - uint32_t pe_src_row_bytes; // one output row's source: IC*KH*IW*4, rounded 256 - uint32_t pe_dst_row_bytes; // one output row's dst: OW*patch_stride*2, rounded 256 + uint32_t pe_row_base; // first N*OH row index assigned to this dev (DMA path) + uint32_t pe_nrows; // number of N*OH rows assigned to this dev (DMA path) + uint32_t pe_rows_per_thread; // N*OH rows per worker + uint32_t pe_src_row_bytes; // one output row's source: IC*KH*IW*4, rounded 256 + uint32_t pe_dst_row_bytes; // one output row's dst: OW*patch_stride*2, rounded 256 // Patch-embed DMA path VTCM ping-pong. uint8_t * pe_vtcm_src; // base of the 2x src buffers region @@ -58,33 +64,27 @@ static inline void htp_im2col_vtcm_layout_build(struct htp_im2col_vtcm_layout * struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; \ struct htp_ops_context * octx = ictx->octx; \ struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; \ + const struct htp_tensor * restrict src0 = octx->src[0]; \ const struct htp_tensor * restrict src1 = octx->src[1]; \ const struct htp_tensor * restrict dst = octx->dst; \ - const int32_t s0 = octx->op_params[0]; \ - const int32_t s1 = octx->op_params[1]; \ - const int32_t p0 = octx->op_params[2]; \ - const int32_t p1 = octx->op_params[3]; \ - const int32_t d0 = octx->op_params[4]; \ - const int32_t d1 = octx->op_params[5]; \ - const uint32_t N = src1->ne[3]; \ - const uint32_t IC = src1->ne[2]; \ - const uint32_t IH = src1->ne[1]; \ - const uint32_t IW = src1->ne[0]; \ - const uint32_t KH = octx->src[0]->ne[1]; \ - const uint32_t KW = octx->src[0]->ne[0]; \ + const int32_t s0 = octx->op_params[0], s1 = octx->op_params[1]; \ + const int32_t p0 = octx->op_params[2], p1 = octx->op_params[3]; \ + const int32_t d0 = octx->op_params[4], d1 = octx->op_params[5]; \ + const uint32_t N = src1->ne[3], IC = src1->ne[2], IH = src1->ne[1], IW = src1->ne[0]; \ + const uint32_t KH = src0->ne[1], KW = src0->ne[0]; \ const uint32_t OH = dst->ne[2]; \ const uint32_t OW = dst->ne[1]; \ const uint32_t patch_stride = IC * KH * KW; \ const float * restrict src_data = (const float *) src1->data; \ DST_CTYPE * restrict dst_data = (DST_CTYPE *) dst->data; \ - const uint32_t npatches = N * OH * OW; \ - const uint32_t patch_start = ictx->npatches_per_thread * ith; \ - const uint32_t patch_end = MIN(patch_start + ictx->npatches_per_thread, npatches); \ - if (patch_start >= patch_end) { \ + const uint32_t patch_end = ictx->patch_base + ictx->npatches; \ + const uint32_t patch_start = ictx->patch_base + ictx->npatches_per_thread * ith; \ + const uint32_t patch_stop = MIN(patch_start + ictx->npatches_per_thread, patch_end);\ + if (patch_start >= patch_stop) { \ return; \ } \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, patch_start); \ - for (uint32_t p = patch_start; p < patch_end; p++) { \ + for (uint32_t p = patch_start; p < patch_stop; p++) { \ const uint32_t iow = p % OW; \ const uint32_t ioh = (p / OW) % OH; \ const uint32_t in = p / (OW * OH); \ @@ -154,10 +154,10 @@ IM2COL_PATCHEMBED_BODY(im2col_patchembed_f32_thread, float, hvx_copy_f32_uu, hvx uint8_t * dst_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread; \ float * srcb = (float *) src_base; \ DST_CTYPE * dstb = (DST_CTYPE *) dst_base; \ - const uint32_t nrows = N * OH; \ + const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows; \ const uint32_t per_thread = ictx->pe_rows_per_thread; \ - const uint32_t row_start = per_thread * ith; \ - const uint32_t row_end = MIN(row_start + per_thread, nrows); \ + const uint32_t row_start = ictx->pe_row_base + per_thread * ith; \ + const uint32_t row_end = MIN(row_start + per_thread, row_end_max); \ if (row_start >= row_end) \ return; \ for (uint32_t r = row_start; r < row_end; r++) { \ @@ -266,26 +266,55 @@ int op_im2col(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - const uint32_t N = src1->ne[3]; - const uint32_t OH = dst->ne[2]; - const uint32_t OW = dst->ne[1]; - const uint32_t npatches = N * OH * OW; - const uint32_t n_threads = MIN(octx->n_threads, npatches); - - if ((octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) || n_threads == 0) { + if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { return HTP_STATUS_OK; } + const uint32_t N = src1->ne[3]; + const uint32_t OH = dst->ne[2]; + const uint32_t OW = dst->ne[1]; + const uint32_t total_patches = N * OH * OW; + const uint32_t total_rows = N * OH; + + uint32_t patch_base = 0; + uint32_t npatches = total_patches; + if (octx->ctx->mdev.count > 1) { + const uint32_t patch_size = dst->nb[1]; + const uint32_t patches_per_chunk = (patch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(patch_size, HEX_L2_LINE_SIZE)) : 1; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_patches, htp_tensor_mdev_data_aligned(dst) ? patches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + patch_base = range.start; + npatches = range.count; + } + + uint32_t row_base = 0; + uint32_t nrows = total_rows; + if (octx->ctx->mdev.count > 1) { + const uint32_t row_size = dst->nb[2]; + const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, htp_tensor_mdev_data_aligned(dst) ? rows_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_base = range.start; + nrows = range.count; + } + + if (npatches == 0 && nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = MIN(octx->n_threads, MAX(npatches, 1)); + struct htp_im2col_context ictx = { 0 }; - ictx.octx = octx; - ictx.npatches_per_thread = (npatches + n_threads - 1) / n_threads; + ictx.octx = octx; + ictx.patch_base = patch_base; + ictx.npatches = npatches; + ictx.npatches_per_thread = (npatches + n_threads - 1) / n_threads; // Clean non-overlapping patch-embed -> DMA kernel (if it fits VTCM); // everything else (padding/dilation/stride edges) -> pure-DDR kernel. - if (im2col_use_patchembed_dma(octx)) { - const uint32_t nrows = N * OH; - const uint32_t pth = MIN(octx->n_threads, nrows); + if (im2col_use_patchembed_dma(octx) && nrows > 0) { + const uint32_t pth = MIN(octx->n_threads, nrows); if (pth > 0 && im2col_patchembed_dma_fits(octx, &ictx, pth)) { + ictx.pe_row_base = row_base; + ictx.pe_nrows = nrows; ictx.pe_rows_per_thread = (nrows + pth - 1) / pth; if (dst->type == HTP_TYPE_F16) { work_queue_run(octx->ctx->work_queue, im2col_patchembed_dma_thread, &ictx, pth); @@ -297,6 +326,10 @@ int op_im2col(struct htp_ops_context * octx) { // else: doesn't fit -> fall through to the pure-DDR kernel below. } + if (npatches == 0) { + return HTP_STATUS_OK; + } + if (dst->type == HTP_TYPE_F16) { work_queue_run(octx->ctx->work_queue, im2col_patchembed_thread, &ictx, n_threads); } else { diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index be54d4fe9..1d291e16b 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -34,6 +34,7 @@ #include "work-queue.h" #include "hex-profile.h" #include "allreduce-ops.h" +#include "htp-fence.h" #define HMX_QUEUE_CAPACITY 16 #define HMX_QUEUE_STACK_SIZE 16384 @@ -710,22 +711,43 @@ static inline void profile_stop(uint32_t mode, struct profile_data * d) { static int op_fence(struct htp_ops_context * octx) { struct htp_context *ctx = octx->ctx; struct htp_thread_trace * tr = &ctx->trace[0]; - const uint32_t seq = (uint32_t) octx->op_params[0]; + const uint32_t seq = (uint32_t) octx->op_params[0]; + const uint32_t mode = (uint32_t) octx->op_params[1]; htp_trace_event_start(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); const struct htp_tensor * sync = octx->src[0]; - atomic_uint * sync_fence = (atomic_uint *) sync->data; + atomic_uint * sync_fence = (atomic_uint *) (uintptr_t) sync->data; + + if (mode == 1) { + htp_flush_dirty_ranges(ctx); + + htp_mdev_group_barrier(octx); + + if (ctx->mdev.idx == 0) { + htp_fence_write(sync_fence, seq, octx->status); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); + FARF(HIGH, "ggml-hex: sync-signal : fence %p seq 0x%x status %d\n", sync_fence, seq, octx->status); + return octx->status; + } + + int status = HTP_STATUS_OK; uint64_t spins = 0; while (1) { - Q6_dccleaninva_A((void *) sync_fence); - asm volatile ("syncht" : : : "memory"); - uint32_t val = atomic_load(&sync_fence[0]); - if ((int32_t)(val - seq) >= 0) { + uint32_t sync_seq; + uint32_t sync_status; + htp_fence_read(sync_fence, &sync_seq, &sync_status); + if ((int32_t)(sync_seq - seq) >= 0) { + if (sync_status > HTP_STATUS_OK) { + FARF(ERROR, "ggml-hex: sync-wait peer failed with status %u : fence %p seq 0x%x\n", sync_status, sync_fence, seq); + status = sync_status; + } break; } if (++spins > HTP_FENCE_TIMEOUT) { - FARF(ERROR, "ggml-hex: sync-wait TIMEOUT : fence %p spins %llu seq %u\n", sync_fence, spins, seq); + FARF(ERROR, "ggml-hex: sync-wait TIMEOUT : fence %p spins %llu seq 0x%x\n", sync_fence, spins, seq); + status = HTP_STATUS_INTERNAL_ERR; break; } hex_pause(); @@ -733,12 +755,27 @@ static int op_fence(struct htp_ops_context * octx) { htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); - FARF(HIGH, "ggml-hex: sync-done : fence %p spins %llu seq %u\n", sync_fence, spins, seq); + FARF(HIGH, "ggml-hex: sync-done : fence %p spins %llu seq 0x%x\n", sync_fence, spins, seq); + return status; +} + +static int op_mdev_group(struct htp_ops_context * octx) { + struct htp_context * ctx = octx->ctx; + const struct htp_tensor * sync = octx->src[0]; + ctx->mdev.idx = (uint16_t) octx->op_params[0]; + ctx->mdev.count = (uint16_t) sync->ne[1]; + if (ctx->mdev.count > 1) { + ctx->mdev.count_div = init_fastdiv_values(ctx->mdev.count); + ctx->mdev.fence_base = (uint8_t *) sync->data; + } return HTP_STATUS_OK; } static int execute_op(struct htp_ops_context * octx) { switch (octx->op) { + case HTP_OP_MDEV_GROUP: + return op_mdev_group(octx); + case HTP_OP_FENCE: return op_fence(octx); @@ -812,6 +849,7 @@ static int execute_op(struct htp_ops_context * octx) { return op_sum_rows(octx); case HTP_OP_CPY: + case HTP_OP_CPY_FENCE: return op_cpy(octx); case HTP_OP_REPEAT: @@ -855,7 +893,7 @@ static int execute_op(struct htp_ops_context * octx) { } FARF(ERROR, "Unknown Op %u", octx->op); - return -1; + return HTP_STATUS_NO_SUPPORT; } static inline bool reuse_buf(struct htp_context *ctx, uint32_t *m_reuse, struct htp_buf_desc *b) { @@ -984,11 +1022,19 @@ static void prep_tensors(struct htp_context *ctx, struct htp_buf_desc *bufs, str } } -static int proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, uint32_t idx, struct htp_op_desc * op) { - memcpy(octx->op_params, op->params, sizeof(octx->op_params)); +static void mdev_group_init(struct htp_context * ctx, const struct htp_opbatch_req * req) { + memset(&ctx->mdev, 0, sizeof(ctx->mdev)); + ctx->mdev.fence_seq = (uint32_t)((req->seq & 0xfffff) << 12); +} + +static int proc_op_req(struct htp_ops_context * octx, struct htp_buf_desc * bufs, uint32_t n_bufs, + struct htp_tensor * tens, uint32_t idx, struct htp_op_desc * op) { + memcpy(octx->op_params, op->params, sizeof(octx->op_params)); memcpy(octx->kernel_params, op->kernel_params, sizeof(octx->kernel_params)); - octx->flags = op->flags; - octx->op = op->opcode; + octx->flags = op->flags; + octx->op = op->opcode; + octx->n_threads = octx->ctx->n_threads; + octx->n_threads_div = octx->ctx->n_threads_div; FARF(HIGH, "proc-op #%u: opcode %u flags 0x%x", idx, octx->op, octx->flags); @@ -1027,9 +1073,13 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, u dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); } + htp_tensor_dirty_all(octx->ctx, octx->dsts, HTP_OP_MAX_OUTPUTS); + + htp_mdev_group_barrier(octx); + int status = execute_op(octx); - htp_tensor_dirty_all(octx->ctx, octx->dsts, HTP_OP_MAX_OUTPUTS); + htp_ops_context_set_status(octx, status); octx->src0_spad.src = NULL; octx->src1_spad.src = NULL; @@ -1037,7 +1087,7 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, u octx->src3_spad.src = NULL; octx->dst_spad.src = NULL; - return status; + return octx->status; } static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_req * req, const struct dspqueue_buffer * dbuf) { @@ -1059,7 +1109,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r return; } - FARF(HIGH, "processing opbatch #%u: n-bufs %u n-tensors %u n-ops %u n-traces %u : m-size %u b-size %u t-size %u o-size %u", req->id, + FARF(HIGH, "processing opbatch #%llu: n-bufs %u n-tensors %u n-ops %u n-traces %u : m-size %u b-size %u t-size %u o-size %u", (unsigned long long) req->seq, n_bufs, n_tens, n_ops, req->n_traces, dbuf->size, b_size, t_size, o_size); // Setup descriptor pointers @@ -1096,8 +1146,11 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r struct htp_ops_context *octx = &ctx->octx; memset(octx, 0, sizeof(*octx)); - octx->n_threads = ctx->n_threads; - octx->ctx = ctx; + octx->n_threads = ctx->n_threads; + octx->n_threads_div = ctx->n_threads_div; + octx->ctx = ctx; + + mdev_group_init(ctx, req); work_queue_wakeup(ctx->work_queue); if (ctx->hmx_queue) { @@ -1105,15 +1158,18 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r } int op_status = HTP_STATUS_OK; - for (uint32_t i = 0; i < n_ops && op_status == HTP_STATUS_OK; i++) { + octx->status = HTP_STATUS_OK; + for (uint32_t i = 0; i < n_ops; i++) { struct profile_data prof; profile_start(ctx->profiler, &prof); - op_status = proc_op_req(octx, tens, i, &ops[i]); + op_status = proc_op_req(octx, bufs, n_bufs, tens, i, &ops[i]); profile_stop(ctx->profiler, &prof); + htp_ops_context_set_status(octx, op_status); + if (ctx->profiler) { pds[i].opcode = ops[i].opcode; pds[i].usecs = prof.usecs; @@ -1136,19 +1192,20 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE); htp_trace_event_stop(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0); + htp_mdev_group_barrier(octx); + profile_stop(HTP_PROF_BASIC, &batch_prof); struct htp_opbatch_rsp rsp; memset(&rsp, 0, sizeof(rsp)); - rsp.id = req->id; - rsp.status = op_status; + rsp.seq = req->seq; + rsp.status = octx->status; rsp.n_bufs = n_bufs; rsp.n_tensors = n_tens; rsp.n_ops = n_ops; rsp.usecs = batch_prof.usecs; rsp.cycles_start = batch_prof.cycles_start; rsp.cycles_stop = batch_prof.cycles_stop; - rsp.seq = req->seq; if (ctx->profiler == HTP_PROF_TRACE) { for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c index 2a87dd19e..1b597dcd9 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c @@ -21,6 +21,7 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" #include "matmul-ops.h" #include "htp-vtcm.h" @@ -89,6 +90,8 @@ struct htp_mm_context { // Precomputed values uint32_t src0_nrows_per_thread; + uint32_t src0_row_start; + uint32_t src0_row_end; uint32_t src0_row_size_padded; uint32_t src1_nrows; @@ -135,6 +138,23 @@ struct htp_mm_context { uint32_t vtcm_dst_size_per_thread; }; +static int htp_mm_init_context( + struct htp_ops_context * octx, + const struct htp_mm_kernel_params * kparams +) { + if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + if (kparams->n_hmx) { + if (kparams->n_act_threads <= 0 || kparams->n_act_threads > (int32_t) octx->n_threads) { + return HTP_STATUS_INVAL_PARAMS; + } + } + + return HTP_STATUS_OK; +} + // vdelta control to expand first 32 e8m0 values into 32 uint32 elements static const uint8_t __attribute__((aligned(128))) expand_x32_e8m0[128] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x02, 0x00, 0x08, 0x08, 0x01, 0x02, 0x00, 0x04, 0x04, 0x00, 0x00, @@ -238,22 +258,24 @@ static void hvx_mm_4d(unsigned int nth, unsigned int ith, void * data) { // This is the size of the rest of the dimensions of the result const uint32_t nr1 = ne1 * ne2 * ne3; + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; + // distribute the thread work across the inner or outer loop based on which one is larger uint32_t dr0, dr1, ith0, ith1; if (nr0 > nr1) { - dr0 = fastdiv(nr0 + nth - 1, &octx->ctx->n_threads_div); + dr0 = fastdiv(src0_nrows + nth - 1, &octx->n_threads_div); dr1 = nr1; ith0 = ith; ith1 = 0; } else { - dr0 = nr0; - dr1 = fastdiv(nr1 + nth - 1, &octx->ctx->n_threads_div); + dr0 = src0_nrows; + dr1 = fastdiv(nr1 + nth - 1, &octx->n_threads_div); ith0 = 0; ith1 = ith; } - const uint32_t ir0_start = dr0 * ith0; - const uint32_t ir0_end = MIN(ir0_start + dr0, nr0); + const uint32_t ir0_start = mmctx->src0_row_start + dr0 * ith0; + const uint32_t ir0_end = MIN(ir0_start + dr0, mmctx->src0_row_end); const uint32_t ir1_start = dr1 * ith1; const uint32_t ir1_end = MIN(ir1_start + dr1, nr1); @@ -312,11 +334,11 @@ static void hvx_mm_4d(unsigned int nth, unsigned int ith, void * data) { static void hvx_mm_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ htp_matmul_preamble; \ \ - const uint32_t src0_nrows = ne01 * ne02 * ne03; \ + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; \ const uint32_t src1_nrows = ne11 * ne12 * ne13; \ \ - const uint32_t src0_start_row = src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \ + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; \ + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); \ \ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ \ @@ -414,10 +436,10 @@ static void hvx_mm_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void static void hvx_mv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ htp_matmul_preamble; \ \ - const uint32_t src0_nrows = ne01; \ + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; \ \ - const uint32_t src0_start_row = src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \ + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; \ + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); \ \ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ \ @@ -549,12 +571,22 @@ static void hvx_mm_nx_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, v uint32_t n_k_tiles_w = ne00 / 32; \ uint32_t tile_row_stride = n_k_tiles_w * tile_size; \ \ - const uint32_t src0_nrows = ne01 * src_w->ne[2] * src_w->ne[3]; \ - uint32_t src0_nrows_per_thread = fastdiv(src0_nrows + nth - 1, &octx->ctx->n_threads_div); \ + uint32_t src0_start_row = 0; \ + uint32_t src0_end_row = ne01; \ + if (octx->ctx->mdev.count > 1) { \ + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); \ + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, \ + octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); \ + src0_start_row = range.start; \ + src0_end_row = range.start + range.count; \ + } \ + \ + const uint32_t nrows = src0_end_row - src0_start_row; \ + uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); \ src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); \ \ - const uint32_t start_row = src0_nrows_per_thread * ith; \ - const uint32_t end_row = MIN(start_row + src0_nrows_per_thread, src0_nrows); \ + const uint32_t start_row = src0_start_row + src0_nrows_per_thread * ith; \ + const uint32_t end_row = MIN(start_row + src0_nrows_per_thread, src0_end_row); \ if (start_row >= end_row) continue; \ \ uint32_t ct_start = start_row / 32; \ @@ -735,11 +767,11 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); const uint32_t prefetch_mask = n_prefetch - 1; - const uint32_t src0_nrows = ne01 * ne02 * ne03; // src0 rows - const uint32_t src1_nrows = ne11 * ne12 * ne13; // src1 rows + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows + const uint32_t src1_nrows = ne11 * ne12 * ne13; // src1 rows - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); struct htp_thread_trace * tr = &octx->ctx->trace[ith]; @@ -781,7 +813,7 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); - // Process src1 columns in pairs (2×2 tiling) + // Process src1 columns in pairs (2x2 tiling) uint32_t ir1 = 0; for (; ir1 + 1 < src1_nrows; ir1 += 2) { const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride); @@ -791,7 +823,7 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { mmctx->vec_dot_2x2(ne00, &dst_row0[ir0], &dst_row1[ir0], ss0, ss0 + src0_stride, src1_col0, src1_col1); } - // Handle remaining src1 rows (fallback to 2×1) + // Handle remaining src1 rows (fallback to 2x1) for (; ir1 < src1_nrows; ++ir1) { const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size)); @@ -833,10 +865,10 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; - const uint32_t src0_nrows = ne01; + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); struct htp_thread_trace * tr = &octx->ctx->trace[ith]; @@ -943,13 +975,10 @@ static void hvx_mm_id(unsigned int nth, unsigned int ith, void * data) { const struct htp_tensor * restrict ids = octx->src[2]; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - - const uint32_t src0_nrows = ne01; // src0 rows per expert + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows per expert const uint32_t src1_nrows = ne11; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); hvx_mm_run_quant_task(mmctx, ith); @@ -1036,9 +1065,9 @@ static void hvx_mv_id(unsigned int nth, unsigned int ith, void * data) { const struct htp_tensor * restrict ids = octx->src[2]; - const uint32_t src0_nrows = ne01; // src0 rows per expert - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows per expert + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); hvx_mm_run_quant_task(mmctx, ith); @@ -1143,12 +1172,22 @@ static void hvx_mv_id_nx(unsigned int nth, unsigned int ith, void * data) { const struct htp_tensor * restrict dst = octx->dsts[p]; if (!src_w || !dst) continue; - const uint32_t src0_nrows = src_w->ne[1]; - uint32_t src0_nrows_per_thread = fastdiv(src0_nrows + nth - 1, &octx->ctx->n_threads_div); + const uint32_t ne01 = src_w->ne[1]; + uint32_t start_row = 0; + uint32_t end_row = ne01; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + start_row = range.start; + end_row = range.start + range.count; + } + + const uint32_t nrows = end_row - start_row; + uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row); if (src0_start_row >= src0_end_row) continue; const uint8_t * restrict src0_row = (const uint8_t *) src_w->data + eid * src_w->nb[2]; @@ -1227,12 +1266,22 @@ static void hvx_mm_id_nx(unsigned int nth, unsigned int ith, void * data) { const struct htp_tensor * restrict dst = octx->dsts[p]; if (!src_w || !dst) continue; - const uint32_t src0_nrows = src_w->ne[1]; - uint32_t src0_nrows_per_thread = fastdiv(src0_nrows + nth - 1, &octx->ctx->n_threads_div); + const uint32_t ne01 = src_w->ne[1]; + uint32_t start_row = 0; + uint32_t end_row = ne01; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + start_row = range.start; + end_row = range.start + range.count; + } + + const uint32_t nrows = end_row - start_row; + uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row); if (src0_start_row >= src0_end_row) continue; const uint8_t * src0_row = (const uint8_t *) src_w->data + cur_a * src_w->nb[2]; @@ -1323,15 +1372,33 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; - const uint32_t src0_nrows = ne01 * ne02 * ne03; + const uint32_t src0_nrows = ne01; const uint32_t src1_nrows = ne11 * ne12 * ne13; + uint32_t src0_row_start = 0; + uint32_t src0_row_end = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + src0_row_start = range.start; + src0_row_end = range.start + range.count; + } + + if (src0_row_start >= src0_row_end) { + return HTP_STATUS_OK; + } + + const uint32_t nrows = src0_row_end - src0_row_start; + mmctx->src0_row_start = src0_row_start; + mmctx->src0_row_end = src0_row_end; + bool is_repacked = (src0->type == HTP_TYPE_Q4_0 || src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q8_0 || src0->type == HTP_TYPE_IQ4_NL || src0->type == HTP_TYPE_MXFP4); // Compute src0_nrows_per_thread - mmctx->src0_nrows_per_thread = fastdiv(src0_nrows + octx->n_threads - 1, &octx->ctx->n_threads_div); + mmctx->src0_nrows_per_thread = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div); if (is_repacked) { mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); } else { @@ -1503,13 +1570,13 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) { mmctx->vtcm_src1_size_per_thread = L.src1_bytes; } else { - mmctx->vtcm_src1_size_per_thread = fastdiv(L.src1_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_src1_size_per_thread = fastdiv(L.src1_bytes, &octx->n_threads_div); } - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->ctx->n_threads_div); - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); + mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); - size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes; + const size_t vtcm_size = L.total_bytes; FARF(HIGH, "matmul-%s : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu (%zu)\n", mmctx->type, L.src0_bytes, L.src1_bytes, L.dst_bytes, vtcm_size); @@ -1583,13 +1650,21 @@ static void hvx_mm_nx_2d(unsigned int nth, unsigned int ith, void * data) { const uint32_t ne00 = src_w->ne[0]; const uint32_t ne01 = src_w->ne[1]; - const uint32_t src0_nrows = ne01 * src_w->ne[2] * src_w->ne[3]; + uint32_t start_row = 0; + uint32_t end_row = ne01; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + start_row = range.start; + end_row = range.start + range.count; + } - uint32_t src0_nrows_per_thread = fastdiv(src0_nrows + nth - 1, &octx->ctx->n_threads_div); + const uint32_t nrows = end_row - start_row; + uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); src0_nrows_per_thread += (src0_nrows_per_thread & 1); - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row); const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); if (src0_start_row >= src0_end_row) continue; @@ -2638,10 +2713,6 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k const struct htp_tensor * restrict src0 = octx->src[0]; const struct htp_tensor * restrict act = octx->src[n_weights]; - if (!src0 || !act) { - return HTP_STATUS_INVAL_PARAMS; - } - const int weight_type = (int) src0->type; const int k = (int) act->ne[0]; const int k_valid = (int) act->ne[0]; @@ -2714,16 +2785,31 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // scale: 1.0, bias: 0.0 in FP16 - FARF(HIGH, "hmx-mm-nx-2d: n_weights %u m %d k %d wtype %d mc %d nc %d vtcm %zu/%zu", - n_weights, m, k, weight_type, m_chunk_n_rows, n_chunk_n_cols, L.total_bytes, vtcm_budget); + int m_start = 0; + int m_rows = m; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(octx->dsts[0], sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + m_start = (int) range.start; + m_rows = (int) range.count; + } + + if (m_rows == 0) { + return HTP_STATUS_OK; + } + + FARF(HIGH, "hmx-mm-nx-2d: n_weights %u m %d (%d..%d) k %d wtype %d mc %d nc %d vtcm %zu/%zu", + n_weights, m, m_start, m_start + m_rows, k, weight_type, m_chunk_n_rows, n_chunk_n_cols, L.total_bytes, vtcm_budget); htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + const size_t mr_end = (size_t)(m_start + m_rows); + if (pipeline) { hmx_matmul_job_t job_slots[2]; - for (size_t mr = 0; mr < (size_t) m; mr += m_chunk_n_rows) { - const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); + for (size_t mr = (size_t) m_start; mr < mr_end; mr += m_chunk_n_rows) { + const size_t n_rows = hex_smin(mr_end - mr, m_chunk_n_rows); void *vtcm_weight_bufs[2] = { vtcm_scratch0, vtcm_scratch1 }; void *vtcm_output_bufs[2] = { vtcm_output, vtcm_scratch2 }; @@ -2822,8 +2908,8 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k } } else { hmx_matmul_job_t job; - for (size_t mr = 0; mr < (size_t) m; mr += m_chunk_n_rows) { - const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); + for (size_t mr = (size_t) m_start; mr < mr_end; mr += m_chunk_n_rows) { + const size_t n_rows = hex_smin(mr_end - mr, m_chunk_n_rows); struct activation_transfer_params act_params = { .ctx = ctx, @@ -3095,7 +3181,7 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ int chunk_dst_cols = params->n - (int)nc; if (chunk_dst_cols > 0) { transfer_output_chunk_threaded(ctx, output, src2_chunk, vtcm_output, (int) n_rows, (int) n_cols, - params->dst_stride, params->src2_stride, chunk_dst_cols, ctx->n_threads); + params->dst_stride, params->src2_stride, chunk_dst_cols, n_threads); } } } @@ -3216,7 +3302,10 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, int weight_type, const struct mmid_row_mapping *matrix_rows, int cur_a, - int mapping_stride) { + int mapping_stride, + int m_start, + int m_end, + int n_threads) { struct htp_thread_trace * tr = &ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); @@ -3247,7 +3336,6 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, const int n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS; const struct fastdiv_values n_k_tiles_div = init_fastdiv_values(n_k_tiles); - const int n_threads = ctx->n_threads; const bool is_quant = (weight_type != HTP_TYPE_F16 && weight_type != HTP_TYPE_F32); const size_t vec_dot_size = k * sizeof(__fp16); @@ -3303,8 +3391,8 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, hmx_matmul_job_t job; - for (size_t mr = 0; mr < (size_t) m_padded; mr += m_chunk_n_rows) { - const size_t n_rows = hex_smin(m_padded - mr, m_chunk_n_rows); + for (size_t mr = (size_t) m_start; mr < (size_t) m_end; mr += m_chunk_n_rows) { + const size_t n_rows = hex_smin((size_t) m_end - mr, m_chunk_n_rows); const size_t n_row_tiles = hmx_ceil_div(n_rows, HTP_MM_HMX_TILE_N_ROWS); transfer_activation_chunk_gathered_threaded( @@ -3368,31 +3456,48 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k const int act_stride = (int)(src1->nb[1] / sizeof(float)); const int wgt_stride = (int)(src0->nb[1] / sizeof(__fp16)); + int m_start = 0; + int m_rows = m_total; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m_total, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + m_start = (int) range.start; + m_rows = (int) range.count; + } + + if (m_rows == 0) { + return HTP_STATUS_OK; + } + const float * src2_ptr = NULL; uint32_t src2_stride = 0; size_t src2_nb2 = 0; size_t src2_nb3 = 0; if (src2) { - src2_ptr = (const float *) src2->data; src2_stride = (src2->ne[1] == 1) ? 0 : (uint32_t) (src2->nb[1] / sizeof(float)); + src2_ptr = (const float *) src2->data + m_start * src2_stride; src2_nb2 = (src2->ne[2] == 1) ? 0 : src2->nb[2]; src2_nb3 = (src2->ne[3] == 1) ? 0 : src2->nb[3]; } + const int dst_stride = (int)(dst->nb[1] / sizeof(float)); + float * dst_ptr = (float *) dst->data + m_start * dst_stride; + const float * act_ptr = (const float *) src1->data + m_start * act_stride; + int ret = -1; - const int n_threads = MIN(kparams->n_threads, (int) octx->n_threads); + const int n_threads = kparams->n_threads; if (kparams->kernel_type == HTP_MM_KERNEL_HMX_F16_BATCHED) { hmx_mm_f16_f32_batched_params_t batch_params = { - .dst = (float *) dst->data, + .dst = dst_ptr, .src2 = src2_ptr, - .activation = (float *) src1->data, + .activation = act_ptr, .weight = (const __fp16 *) src0->data, - .m = m_total, + .m = m_rows, .k = k, .n = n, .act_stride = act_stride, .weight_stride = wgt_stride, - .dst_stride = (int) (dst->nb[1] / sizeof(float)), + .dst_stride = dst_stride, .src2_stride = src2_stride, .ne02 = ne02, .ne03 = ne03, @@ -3420,9 +3525,9 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k kparams->vtcm_size); } else { ret = hmx_mm_2d_f32( - octx->ctx, (float*) dst->data, src2_ptr, (float*) src1->data, (const uint8_t *) src0->data, - m_total, k, n, act_stride, (int) src0->nb[1], (int) src0->type, (int) src1->ne[0], - (int)(dst->nb[1] / sizeof(float)), src2_stride, (int)dst->ne[0], + octx->ctx, dst_ptr, src2_ptr, act_ptr, (const uint8_t *) src0->data, + m_rows, k, n, act_stride, (int) src0->nb[1], (int) src0->type, (int) src1->ne[0], + dst_stride, src2_stride, (int)dst->ne[0], kparams->m_chunk, kparams->n_chunk, kparams->pipeline, n_threads, kparams->n_act_threads, &kparams->div_n_act_threads, @@ -3441,6 +3546,11 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k int op_matmul(struct htp_ops_context * octx) { const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const int status = htp_mm_init_context(octx, kparams); + if (status != HTP_STATUS_OK) { + return status; + } + if (kparams->n_hmx) { return hmx_mm_op_matmul(octx, kparams); } @@ -3463,6 +3573,16 @@ static int hmx_mm_op_matmul_id( const int32_t cne1 = matrix_row_counts[cur_a]; if (cne1 == 0) continue; + const int m_padded = hex_align_up(cne1, 32); + int m_start = 0, m_end = m_padded; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && (uint32_t) cne1 >= octx->ctx->mdev.count; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m_padded, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + m_start = (int) range.start; + m_end = (int) (range.start + range.count); + } + if (m_start >= m_end) continue; + int ret = hmx_mm_id_2d_f32(octx->ctx, (float*) dst->data, (float*) src1->data, (const uint8_t *) src0->data + cur_a * nb02, cne1, ne00, ne01, @@ -3471,7 +3591,8 @@ static int hmx_mm_op_matmul_id( nb11, nb12, nb1, nb2, (int) src0->nb[1], (int) src0->type, - matrix_rows, cur_a, mmctx->mapping_stride); + matrix_rows, cur_a, mmctx->mapping_stride, + m_start, m_end, (int) octx->n_threads); if (ret != 0) { FARF(ERROR, "HMX matmul failed for expert %u, error %d\n", cur_a, ret); return HTP_STATUS_NO_SUPPORT; @@ -3524,7 +3645,7 @@ static int hvx_mm_matmul_id( htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, src1_nrows, octx->n_threads, 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, true, false); - size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes; + const size_t vtcm_size = L.total_bytes; FARF(HIGH, "matmul-id-%s : src0-spad-size %zu src1-spad-size %zu src2-spad-size 0 dst-spad-size %zu (%zu)\n", mmctx->type, L.src0_bytes, L.src1_bytes, L.dst_bytes, vtcm_size); @@ -3554,10 +3675,10 @@ static int hvx_mm_matmul_id( mmctx->vtcm_src0_stride = src0_row_size_padded; mmctx->vtcm_src1_stride = src1_row_size; - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); mmctx->vtcm_src1_size_per_thread = L.src1_bytes; mmctx->vtcm_src2_size_per_thread = 0; - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->quant_task_func = quant_task_func; @@ -3587,6 +3708,20 @@ static int hmx_mm_op_matmul_id_nx( const int32_t cne1 = matrix_row_counts[cur_a]; if (cne1 == 0) continue; + const int m_padded = hex_align_up(cne1, 32); + int m_start = 0, m_end = m_padded; + if (octx->ctx->mdev.count > 1) { + bool can_split = (uint32_t) cne1 >= octx->ctx->mdev.count; + for (uint32_t p = 0; p < n_weights && can_split; ++p) { + const struct htp_tensor * restrict dst = octx->dsts[p]; + can_split = !dst || htp_tensor_mdev_data_aligned(dst); + } + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m_padded, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + m_start = (int) range.start; + m_end = (int) (range.start + range.count); + } + if (m_start >= m_end) continue; + for (uint32_t p = 0; p < n_weights; ++p) { const struct htp_tensor * restrict src_w = octx->src[p]; const struct htp_tensor * restrict dst = octx->dsts[p]; @@ -3600,7 +3735,8 @@ static int hmx_mm_op_matmul_id_nx( act->nb[1], act->nb[2], dst->nb[1], dst->nb[2], (int) src_w->nb[1], (int) src_w->type, - matrix_rows, cur_a, mmctx->mapping_stride); + matrix_rows, cur_a, mmctx->mapping_stride, + m_start, m_end, (int) octx->n_threads); if (ret != 0) { FARF(ERROR, "HMX matmul ID NX failed for expert %u weight %u, error %d\n", cur_a, p, ret); return HTP_STATUS_NO_SUPPORT; @@ -3656,7 +3792,7 @@ static int hvx_mm_matmul_id_nx( htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], src1_nrows, octx->n_threads, 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, true, false); - size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes; + const size_t vtcm_size = L.total_bytes; if (octx->ctx->vtcm_size < vtcm_size) { FARF(ERROR, "matmul-id-nx: current VTCM reservation %zu is too small, needed %zu\n", @@ -3678,9 +3814,9 @@ static int hvx_mm_matmul_id_nx( mmctx->vtcm_src0_stride = 0; mmctx->vtcm_src1_stride = src1_row_size; - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); mmctx->vtcm_src1_size_per_thread = L.src1_bytes; - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->quant_task_func = quant_task_func; @@ -3769,16 +3905,21 @@ static inline void scan_expert_ids( int op_matmul_id(struct htp_ops_context * octx) { htp_matmul_tensors_preamble; + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + struct htp_mm_context mmctx_struct = {0}; + struct htp_mm_context * mmctx = &mmctx_struct; + + const int status = htp_mm_init_context(octx, kparams); + if (status != HTP_STATUS_OK) { + return status; + } + struct htp_thread_trace * tr = &octx->ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); - struct htp_mm_context mmctx_struct = {0}; - struct htp_mm_context * mmctx = &mmctx_struct; mmctx->octx = octx; mmctx->act = src1; - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; - const struct htp_tensor * restrict ids = octx->src[2]; const size_t src0_row_size = nb01; @@ -3789,9 +3930,6 @@ int op_matmul_id(struct htp_ops_context * octx) { const uint32_t src0_nrows = ne01; // per expert const uint32_t src1_nrows = ne11 * ne12 * ne13; - mmctx->src0_nrows_per_thread = fastdiv(src0_nrows + octx->n_threads - 1, &octx->ctx->n_threads_div); - mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); - // row groups const int n_ids = ids->ne[0]; // n_expert_used const int n_as = ne02; // n_expert @@ -3843,6 +3981,29 @@ int op_matmul_id(struct htp_ops_context * octx) { if (kparams->n_hmx) { s = hmx_mm_op_matmul_id(octx, mmctx); } else { + uint32_t src0_row_start = 0; + uint32_t src0_row_end = src0_nrows; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + src0_row_start = range.start; + src0_row_end = range.start + range.count; + } + + if (src0_row_start >= src0_row_end) { + if (mapping_buf != octx->ctx->ddr_spad_base) { + free(mapping_buf); + } + return HTP_STATUS_OK; + } + + const uint32_t nrows = src0_row_end - src0_row_start; + mmctx->src0_row_start = src0_row_start; + mmctx->src0_row_end = src0_row_end; + + mmctx->src0_nrows_per_thread = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div); + mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); + if (hvx_mm_init_vec_dot(mmctx, src0->type) == 0) { s = hvx_mm_matmul_id(octx, mmctx, src1_nrows > 1 ? hvx_mm_id : hvx_mv_id); } else { @@ -3858,29 +4019,31 @@ int op_matmul_id(struct htp_ops_context * octx) { } int op_matmul_id_nx(struct htp_ops_context * octx) { + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + struct htp_mm_context mmctx_struct = {0}; + struct htp_mm_context * mmctx = &mmctx_struct; + + const int status = htp_mm_init_context(octx, kparams); + if (status != HTP_STATUS_OK) { + return status; + } + struct htp_thread_trace * tr = &octx->ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + mmctx->octx = octx; const uint32_t n_weights = kparams->n_weights; const struct htp_tensor * restrict src0 = octx->src[0]; const struct htp_tensor * restrict act = octx->src[n_weights]; const struct htp_tensor * restrict ids = octx->src[n_weights + 1]; - struct htp_mm_context mmctx_struct = {0}; - struct htp_mm_context * mmctx = &mmctx_struct; - mmctx->octx = octx; mmctx->act = act; const size_t src0_row_size = src0->nb[1]; const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); - const uint32_t src0_nrows = src0->ne[1]; const uint32_t src1_nrows = act->ne[1] * act->ne[2] * act->ne[3]; - mmctx->src0_nrows_per_thread = fastdiv(src0_nrows + octx->n_threads - 1, &octx->ctx->n_threads_div); - mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); - const int n_ids = ids->ne[0]; const int n_as = src0->ne[2]; @@ -3946,6 +4109,12 @@ int op_matmul_id_nx(struct htp_ops_context * octx) { } int op_matmul_nx(struct htp_ops_context * octx) { const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + + const int status = htp_mm_init_context(octx, kparams); + if (status != HTP_STATUS_OK) { + return status; + } + if (kparams->n_hmx) { return hmx_mm_nx_2d_f32(octx, kparams); } @@ -4012,7 +4181,7 @@ int op_matmul_nx(struct htp_ops_context * octx) { htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], src1_nrows, octx->n_threads, 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, false, true); - size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes; + const size_t vtcm_size = L.total_bytes; if (octx->ctx->vtcm_size < vtcm_size) { FARF(ERROR, "matmul-nx: current VTCM reservation %zu is too small, needed %zu\n", @@ -4034,9 +4203,9 @@ int op_matmul_nx(struct htp_ops_context * octx) { mmctx->vtcm_src0_stride = is_repacked ? 0 : src0_row_size_padded; mmctx->vtcm_src1_stride = src1_row_size; - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); mmctx->vtcm_src1_size_per_thread = L.src1_bytes; - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->quant_task_func = quant_task_func; diff --git a/ggml/src/ggml-hexagon/htp/pad-ops.c b/ggml/src/ggml-hexagon/htp/pad-ops.c index aaa72b315..0222f24dc 100644 --- a/ggml/src/ggml-hexagon/htp/pad-ops.c +++ b/ggml/src/ggml-hexagon/htp/pad-ops.c @@ -12,8 +12,11 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" /* Circular wrap: maps any integer x into [0, n) */ static inline uint32_t wrap_around(int32_t x, uint32_t n) { @@ -68,6 +71,7 @@ struct htp_pad_context { uint32_t nrows_per_thread; uint32_t total_dst_rows; + uint32_t row_start; size_t type_size; @@ -78,39 +82,39 @@ struct htp_pad_context { size_t dst_row_size_aligned; }; -#define htp_pad_preamble \ - const struct htp_tensor * src = octx->src[0]; \ - const struct htp_tensor * dst = octx->dst; \ - \ - const uint32_t ne00 = src->ne[0]; \ - const uint32_t nb00 = src->nb[0]; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; \ - \ - const int32_t lp0 = pctx->lp0, rp0 = pctx->rp0; \ - const int32_t lp1 = pctx->lp1, rp1 = pctx->rp1; \ - const int32_t lp2 = pctx->lp2, rp2 = pctx->rp2; \ - const int32_t lp3 = pctx->lp3, rp3 = pctx->rp3; \ - \ - const size_t type_size = pctx->type_size; \ - \ - const uint32_t row_start = pctx->nrows_per_thread * ith; \ - const uint32_t row_end = MIN(row_start + pctx->nrows_per_thread, pctx->total_dst_rows); +#define htp_pad_preamble \ + const struct htp_tensor * src = octx->src[0]; \ + const struct htp_tensor * dst = octx->dst; \ + \ + const uint32_t ne00 = src->ne[0]; \ + const uint32_t nb00 = src->nb[0]; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ + const uint32_t nb3 = dst->nb[3]; \ + \ + const int32_t lp0 = pctx->lp0, rp0 = pctx->rp0; \ + const int32_t lp1 = pctx->lp1, rp1 = pctx->rp1; \ + const int32_t lp2 = pctx->lp2, rp2 = pctx->rp2; \ + const int32_t lp3 = pctx->lp3, rp3 = pctx->rp3; \ + \ + const size_t type_size = pctx->type_size; \ + \ + const uint32_t row_start = pctx->row_start + pctx->nrows_per_thread * ith; \ + const uint32_t row_end = MIN(row_start + pctx->nrows_per_thread, pctx->row_start + pctx->total_dst_rows); -#define htp_pad_dma_preamble \ - const size_t src_row_size = pctx->src_row_size; \ - const size_t src_row_size_aligned = pctx->src_row_size_aligned; \ - const size_t dst_row_size = pctx->dst_row_size; \ - const size_t dst_row_size_aligned = pctx->dst_row_size_aligned; \ - \ +#define htp_pad_dma_preamble \ + const size_t src_row_size = pctx->src_row_size; \ + const size_t src_row_size_aligned = pctx->src_row_size_aligned; \ + const size_t dst_row_size = pctx->dst_row_size; \ + const size_t dst_row_size_aligned = pctx->dst_row_size_aligned; \ + \ uint8_t * src_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; \ uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; \ \ @@ -125,8 +129,8 @@ static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * da struct htp_ops_context * octx = pctx->octx; htp_pad_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) { uint32_t i1, i2, i3; @@ -165,18 +169,17 @@ static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * da } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, row_start); - FARF(HIGH, "pad-hvx %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n", + FARF(HIGH, "pad-hvx %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n", ith, nth, src->ne[0], src->ne[1], src->ne[2], src->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - row_start, row_end, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + row_start, row_end); } // --------------------------------------------------------------------------- -// HVX + DMA PAD kernel — aligned, double-buffered +// HVX + DMA PAD kernel - aligned, double-buffered // --------------------------------------------------------------------------- static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void * data) { @@ -185,9 +188,6 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void htp_pad_preamble; htp_pad_dma_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - // ----------------------------------------------------------------------- // Priming phase: push 2 pairs of (dummy_dst_DMA, src_DMA) to seed the // double-buffer pipeline before the main loop begins. @@ -222,6 +222,8 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void // Main loop: pop completed DMAs, compute in VTCM with aligned HVX ops, // push dst DMA and prefetch src for the next+1 row. // ----------------------------------------------------------------------- + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = row_start; ir < row_end; ir++) { uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma).src; uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma).dst; @@ -236,6 +238,7 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void lp2, rp2, ne2, lp3, rp3, ne3); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); if (!interior) { hvx_splat_f32_a(dst_spad_cur, 0.0f, ne0); } else { @@ -249,6 +252,7 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void hvx_copy_f32_ua(dst_interior, src_spad_cur, ne00); } } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); dma_queue_push_vtcm_to_ddr(dma, dma_make_ptr(dst_ptr, dst_spad_cur), @@ -274,14 +278,11 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void dma_queue_flush(dma); - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "pad-hvx-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n", + FARF(HIGH, "pad-hvx-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n", ith, nth, src->ne[0], src->ne[1], src->ne[2], src->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - row_start, row_end, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + row_start, row_end); } // --------------------------------------------------------------------------- @@ -293,8 +294,8 @@ static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith, struct htp_ops_context * octx = pctx->octx; htp_pad_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) { uint32_t i1, i2, i3; @@ -344,18 +345,17 @@ static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith, } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, row_start); - FARF(HIGH, "pad-hvx-circ %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n", + FARF(HIGH, "pad-hvx-circ %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n", ith, nth, src->ne[0], src->ne[1], src->ne[2], src->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - row_start, row_end, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + row_start, row_end); } // --------------------------------------------------------------------------- -// HVX + DMA circular PAD kernel — aligned, double-buffered +// HVX + DMA circular PAD kernel - aligned, double-buffered // --------------------------------------------------------------------------- static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int ith, void * data) { @@ -364,9 +364,6 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i htp_pad_preamble; htp_pad_dma_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - // ----------------------------------------------------------------------- // Priming phase: push 2 pairs of (dummy_dst_DMA, src_DMA) to seed the // double-buffer pipeline. Every row is a real src DMA (no null DMAs). @@ -390,6 +387,8 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i // Main loop: pop completed DMAs, assemble circular row in VTCM with // aligned HVX ops, push dst DMA and prefetch src for the next+1 row. // ----------------------------------------------------------------------- + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = row_start; ir < row_end; ir++) { uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma).src; uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma).dst; @@ -398,7 +397,7 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i pad_decompose_row(ir, ne1, ne2, &i1, &i2, &i3); uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3; - + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); if (lp0 > 0) { uint8_t * dst_left = dst_spad_cur; const uint8_t * src_left = src_spad_cur + (size_t)(ne00 - (uint32_t)lp0) * type_size; @@ -430,6 +429,7 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i } } } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); dma_queue_push_vtcm_to_ddr(dma, dma_make_ptr(dst_ptr, dst_spad_cur), @@ -448,14 +448,11 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i dma_queue_flush(dma); - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "pad-hvx-circ-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n", + FARF(HIGH, "pad-hvx-circ-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n", ith, nth, src->ne[0], src->ne[1], src->ne[2], src->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - row_start, row_end, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + row_start, row_end); } int op_pad(struct htp_ops_context * octx) { @@ -489,19 +486,33 @@ int op_pad(struct htp_ops_context * octx) { const uint32_t ne00 = src0->ne[0]; const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, total_dst_rows > 0 ? total_dst_rows : 1); + const size_t dst_row_size = (size_t)ne0 * type_size; + + uint32_t row_start = 0; + uint32_t nrows = total_dst_rows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, type_size, (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_dst_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; const size_t src_row_size = (size_t)ne00 * type_size; - const size_t dst_row_size = (size_t)ne0 * type_size; const size_t src_row_size_aligned = hex_round_up(src_row_size, VLEN); const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN); // Total VTCM needed: 2 buffers (ping+pong) for src and dst, per thread const size_t vtcm_needed = (size_t)n_threads * 2 * (src_row_size_aligned + dst_row_size_aligned); - const int use_dma = (src0->nb[0] == (uint32_t)type_size) && - (ne00 >= 512) && - (octx->ctx->vtcm_base != NULL) && + const int use_dma = (src0->nb[0] == (uint32_t)type_size) && (ne00 >= 512) && (octx->ctx->vtcm_size >= vtcm_needed); if (use_dma) { @@ -521,8 +532,9 @@ int op_pad(struct htp_ops_context * octx) { .lp1 = lp1, .rp1 = rp1, .lp2 = lp2, .rp2 = rp2, .lp3 = lp3, .rp3 = rp3, - .nrows_per_thread = (total_dst_rows + n_threads - 1) / n_threads, - .total_dst_rows = total_dst_rows, + .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .total_dst_rows = nrows, + .row_start = row_start, .type_size = type_size, .src_row_size = src_row_size, .src_row_size_aligned = src_row_size_aligned, @@ -537,11 +549,10 @@ int op_pad(struct htp_ops_context * octx) { dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3); - if (circular && use_dma) { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx_circular_dma, &pctx, n_threads); } - else if (circular) { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx_circular, &pctx, n_threads); } - else if (use_dma) { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx_dma, &pctx, n_threads); } - else { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx, &pctx, n_threads); } + if (circular && use_dma) { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular_dma, &pctx, n_threads); } + else if (circular) { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular, &pctx, n_threads); } + else if (use_dma) { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_dma, &pctx, n_threads); } + else { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx, &pctx, n_threads); } return HTP_STATUS_OK; } - diff --git a/ggml/src/ggml-hexagon/htp/repeat-ops.c b/ggml/src/ggml-hexagon/htp/repeat-ops.c index a6f2f0ed5..530279d65 100644 --- a/ggml/src/ggml-hexagon/htp/repeat-ops.c +++ b/ggml/src/ggml-hexagon/htp/repeat-ops.c @@ -12,8 +12,10 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "htp-ctx.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" struct htp_repeat_context { struct htp_ops_context * octx; @@ -25,6 +27,7 @@ struct htp_repeat_context { uint32_t nrows_per_thread; uint32_t total_dst_rows; // ne1 * ne2 * ne3 + uint32_t row_start; size_t type_size; }; @@ -62,11 +65,11 @@ static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * dat const size_t row_bytes = ne00 * rctx->type_size; - const uint32_t row_start = rctx->nrows_per_thread * ith; - const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->total_dst_rows); + const uint32_t row_start = rctx->row_start + rctx->nrows_per_thread * ith; + const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->row_start + rctx->total_dst_rows); - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) { // Decompose flat dst row index into (i1, i2, i3) @@ -89,12 +92,12 @@ static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * dat } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, row_start); - FARF(HIGH, "repeat %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n", + FARF(HIGH, "repeat %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n", ith, nth, src->ne[0], src->ne[1], src->ne[2], src->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - row_start, row_end, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + row_start, row_end); } int op_repeat(struct htp_ops_context * octx) { @@ -119,21 +122,39 @@ int op_repeat(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, total_dst_rows); - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { return HTP_STATUS_OK; } + const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3]; + const size_t dst_row_size = dst->ne[0] * type_size; + + uint32_t row_start = 0; + uint32_t nrows = total_dst_rows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, type_size, (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_dst_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + struct htp_repeat_context rctx = { .octx = octx, .nr0 = dst->ne[0] / src0->ne[0], .nr1 = dst->ne[1] / src0->ne[1], .nr2 = dst->ne[2] / src0->ne[2], .nr3 = dst->ne[3] / src0->ne[3], - .nrows_per_thread = (total_dst_rows + n_threads - 1) / n_threads, - .total_dst_rows = total_dst_rows, + .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .total_dst_rows = nrows, + .row_start = row_start, .type_size = type_size, }; @@ -142,7 +163,7 @@ int op_repeat(struct htp_ops_context * octx) { dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], rctx.nr0, rctx.nr1, rctx.nr2, rctx.nr3); - worker_pool_run_func(octx->ctx->worker_pool, repeat_job_per_thread, &rctx, n_threads); + work_queue_run(octx->ctx->work_queue, repeat_job_per_thread, &rctx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c index 0a4b31ccb..c36976ed0 100644 --- a/ggml/src/ggml-hexagon/htp/rope-ops.c +++ b/ggml/src/ggml-hexagon/htp/rope-ops.c @@ -80,6 +80,8 @@ struct htp_rope_context { size_t dst_row_stride; size_t src0_row_size_aligned; uint32_t src0_nrows; + uint32_t row_start; + uint32_t nrows; struct fastdiv_values div_ne2_ne1; struct fastdiv_values div_ne1; @@ -539,11 +541,11 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { htp_rope_preamble; - const uint32_t src0_nrows = rctx->src0_nrows; + const uint32_t src0_nrows = rctx->nrows; const uint32_t src0_nrows_per_thread = rctx->src0_nrows_per_thread; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = rctx->row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, rctx->row_start + src0_nrows); // no work for this thread if (src0_start_row >= src0_end_row) { @@ -706,9 +708,32 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { } const struct htp_rope_kernel_params * kparams = (const struct htp_rope_kernel_params *) octx->kernel_params; - assert(kparams->n_threads > 0); + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } assert(octx->ctx->vtcm_size >= kparams->vtcm_size); + const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const size_t dst_data_row_size = dst->ne[0] * sizeof(float); + + uint32_t row_start = 0; + uint32_t nrows = total_rows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_data_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + const uint32_t ne0 = dst->ne[0]; const size_t src0_row_size = src0->ne[0] * sizeof(float); const size_t src0_row_stride = src0->nb[1]; @@ -752,15 +777,17 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { rctx.dst_row_stride = dst_row_stride; rctx.src0_row_size_aligned = kparams->src0_row_size_aligned; - rctx.src0_nrows = kparams->src0_nrows; - rctx.src0_nrows_per_thread = kparams->src0_nrows_per_thread; + rctx.src0_nrows = nrows; + rctx.nrows = nrows; + rctx.row_start = row_start; + rctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); rctx.div_ne2_ne1 = kparams->div_ne2_ne1; rctx.div_ne1 = kparams->div_ne1; FARF(HIGH, "rope-f32 n-rows %u n-dims %d ne0 %u ext-factor %.6f theta-scale %.6f attn-factor %.6f\n", rctx.src0_nrows, rctx.n_dims, ne0, rctx.ext_factor, rctx.theta_scale, rctx.attn_factor); - work_queue_run(octx->ctx->work_queue, rope_job_f32, &rctx, kparams->n_threads); + work_queue_run(octx->ctx->work_queue, rope_job_f32, &rctx, n_threads); return err; } diff --git a/ggml/src/ggml-hexagon/htp/set-rows-ops.c b/ggml/src/ggml-hexagon/htp/set-rows-ops.c index 340a497f7..fbd5162a7 100644 --- a/ggml/src/ggml-hexagon/htp/set-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/set-rows-ops.c @@ -18,6 +18,7 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -58,6 +59,9 @@ struct set_rows_context { const struct htp_set_rows_kernel_params * kparams; struct htp_set_rows_vtcm_layout vtcm_layout; uint8_t * vtcm_base; + uint32_t task_start; + uint32_t tasks; + uint32_t tasks_per_thread; }; #define SET_ROWS_THREAD_DMA_FN(TYPE_NAME, IDX_TYPE, COMPUTE_EXPR) \ @@ -67,12 +71,12 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsig const struct htp_set_rows_kernel_params * kparams = srctx->kparams; \ set_rows_preamble; \ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - const uint32_t dr = kparams->tasks_per_thread; \ - const uint32_t ir0 = dr * ith; \ - if (ir0 >= kparams->total_tasks) { \ + const uint32_t dr = srctx->tasks_per_thread; \ + const uint32_t ir0 = srctx->task_start + dr * ith; \ + if (ir0 >= srctx->task_start + srctx->tasks) { \ return; \ } \ - const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \ + const uint32_t ir1 = MIN(ir0 + dr, srctx->task_start + srctx->tasks); \ dma_queue * dma_queue = octx->ctx->dma[ith]; \ const struct htp_set_rows_vtcm_layout * vtcm_layout = &srctx->vtcm_layout; \ uint8_t * vtcm_src0 = srctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; \ @@ -192,18 +196,44 @@ int op_set_rows(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - if (octx->src[1]->type != HTP_TYPE_I32 && octx->src[1]->type != HTP_TYPE_I64) { - return HTP_STATUS_NO_SUPPORT; + if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { + return HTP_STATUS_OK; } + const struct htp_tensor * dst = octx->dst; + const uint32_t total_tasks = kparams->total_tasks; + + uint32_t task_start = 0; + uint32_t tasks = total_tasks; + + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && (dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0 && !htp_tensor_is_permuted(dst); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_tasks, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + task_start = range.start; + tasks = range.count; + } + + if (tasks == 0) { + return HTP_STATUS_OK; + } + + if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t n_threads = octx->n_threads; + // l2fetch the src1 (indices) tensor in the main thread hex_l2fetch_block((const void *)octx->src[1]->data, octx->src[1]->ne[3] * octx->src[1]->nb[3]); struct set_rows_context srctx; srctx.octx = octx; srctx.kparams = kparams; + srctx.task_start = task_start; + srctx.tasks = tasks; + srctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div); - htp_set_rows_vtcm_layout_build(&srctx.vtcm_layout, octx->dst->type, ne00, kparams->n_threads); + htp_set_rows_vtcm_layout_build(&srctx.vtcm_layout, octx->dst->type, ne00, n_threads); srctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base; work_queue_func_t q_func = NULL; @@ -216,15 +246,15 @@ int op_set_rows(struct htp_ops_context * octx) { default: return HTP_STATUS_NO_SUPPORT; } - FARF(HIGH, "set-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu n_threads %d\n", + FARF(HIGH, "set-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu n-threads %d\n", octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3], octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3], octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3], - srctx.vtcm_layout.src0_bytes_per_thread * kparams->n_threads, - srctx.vtcm_layout.dst_bytes_per_thread * kparams->n_threads, - kparams->n_threads); + srctx.vtcm_layout.src0_bytes_per_thread * n_threads, + srctx.vtcm_layout.dst_bytes_per_thread * n_threads, + n_threads); - work_queue_run(octx->ctx->work_queue, q_func, &srctx, kparams->n_threads); + work_queue_run(octx->ctx->work_queue, q_func, &srctx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/softmax-ops.c b/ggml/src/ggml-hexagon/htp/softmax-ops.c index d78bcc0eb..2497ec763 100644 --- a/ggml/src/ggml-hexagon/htp/softmax-ops.c +++ b/ggml/src/ggml-hexagon/htp/softmax-ops.c @@ -14,9 +14,11 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" #define htp_softmax_preamble3 \ const uint32_t ne00 = src0->ne[0]; \ @@ -69,6 +71,8 @@ struct htp_softmax_context { struct fastdiv_values fastdiv_ne13; // For mask broadcasting uint32_t src0_nrows_per_thread; + uint32_t row_start; + uint32_t nrows; }; static void apply_mask(float * restrict wp0, @@ -223,19 +227,17 @@ static void softmax_job_f32(unsigned int nth, unsigned int ith, void * data) { htp_softmax_preamble3; - const uint32_t src0_nrows = ne01 * ne02 * ne03; // src0 rows + const uint32_t src0_nrows = smctx->nrows; const uint32_t src0_nrows_per_thread = smctx->src0_nrows_per_thread; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, smctx->row_start + src0_nrows); // no work for this thread if (src0_start_row >= src0_end_row) { return; } - uint64_t qt = HAP_perf_get_qtimer_count(); - int is_aligned = 1; int opt_path = 0; @@ -262,6 +264,9 @@ static void softmax_job_f32(unsigned int nth, unsigned int ith, void * data) { uint32_t prev_i2 = (uint32_t)-1; float slope = 1.0f; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, src0_start_row); + for (uint32_t r = src0_start_row; r < src0_end_row; ++r) { uint32_t i1 = fastmodulo(r, ne01, &smctx->fastdiv_ne01); uint32_t r_div_ne01 = fastdiv(r, &smctx->fastdiv_ne01); @@ -323,10 +328,11 @@ static void softmax_job_f32(unsigned int nth, unsigned int ith, void * data) { } } - qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt); - FARF(HIGH, "softmax-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u : opt %u f16 %u usec %u\n", ith, nth, + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, src0_start_row); + + FARF(HIGH, "softmax-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u : opt %u f16 %u\n", ith, nth, ne00, ne01, ne02, ne03, src0_start_row, src0_end_row, ne10, ne11, ne12, ne13, - ne0, ne1, ne2, ne3, opt_path, smctx->use_f16, (unsigned) qt); + ne0, ne1, ne2, ne3, opt_path, smctx->use_f16); } static int execute_op_softmax_f32(struct htp_ops_context * octx) { @@ -342,13 +348,32 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { init_softmax_ctx(&smctx, octx); const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); + const size_t elem_size = sizeof(float); + const size_t dst_row_size = dst->nb[1]; - smctx.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) elem_size, (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + + smctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + smctx.row_start = row_start; + smctx.nrows = nrows; const size_t src0_row_size = src0->nb[1]; const size_t src1_row_size = src0_row_size; - const size_t dst_row_size = dst->nb[1]; // VTCM scratchpads for all tensors // 4 rows per thread, padded to HVX vector size @@ -383,9 +408,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->src1_spad.src = NULL; octx->dst_spad.data = octx->src1_spad.data + octx->src1_spad.size; octx->dst_spad.src = NULL; - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) return err; - - worker_pool_run_func(octx->ctx->worker_pool, softmax_job_f32, &smctx, n_threads); + work_queue_run(octx->ctx->work_queue, softmax_job_f32, &smctx, n_threads); return err; } diff --git a/ggml/src/ggml-hexagon/htp/solve-tri-ops.c b/ggml/src/ggml-hexagon/htp/solve-tri-ops.c index ae8e1a504..847a78712 100644 --- a/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +++ b/ggml/src/ggml-hexagon/htp/solve-tri-ops.c @@ -1,13 +1,16 @@ #pragma clang diagnostic ignored "-Wunused-but-set-variable" #include -#include #include +#include "hex-common.h" +#include "hex-profile.h" + #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" #include "hvx-types.h" #include "hvx-utils.h" @@ -15,6 +18,7 @@ struct htp_solve_tri_context { struct htp_ops_context * octx; uint32_t jobs_per_thread; uint32_t total_jobs; + uint32_t job_start; uint32_t k_chunks; uint32_t col_block; }; @@ -89,11 +93,11 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void const uint32_t col_block = VLEN_FP32; const uint32_t k_full = (k / col_block) * col_block; - const uint32_t start_batch = sctx->jobs_per_thread * ith; - const uint32_t end_batch = MIN(start_batch + sctx->jobs_per_thread, sctx->total_jobs); + const uint32_t start_batch = sctx->job_start + sctx->jobs_per_thread * ith; + const uint32_t end_batch = MIN(start_batch + sctx->jobs_per_thread, sctx->job_start + sctx->total_jobs); - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_batch); for (uint32_t batch = start_batch; batch < end_batch; ++batch) { const uint32_t i03 = batch / ne02; @@ -127,11 +131,10 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) end_batch); - FARF(HIGH, "solve-tri-batch %d/%d: A=(%ux%u) B=(%ux%u) batch %u:%u usec %u\n", - ith, nth, n, n, k, n, start_batch, end_batch, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + FARF(HIGH, "solve-tri-batch %d/%d: A=(%ux%u) B=(%ux%u) batch %u:%u\n", + ith, nth, n, n, k, n, start_batch, end_batch); } // Chunk-level thread: each job is one (batch, col_chunk) pair. @@ -148,11 +151,11 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void const uint32_t ne02 = src0->ne[2]; - const uint32_t start_job = sctx->jobs_per_thread * ith; - const uint32_t end_job = MIN(start_job + sctx->jobs_per_thread, sctx->total_jobs); + const uint32_t start_job = sctx->job_start + sctx->jobs_per_thread * ith; + const uint32_t end_job = MIN(start_job + sctx->jobs_per_thread, sctx->job_start + sctx->total_jobs); - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_job); for (uint32_t job = start_job; job < end_job; ++job) { const uint32_t batch = job / sctx->k_chunks; @@ -161,16 +164,14 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void const uint32_t i03 = batch / ne02; const uint32_t i02 = batch - i03 * ne02; - const uint32_t col0 = chunk * sctx->col_block; - const uint32_t coln = MIN(sctx->col_block, k - col0); - const float * A_batch = (const float *) ((const uint8_t *) (uintptr_t) src0->data + i02 * src0->nb[2] + i03 * src0->nb[3]); const float * B_batch = (const float *) ((const uint8_t *) (uintptr_t) src1->data + i02 * src1->nb[2] + i03 * src1->nb[3]); float * X_batch = (float *) ((uint8_t *) (uintptr_t) dst->data + i02 * dst->nb[2] + i03 * dst->nb[3]); - const bool use_hvx = (coln >= 8); + const uint32_t col0 = chunk * sctx->col_block; + const uint32_t coln = MIN(sctx->col_block, k - col0); for (uint32_t row = 0; row < n; ++row) { const float diag = A_batch[row * n + row]; @@ -179,7 +180,7 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void const float * A_row = A_batch + row * n; const float * B_row = B_batch + row * k; - if (use_hvx) { + if (coln >= 8) { solve_tri_row_hvx(A_row, B_row, X_batch, row, k, col0, coln, inv_diag); } else { solve_tri_row_scalar(A_row, B_row, X_batch, row, k, col0, coln, inv_diag); @@ -187,11 +188,10 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) end_job); - FARF(HIGH, "solve-tri-chunk %d/%d: A=(%ux%u) B=(%ux%u) job %u:%u usec %u\n", - ith, nth, n, n, k, n, start_job, end_job, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + FARF(HIGH, "solve-tri-chunk %d/%d: A=(%ux%u) B=(%ux%u) jobs %u:%u\n", + ith, nth, n, n, k, n, start_job, end_job); } int op_solve_tri(struct htp_ops_context * octx) { @@ -235,32 +235,64 @@ int op_solve_tri(struct htp_ops_context * octx) { dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], batched); if (batched) { + uint32_t job_start = 0; + uint32_t njobs = total_batches; + + if (octx->ctx->mdev.count > 1) { + const uint32_t batch_size = dst->nb[2]; + const uint32_t batches_per_chunk = (batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(batch_size, HEX_L2_LINE_SIZE)) : 1; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, htp_tensor_mdev_data_aligned(dst) ? batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + job_start = range.start; + njobs = range.count; + } + + if (njobs == 0) { + return HTP_STATUS_OK; + } + // Batch-level parallelism - const uint32_t n_threads = MIN((uint32_t) octx->n_threads, total_batches); + const uint32_t n_threads = octx->n_threads; struct htp_solve_tri_context sctx = { .octx = octx, - .jobs_per_thread = (total_batches + n_threads - 1) / n_threads, - .total_jobs = total_batches, + .jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div), + .total_jobs = njobs, + .job_start = job_start, .k_chunks = k_chunks, .col_block = col_block, }; - worker_pool_run_func(octx->ctx->worker_pool, solve_tri_batch_thread_f32, &sctx, n_threads); + work_queue_run(octx->ctx->work_queue, solve_tri_batch_thread_f32, &sctx, n_threads); } else { // Chunk-level parallelism const uint32_t total_jobs = total_batches * k_chunks; - const uint32_t n_threads = MIN((uint32_t) octx->n_threads, MAX(total_jobs, 1)); + + uint32_t job_start = 0; + uint32_t njobs = total_jobs; + + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_jobs, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + job_start = range.start; + njobs = range.count; + } + + if (njobs == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; struct htp_solve_tri_context sctx = { .octx = octx, - .jobs_per_thread = (total_jobs + n_threads - 1) / n_threads, - .total_jobs = total_jobs, + .jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div), + .total_jobs = njobs, + .job_start = job_start, .k_chunks = k_chunks, .col_block = col_block, }; - worker_pool_run_func(octx->ctx->worker_pool, solve_tri_chunk_thread_f32, &sctx, n_threads); + work_queue_run(octx->ctx->work_queue, solve_tri_chunk_thread_f32, &sctx, n_threads); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/ssm-conv.c b/ggml/src/ggml-hexagon/htp/ssm-conv.c index a48bc9ed8..bef142536 100644 --- a/ggml/src/ggml-hexagon/htp/ssm-conv.c +++ b/ggml/src/ggml-hexagon/htp/ssm-conv.c @@ -4,7 +4,6 @@ #include #include -#include #include #include #include @@ -16,8 +15,9 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "hex-dma.h" +#include "hex-profile.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" #include "hvx-utils.h" #define htp_ssm_conv_tensors_preamble \ @@ -63,6 +63,8 @@ struct htp_ssm_conv_context { uint32_t nrows_per_thread; uint32_t d_inner_tile; uint64_t t_start; + uint32_t row_start; + uint32_t nrows; }; #define htp_ssm_conv_preamble \ @@ -75,9 +77,6 @@ struct htp_ssm_conv_context { static void ssm_conv_thread_f32_f32(unsigned int nth, unsigned int ith, void *data) { htp_ssm_conv_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - const uint32_t d_conv = src1->ne[0]; const uint32_t d_inner = src0->ne[1]; const uint32_t n_t = dst->ne[1]; @@ -95,14 +94,17 @@ static void ssm_conv_thread_f32_f32(unsigned int nth, unsigned int ith, void *da // Calculate row range for this thread const uint32_t d_inner_per_thread = scctx->nrows_per_thread; - const uint32_t d_inner_start = d_inner_per_thread * ith; - const uint32_t d_inner_end = MIN(d_inner_start + d_inner_per_thread, d_inner); + const uint32_t d_inner_start = scctx->row_start + d_inner_per_thread * ith; + const uint32_t d_inner_end = MIN(d_inner_start + d_inner_per_thread, scctx->row_start + scctx->nrows); // No work for this thread if (d_inner_start >= d_inner_end) { return; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) d_inner_start); + for (uint32_t i3 = 0; i3 < n_s; ++i3) { for (uint32_t i2 = 0; i2 < n_t; ++i2) { for (uint32_t i1 = d_inner_start; i1 < d_inner_end; ++i1) { @@ -121,12 +123,12 @@ static void ssm_conv_thread_f32_f32(unsigned int nth, unsigned int ith, void *da } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) d_inner_end); - FARF(HIGH, "ssm-conv-f32 %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", + FARF(HIGH, "ssm-conv-f32 %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u\n", ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], d_inner_start, d_inner_end, src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1], - dst->ne[2], dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + dst->ne[2], dst->ne[3]); } @@ -257,9 +259,6 @@ static inline void transpose_src0_block(const float * src0_block, static void ssm_conv_thread_f32_f32_hvx(unsigned int nth, unsigned int ith, void *data) { htp_ssm_conv_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - const uint32_t d_conv = src1->ne[0]; const uint32_t d_inner = src0->ne[1]; const uint32_t n_t = dst->ne[1]; @@ -273,13 +272,16 @@ static void ssm_conv_thread_f32_f32_hvx(unsigned int nth, unsigned int ith, void const uint32_t dst_stride_seq = dst->nb[2] / sizeof(float); const uint32_t dr = scctx->nrows_per_thread; - const uint32_t ir0 = dr * ith; - const uint32_t ir1 = MIN(ir0 + dr, d_inner); + const uint32_t ir0 = scctx->row_start + dr * ith; + const uint32_t ir1 = MIN(ir0 + dr, scctx->row_start + scctx->nrows); if (ir0 >= ir1) { return; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); + const uint32_t d_inner_per_thread = ir1 - ir0; const uint32_t d_inner_stride = scctx->nrows_per_thread; const uint32_t d_inner_tile = scctx->d_inner_tile; @@ -319,97 +321,118 @@ static void ssm_conv_thread_f32_f32_hvx(unsigned int nth, unsigned int ith, void HVX_Vector w = *(const HVX_Vector *) (src1_T + j * d_inner_stride + tile_off + cb); acc = Q6_Vqf32_vadd_Vqf32Vqf32(acc, Q6_Vqf32_vmpy_VsfVsf(x, w)); } - HVX_Vector res = Q6_Vsf_equals_Vqf32(acc); - float * dst_ptr = dst_data + i3 * dst_stride_seq + t * dst_stride_token + (ir0 + tile_off + cb); + HVX_Vector y = Q6_Vsf_equals_Vqf32(acc); + + float * dst_ptr = dst_data + (ir0 + tile_off + cb) + t * dst_stride_token + i3 * dst_stride_seq; if (cb_n == C_TILE) { - *(HVX_UVector *) dst_ptr = res; + *(HVX_UVector *) dst_ptr = y; } else { - hvx_vec_store_u(dst_ptr, cb_n * sizeof(float), res); + hvx_vec_store_u(dst_ptr, cb_n * sizeof(float), y); } } } } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir1); - FARF(HIGH, "ssm-conv-f32-hvx %d/%d: %ux%ux%ux%u (%u:%u) tile=%u * %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", - ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1, d_inner_tile, + FARF(HIGH, "ssm-conv-f32-hvx %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u\n", + ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1, src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1], - dst->ne[2], dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + dst->ne[2], dst->ne[3]); } int op_ssm_conv_f32(struct htp_ops_context * octx) { - htp_ssm_conv_tensors_preamble; + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * src1 = octx->src[1]; + const struct htp_tensor * dst = octx->dst; if (src0->type != HTP_TYPE_F32 || src1->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_F32) { - FARF(ERROR, "ssm_conv: only (F32 x F32 -> F32) OPs supported"); return HTP_STATUS_NO_SUPPORT; } - struct htp_ssm_conv_context scctx = { 0 }; - scctx.octx = octx; - const uint32_t d_conv = src1->ne[0]; const uint32_t d_inner = src0->ne[1]; const uint32_t n_t = dst->ne[1]; // tokens per sequence const uint32_t n_s = dst->ne[2]; // number of sequences in the batch - const uint32_t n_threads = MIN(octx->n_threads, d_inner); + if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { + return HTP_STATUS_OK; + } - if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { - uint32_t use_hvx = 0; - if (d_inner >= VLEN_FP32 && n_t >= VLEN_FP32) { - use_hvx = 1; - } + uint32_t row_start = 0; + uint32_t nrows = d_inner; - scctx.nrows_per_thread = hex_round_up((d_inner + n_threads - 1) / n_threads, VLEN_FP32); + if (octx->ctx->mdev.count > 1) { + const uint32_t elems_per_chunk = VLEN_FP32; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(d_inner, htp_tensor_mdev_data_aligned(dst) ? elems_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } - const uint32_t d_inner_per_thread = scctx.nrows_per_thread; - const uint32_t ncs = src0->ne[0]; + if (nrows == 0) { + return HTP_STATUS_OK; + } - const uint32_t src1_T_size = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 256); - const uint32_t src0_T_max = HTP_SSM_CONV_VTCM_BUDGET > src1_T_size ? HTP_SSM_CONV_VTCM_BUDGET - src1_T_size : 0; + const uint32_t n_threads = octx->n_threads; - uint32_t d_inner_tile = (src0_T_max / sizeof(float)) / ncs; - d_inner_tile -= (d_inner_tile % VLEN_FP32); - if (d_inner_tile == 0) { - FARF(HIGH, "ssm_conv-f32: inner tile rounds to 0 (ncs=%u), falling back to scalar\n", ncs); + struct htp_ssm_conv_context scctx = { 0 }; + scctx.octx = octx; + scctx.row_start = row_start; + scctx.nrows = nrows; + + uint32_t use_hvx = 0; + if (nrows >= VLEN_FP32 && n_t >= VLEN_FP32) { + use_hvx = 1; + } + + const uint32_t raw_rpt = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + scctx.nrows_per_thread = hex_round_up(raw_rpt, VLEN_FP32); + + const uint32_t d_inner_per_thread = scctx.nrows_per_thread; + const uint32_t ncs = src0->ne[0]; + + const uint32_t src1_T_size = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 256); + const uint32_t src0_T_max = HTP_SSM_CONV_VTCM_BUDGET > src1_T_size ? HTP_SSM_CONV_VTCM_BUDGET - src1_T_size : 0; + + uint32_t d_inner_tile = (src0_T_max / sizeof(float)) / ncs; + d_inner_tile -= (d_inner_tile % VLEN_FP32); + if (d_inner_tile == 0) { + FARF(HIGH, "ssm_conv-f32: inner tile rounds to 0 (ncs=%u), falling back to scalar\n", ncs); + use_hvx = 0; + } else { + scctx.d_inner_tile = d_inner_tile; + + octx->src0_spad.size_per_thread = hex_round_up(d_inner_tile * ncs * sizeof(float), 256); + octx->src1_spad.size_per_thread = src1_T_size; + octx->dst_spad.size_per_thread = 0; + + octx->src0_spad.size = octx->src0_spad.size_per_thread * n_threads; + octx->src1_spad.size = octx->src1_spad.size_per_thread * n_threads; + octx->dst_spad.size = 0; + + octx->src0_spad.data = octx->ctx->vtcm_base; + octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size; + octx->src0_spad.src = NULL; + octx->src1_spad.src = NULL; + + const size_t total_spad = octx->src0_spad.size + octx->src1_spad.size; + if (total_spad > octx->ctx->vtcm_size) { + FARF(HIGH, "ssm_conv-f32: scratchpad %zu exceeds VTCM %zu, falling back to scalar\n", + total_spad, octx->ctx->vtcm_size); use_hvx = 0; - } else { - scctx.d_inner_tile = d_inner_tile; - - octx->src0_spad.size_per_thread = hex_round_up(d_inner_tile * ncs * sizeof(float), 256); - octx->src1_spad.size_per_thread = src1_T_size; - octx->dst_spad.size_per_thread = 0; - - octx->src0_spad.size = octx->src0_spad.size_per_thread * n_threads; - octx->src1_spad.size = octx->src1_spad.size_per_thread * n_threads; - octx->dst_spad.size = 0; - - octx->src0_spad.data = octx->ctx->vtcm_base; - octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size; - octx->src0_spad.src = NULL; - octx->src1_spad.src = NULL; - - const size_t total_spad = octx->src0_spad.size + octx->src1_spad.size; - if (total_spad > octx->ctx->vtcm_size) { - FARF(HIGH, "ssm_conv-f32: scratchpad %zu exceeds VTCM %zu, falling back to scalar\n", - total_spad, octx->ctx->vtcm_size); - use_hvx = 0; - } } + } - FARF(HIGH, "ssm-conv-f32: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : use_hvx %d\n", src0->ne[0], - src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], - dst->ne[1], dst->ne[2], dst->ne[3], use_hvx); + FARF(HIGH, "ssm-conv-f32: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : use_hvx %d\n", src0->ne[0], + src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], + dst->ne[1], dst->ne[2], dst->ne[3], use_hvx); - if (use_hvx) { - worker_pool_run_func(octx->ctx->worker_pool, ssm_conv_thread_f32_f32_hvx, &scctx, n_threads); - } else { - worker_pool_run_func(octx->ctx->worker_pool, ssm_conv_thread_f32_f32, &scctx, n_threads); - } + if (use_hvx) { + work_queue_run(octx->ctx->work_queue, ssm_conv_thread_f32_f32_hvx, &scctx, n_threads); + } else { + work_queue_run(octx->ctx->work_queue, ssm_conv_thread_f32_f32, &scctx, n_threads); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/sum-rows-ops.c b/ggml/src/ggml-hexagon/htp/sum-rows-ops.c index 874c41ab2..faf716b4b 100644 --- a/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/sum-rows-ops.c @@ -13,35 +13,38 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" #define sum_rows_preamble \ const struct htp_tensor *src0 = octx->src[0]; \ const struct htp_tensor *dst = octx->dst; \ \ - const uint32_t ne00 = src0->ne[0]; \ - const uint32_t ne01 = src0->ne[1]; \ - const uint32_t ne02 = src0->ne[2]; \ - const uint32_t ne03 = src0->ne[3]; \ - \ - const uint32_t nb00 = src0->nb[0]; \ - const uint32_t nb01 = src0->nb[1]; \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb0 = dst->nb[0]; \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; \ + const uint32_t ne00 = src0->ne[0]; \ + const uint32_t ne01 = src0->ne[1]; \ + const uint32_t ne02 = src0->ne[2]; \ + const uint32_t ne03 = src0->ne[3]; \ + \ + const uint32_t nb00 = src0->nb[0]; \ + const uint32_t nb01 = src0->nb[1]; \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb0 = dst->nb[0]; \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ + const uint32_t nb3 = dst->nb[3]; \ struct sum_rows_context { + struct htp_ops_context * octx; const uint8_t * src_data; uint8_t * dst_data; uint32_t ne00; @@ -76,6 +79,9 @@ static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data) // Calculate actual number of rows for this thread const uint32_t n_rows = end_row - start_row; + struct htp_thread_trace * tr = &smctx->octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_row); + for (uint32_t ir = 0; ir < n_rows; ir++) { const float * restrict src_local = src_th + (ir * (src_stride / sizeof(float))); @@ -89,6 +95,8 @@ static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data) dst_th[ir] = hvx_reduce_sum_f32((const uint8_t *) src_local, ne00); } } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_row); } int op_sum_rows(struct htp_ops_context * octx) { @@ -102,9 +110,26 @@ int op_sum_rows(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t src0_nrows = ne01 * ne02 * ne03; - const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); - const uint32_t rows_per_thread = (src0_nrows + n_threads - 1) / n_threads; + const uint32_t src0_nrows = ne01 * ne02 * ne03; + const size_t dst_data_row_size = dst->ne[0] * sizeof(float); + + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_data_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + const uint32_t rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); bool opt_path = false; if ((0 == hex_is_aligned((void *) src0->data, VLEN)) && !(nb01 & (VLEN - 1))) { @@ -112,17 +137,18 @@ int op_sum_rows(struct htp_ops_context * octx) { } struct sum_rows_context smctx = { - .src_data = (const uint8_t *) src0->data, - .dst_data = (uint8_t *) dst->data, + .octx = octx, + .src_data = (const uint8_t *) src0->data + row_start * nb01, + .dst_data = (uint8_t *) dst->data + row_start * nb1, .ne00 = ne00, .src_stride = nb01, .dst_stride = nb1, .rows_per_thread = rows_per_thread, - .total_rows = src0_nrows, + .total_rows = nrows, .opt_path = opt_path, }; - worker_pool_run_func(octx->ctx->worker_pool, sum_rows_thread_f32, &smctx, n_threads); + work_queue_run(octx->ctx->work_queue, sum_rows_thread_f32, &smctx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c index 7850ab27e..cb82bfa3c 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.c +++ b/ggml/src/ggml-hexagon/htp/unary-ops.c @@ -46,6 +46,7 @@ struct htp_unary_context { uint32_t block; uint32_t src0_nrows; uint32_t src0_nrows_per_thread; + uint32_t row_start; uint32_t nc; uint32_t col_tile; // tiled mode bool broadcast_weight; @@ -496,7 +497,7 @@ static void tri_f32(const float * restrict src, } if (boundary > ne0) boundary = ne0; - // Full HVX vectors — each starts at a 128-byte aligned offset + // Full HVX vectors - each starts at a 128-byte aligned offset for (uint32_t i = 0; i < nvec; i++) { const uint32_t vec_start = i * VLEN_FP32; const uint32_t vec_end = vec_start + VLEN_FP32; @@ -563,7 +564,7 @@ static void softplus_f32(const float * restrict src, for (uint32_t i = 0; i < ne0; i++) { float x = src_f[i]; - // For x > 20: softplus(x) ≈ x (avoids exp overflow) + // For x > 20: softplus(x) ~ x (avoids exp overflow) dst_f[i] = (x > 20.0f) ? x : logf(1.0f + expf(x)); } } @@ -661,8 +662,8 @@ static void unary_task_##SUFFIX##_##NAME(unsigned int nth, unsigned int ith, voi const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; \ \ const uint32_t src0_nrows = uctx->src0_nrows; \ - const uint32_t src0_start_row = src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \ + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; \ + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); \ \ if (src0_start_row >= src0_end_row) { \ return; \ @@ -833,124 +834,126 @@ DEFINE_UNARY_TASK_IMPL(unary_abs, _Float16, f16, false, false, abs_f16(src0_vtcm DEFINE_UNARY_TASK_IMPL(unary_log, _Float16, f16, false, false, log_f16(src0_vtcm, dst_vtcm, block_size, uctx)) // Apply a pointwise unary op to one column tile that is already in VTCM. -#define DEFINE_UNARY_TILED_TASK(NAME, IS_TRI, CORE_TILE_EXPR) \ -static void unary_task_f32_tiled_##NAME(unsigned int nth, unsigned int ith, void * data) { \ - const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; \ - struct htp_ops_context * octx = uctx->octx; \ - const struct htp_tensor * src = octx->src[0]; \ - const struct htp_tensor * dst = octx->dst; \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - \ - htp_unary_preamble; \ - \ - int32_t * op_params = octx->op_params; \ - const uint32_t col_tile = uctx->col_tile; \ - \ - const uint32_t src0_nrows = uctx->src0_nrows; \ - const uint32_t src0_start_row = uctx->src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + uctx->src0_nrows_per_thread, src0_nrows); \ - \ - if (src0_start_row >= src0_end_row) { \ - return; \ - } \ - \ - const uint8_t * restrict data_src = uctx->data_src0; \ - uint8_t * restrict data_dst = uctx->data_dst; \ - \ - uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); \ - uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); \ - \ - const size_t src0_half = uctx->src0_vtcm_half_size; \ - const size_t dst_half = uctx->dst_vtcm_half_size; \ - \ - dma_queue * dmaq = octx->ctx->dma[ith]; \ - \ - const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; \ - const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; \ - const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; \ - const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr; \ - \ - const uint32_t tiles_per_row = (ne0 + col_tile - 1) / col_tile; \ - const int32_t tri_ttype = (IS_TRI) ? op_params[0] : 0; \ - \ - const bool src0_contig = (nb02 == (size_t)ne01 * nb01) && \ - (nb03 == (size_t)ne02 * nb02); \ - const bool dst_contig = (nb2 == (size_t)ne1 * nb1) && \ - (nb3 == (size_t)ne2 * nb2); \ - \ - const uint32_t total_tiles = (src0_end_row - src0_start_row) * tiles_per_row; \ - \ - for (uint32_t t = 0, vtcm_idx = 0; t < total_tiles && vtcm_idx < 2; t++, vtcm_idx++) { \ - const uint32_t row = src0_start_row + t / tiles_per_row; \ - const uint32_t col = (t % tiles_per_row) * col_tile; \ - const uint32_t tw = MIN(col_tile, ne0 - col); \ - const size_t tb = (size_t) tw * sizeof(float); \ - const size_t soff = (src0_contig ? (row * nb01) : \ - unary_row_offset(row, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03)) +\ - (size_t) col * sizeof(float); \ - \ - dma_queue_push(dmaq, dma_make_ptr(data_dst, dst_vtcm_data + (vtcm_idx * dst_half)), 0, 0, 0, 0); \ - dma_queue_push(dmaq, dma_make_ptr(src0_vtcm_data + (vtcm_idx * src0_half), data_src + soff), tb, tb, tb, 1);\ - } \ - \ - uint32_t row = src0_start_row; \ - uint32_t col = 0; \ - uint32_t tile_in_row = 0; \ - uint32_t i01 = fastmodulo(row, ne01, div_ne01); \ - \ - uint32_t prow = src0_start_row + fastdiv(2, div_tpr); \ - uint32_t pcol = fastmodulo(2, tiles_per_row, div_tpr) * col_tile; \ - uint32_t ptile_in_row = fastmodulo(2, tiles_per_row, div_tpr); \ - \ - for (uint32_t t = 0; t < total_tiles; t++) { \ - uint8_t * dst_vtcm = (uint8_t *) dma_queue_pop(dmaq).src; \ - uint8_t * src_vtcm = (uint8_t *) dma_queue_pop(dmaq).dst; \ - \ - const uint32_t tw = MIN(col_tile, ne0 - col); \ - \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, t); \ - CORE_TILE_EXPR; \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, t); \ - \ - const size_t doff = (dst_contig ? (row * nb1) : \ - unary_row_offset(row, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3)) + \ - (size_t) col * sizeof(float); \ - const size_t tb = (size_t) tw * sizeof(float); \ - dma_queue_push(dmaq, dma_make_ptr(data_dst + doff, dst_vtcm), tb, tb, tb, 1); \ - \ - const uint32_t pt = t + 2; \ - if (pt < total_tiles) { \ - const uint32_t ptw = MIN(col_tile, ne0 - pcol); \ - const size_t ptb = (size_t) ptw * sizeof(float); \ - const size_t psoff = (src0_contig ? (prow * nb01) : \ - unary_row_offset(prow, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, \ - nb03)) + \ - (size_t) pcol * sizeof(float); \ - dma_queue_push(dmaq, dma_make_ptr(src_vtcm, data_src + psoff), ptb, ptb, ptb, 1); \ - } \ - \ - tile_in_row++; \ - col += col_tile; \ - if (tile_in_row == tiles_per_row) { \ - tile_in_row = 0; \ - col = 0; \ - row++; \ - i01++; \ - if (i01 == ne01) { \ - i01 = 0; \ - } \ - } \ - \ - ptile_in_row++; \ - pcol += col_tile; \ - if (ptile_in_row == tiles_per_row) { \ - ptile_in_row = 0; \ - pcol = 0; \ - prow++; \ - } \ - } \ - \ - dma_queue_flush(dmaq); \ +#define DEFINE_UNARY_TILED_TASK(NAME, IS_TRI, CORE_TILE_EXPR) \ +static void unary_task_f32_tiled_##NAME(unsigned int nth, unsigned int ith, void * data) { \ + const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; \ + struct htp_ops_context * octx = uctx->octx; \ + const struct htp_tensor * src = octx->src[0]; \ + const struct htp_tensor * dst = octx->dst; \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + \ + htp_unary_preamble; \ + \ + uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; \ + \ + int32_t * op_params = octx->op_params; \ + const uint32_t col_tile = uctx->col_tile; \ + \ + const uint32_t src0_nrows = uctx->src0_nrows; \ + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; \ + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); \ + \ + if (src0_start_row >= src0_end_row) { \ + return; \ + } \ + \ + const uint8_t * restrict data_src = uctx->data_src0; \ + uint8_t * restrict data_dst = uctx->data_dst; \ + \ + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); \ + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); \ + \ + const size_t src0_half = uctx->src0_vtcm_half_size; \ + const size_t dst_half = uctx->dst_vtcm_half_size; \ + \ + dma_queue * dmaq = octx->ctx->dma[ith]; \ + \ + const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; \ + const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; \ + const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; \ + const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr; \ + \ + const uint32_t tiles_per_row = (ne0 + col_tile - 1) / col_tile; \ + const int32_t tri_ttype = (IS_TRI) ? op_params[0] : 0; \ + \ + const bool src0_contig = (nb02 == (size_t)ne01 * nb01) && \ + (nb03 == (size_t)ne02 * nb02); \ + const bool dst_contig = (nb2 == (size_t)ne1 * nb1) && \ + (nb3 == (size_t)ne2 * nb2); \ + \ + const uint32_t total_tiles = (src0_end_row - src0_start_row) * tiles_per_row; \ + \ + for (uint32_t t = 0, vtcm_idx = 0; t < total_tiles && vtcm_idx < 2; t++, vtcm_idx++) { \ + const uint32_t row = src0_start_row + t / tiles_per_row; \ + const uint32_t col = (t % tiles_per_row) * col_tile; \ + const uint32_t tw = MIN(col_tile, ne0 - col); \ + const size_t tb = (size_t) tw * sizeof(float); \ + const size_t soff = (src0_contig ? (row * nb01) : \ + unary_row_offset(row, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03)) + \ + (size_t) col * sizeof(float); \ + \ + dma_queue_push(dmaq, dma_make_ptr(data_dst, dst_vtcm_data + (vtcm_idx * dst_half)), 0, 0, 0, 0); \ + dma_queue_push(dmaq, dma_make_ptr(src0_vtcm_data + (vtcm_idx * src0_half), data_src + soff), tb, tb, tb, 1); \ + } \ + \ + uint32_t row = src0_start_row; \ + uint32_t col = 0; \ + uint32_t tile_in_row = 0; \ + uint32_t i01 = fastmodulo(row, ne01, div_ne01); \ + \ + uint32_t prow = src0_start_row + fastdiv(2, div_tpr); \ + uint32_t pcol = fastmodulo(2, tiles_per_row, div_tpr) * col_tile; \ + uint32_t ptile_in_row = fastmodulo(2, tiles_per_row, div_tpr); \ + \ + for (uint32_t t = 0; t < total_tiles; t++) { \ + uint8_t * dst_vtcm = (uint8_t *) dma_queue_pop(dmaq).src; \ + uint8_t * src_vtcm = (uint8_t *) dma_queue_pop(dmaq).dst; \ + \ + const uint32_t tw = MIN(col_tile, ne0 - col); \ + \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, t); \ + CORE_TILE_EXPR; \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, t); \ + \ + const size_t doff = (dst_contig ? (row * nb1) : \ + unary_row_offset(row, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3)) + \ + (size_t) col * sizeof(float); \ + const size_t tb = (size_t) tw * sizeof(float); \ + dma_queue_push(dmaq, dma_make_ptr(data_dst + doff, dst_vtcm), tb, tb, tb, 1); \ + \ + const uint32_t pt = t + 2; \ + if (pt < total_tiles) { \ + const uint32_t ptw = MIN(col_tile, ne0 - pcol); \ + const size_t ptb = (size_t) ptw * sizeof(float); \ + const size_t psoff = (src0_contig ? (prow * nb01) : \ + unary_row_offset(prow, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, \ + nb03)) + \ + (size_t) pcol * sizeof(float); \ + dma_queue_push(dmaq, dma_make_ptr(src_vtcm, data_src + psoff), ptb, ptb, ptb, 1); \ + } \ + \ + tile_in_row++; \ + col += col_tile; \ + if (tile_in_row == tiles_per_row) { \ + tile_in_row = 0; \ + col = 0; \ + row++; \ + i01++; \ + if (i01 == ne01) { \ + i01 = 0; \ + } \ + } \ + \ + ptile_in_row++; \ + pcol += col_tile; \ + if (ptile_in_row == tiles_per_row) { \ + ptile_in_row = 0; \ + pcol = 0; \ + prow++; \ + } \ + } \ + \ + dma_queue_flush(dmaq); \ } static inline void tile_scale_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw, const int32_t * op_params) { @@ -1146,14 +1149,32 @@ static int execute_op_unary(struct htp_ops_context * octx) { const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params; + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = kparams->n_threads; - const size_t elem_size = is_f16 ? sizeof(_Float16) : sizeof(float); - const size_t src0_data_row_size = src0->ne[0] * elem_size; const size_t dst_data_row_size = dst->ne[0] * elem_size; + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) elem_size, (uint32_t) dst_data_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + const size_t src0_row_size_aligned = kparams->src0_row_size_aligned; const size_t dst_row_size_aligned = kparams->dst_row_size_aligned; @@ -1191,8 +1212,9 @@ static int execute_op_unary(struct htp_ops_context * octx) { struct htp_unary_context uctx = { .octx = octx, .kparams = kparams, - .src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads, - .src0_nrows = src0_nrows, + .src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .src0_nrows = nrows, + .row_start = row_start, .data_src0 = (const uint8_t *)src0->data, .data_src1 = (octx->op == HTP_OP_RMS_NORM_MUL) ? (const uint8_t *)src1->data : NULL, @@ -1287,7 +1309,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { } if (task_func) { - worker_pool_run_func(octx->ctx->worker_pool, task_func, &uctx, n_threads); + work_queue_run(octx->ctx->work_queue, task_func, &uctx, n_threads); } else { FARF(ERROR, "execute_op_unary: task function is NULL for op %d\n", octx->op); err = HTP_STATUS_NO_SUPPORT; diff --git a/scripts/snapdragon/ggml-hexagon-align-macros.py b/scripts/snapdragon/ggml-hexagon-align-macros.py new file mode 100755 index 000000000..b64db3e65 --- /dev/null +++ b/scripts/snapdragon/ggml-hexagon-align-macros.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +""" +align-macros.py - Inspect and align trailing backslashes in multiline C/C++ macros. + +Usage: + align-macros.py [paths...] # Check and report misaligned macros + align-macros.py --diff [paths...] # Show unified diff of fixes + align-macros.py --fix [paths...] # Fix misaligned macros in-place + align-macros.py --fix --mode majority ... # Align to the dominant column + align-macros.py --fix --pad 2 ... # Align to (max_content_len + pad) + +Safety rules: + - Macros that are ALREADY aligned are NEVER touched (unless --all is given). + - Whitespace after trailing backslashes is flagged and cleaned. +""" + +import argparse +import difflib +import logging +import os +import re +import sys +from collections import Counter +from typing import List, Optional, Tuple, NamedTuple + +logger = logging.getLogger("ggml-hexagon-align-macros") + + +class MacroLine(NamedTuple): + line_num: int # 1-indexed + raw: str # Original line including newline + content: str # Line content before trailing backslash (stripped of trailing whitespace) + bs_col: Optional[int] # 1-indexed column of backslash, or None if last line has no backslash + trailing_ws: bool # True if whitespace existed after the backslash + + +class MacroDef(NamedTuple): + name: str + filepath: str + start_line: int + end_line: int + lines: List[MacroLine] + + +def parse_macros(filepath: str) -> List[MacroDef]: + """Extract all multiline macros from a C/C++ source file.""" + try: + with open(filepath, "r", encoding="utf-8", errors="replace") as f: + lines = f.readlines() + except Exception as e: + logger.error(f"Error reading {filepath}: {e}") + return [] + + macros: List[MacroDef] = [] + i = 0 + n = len(lines) + + while i < n: + line = lines[i] + m = re.match(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z0-9_]*)", line) + if m: + macro_name = m.group(1) + macro_start = i + 1 + macro_lines: List[MacroLine] = [] + cur = i + + while cur < n: + l_raw = lines[cur] + l_rstrip = l_raw.rstrip("\r\n") + + # Check if line has a trailing backslash + # Note: handle possible accidental spaces after backslash + match_bs = re.search(r"\\([ \t]*)$", l_rstrip) + if match_bs: + has_trailing_ws = len(match_bs.group(1)) > 0 + bs_index = match_bs.start() + content = l_rstrip[:bs_index].rstrip() + # 1-indexed column of the backslash + bs_col = bs_index + 1 + macro_lines.append(MacroLine( + line_num=cur + 1, + raw=l_raw, + content=content, + bs_col=bs_col, + trailing_ws=has_trailing_ws + )) + cur += 1 + else: + # Line does not end with backslash + if cur == i: + # Single-line macro, not multiline + break + else: + # Final line of a multiline macro + macro_lines.append(MacroLine( + line_num=cur + 1, + raw=l_raw, + content=l_rstrip.rstrip(), + bs_col=None, + trailing_ws=False + )) + break + + # Only record if it is a multiline macro (has at least one continuation line) + continuation_lines = [ml for ml in macro_lines if ml.bs_col is not None] + if continuation_lines: + macro_end = macro_lines[-1].line_num + macros.append(MacroDef( + name=macro_name, + filepath=filepath, + start_line=macro_start, + end_line=macro_end, + lines=macro_lines + )) + i = cur + i += 1 + + return macros + + +def is_macro_aligned(macro: MacroDef) -> bool: + """A macro is aligned if all continuation lines have backslashes at the same column.""" + bs_cols = [ml.bs_col for ml in macro.lines if ml.bs_col is not None] + if not bs_cols: + return True + has_trailing_ws = any(ml.trailing_ws for ml in macro.lines) + return len(set(bs_cols)) == 1 and not has_trailing_ws + + +def compute_target_column(macro: MacroDef, mode: str, pad: int, target_col: Optional[int]) -> int: + """Determine the column where backslashes should be aligned.""" + max_content_len = max(len(ml.content) for ml in macro.lines) + min_needed = max_content_len + pad + + if target_col is not None: + return max(target_col, min_needed) + + bs_cols = [ml.bs_col for ml in macro.lines if ml.bs_col is not None] + if not bs_cols: + return min_needed + + if mode == "min": + return min_needed + elif mode == "max": + return max(max(bs_cols), min_needed) + elif mode == "majority": + counts = Counter(bs_cols) + # Sort by frequency descending, then by column descending + majority_col = sorted(counts.items(), key=lambda x: (-x[1], -x[0]))[0][0] + return max(majority_col, min_needed) + else: + return min_needed + + +def realign_macro_lines(macro: MacroDef, target_col: int) -> List[str]: + """Format macro lines with backslashes aligned at target_col.""" + new_lines: List[str] = [] + for ml in macro.lines: + nl = "\r\n" if ml.raw.endswith("\r\n") else "\n" + if ml.bs_col is None: + # Last line without backslash + new_lines.append(ml.raw) + else: + if not ml.content: + spaces = " " * (target_col - 1) + new_lines.append(f"{spaces}\\{nl}") + else: + spaces_needed = max(1, target_col - len(ml.content) - 1) + new_lines.append(f"{ml.content}{' ' * spaces_needed}\\{nl}") + return new_lines + + +def process_file(filepath: str, args: argparse.Namespace) -> Tuple[int, int, Optional[str]]: + macros = parse_macros(filepath) + if not macros: + return 0, 0, None + + with open(filepath, "r", encoding="utf-8", errors="replace") as f: + file_lines = f.readlines() + + misaligned_count = 0 + modified = False + new_file_lines = list(file_lines) + + for macro in macros: + aligned = is_macro_aligned(macro) + if not aligned or args.all: + if not aligned: + misaligned_count += 1 + + bs_cols = [ml.bs_col for ml in macro.lines if ml.bs_col is not None] + max_content = max(len(ml.content) for ml in macro.lines) + col_counts = Counter(bs_cols) + + if not args.quiet: + logger.info(f"{filepath}:{macro.start_line}-{macro.end_line} [{macro.name}]") + logger.info(f" Max content width: {max_content}, Min needed column (+{args.pad}): {max_content + args.pad}") + logger.info(f" Current backslash columns: {dict(sorted(col_counts.items()))}") + trailing_ws_lines = [ml.line_num for ml in macro.lines if ml.trailing_ws] + if trailing_ws_lines: + logger.warning(f" Warning: Trailing whitespace after backslash on line(s): {trailing_ws_lines}") + + target_col = compute_target_column(macro, args.mode, args.pad, args.target_col) + if not args.quiet: + logger.info(f" -> Target alignment column: {target_col}") + + realigned = realign_macro_lines(macro, target_col) + + start_idx = macro.start_line - 1 + end_idx = start_idx + len(macro.lines) + if new_file_lines[start_idx:end_idx] != realigned: + new_file_lines[start_idx:end_idx] = realigned + modified = True + + diff_text = None + if modified: + diff = difflib.unified_diff( + file_lines, + new_file_lines, + fromfile=f"a/{filepath}", + tofile=f"b/{filepath}", + lineterm="" + ) + diff_text = "\n".join(diff) + + if args.fix: + with open(filepath, "w", encoding="utf-8") as f: + f.writelines(new_file_lines) + if not args.quiet: + logger.info(f" [FIXED] Updated {filepath}") + + return len(macros), misaligned_count, diff_text + + +def find_source_files(paths: List[str]) -> List[str]: + extensions = {".c", ".cpp", ".cc", ".cxx", ".h", ".hpp", ".inl"} + result: List[str] = [] + for p in paths: + if os.path.isfile(p): + result.append(p) + elif os.path.isdir(p): + for root, _, files in os.walk(p): + for file in sorted(files): + _, ext = os.path.splitext(file) + if ext.lower() in extensions: + result.append(os.path.join(root, file)) + return sorted(result) + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(message)s") + parser = argparse.ArgumentParser( + description="Inspect and align backslashes in multiline C/C++ macros." + ) + parser.add_argument("paths", nargs="*", default=["."], help="Files or directories to scan (default: current dir)") + parser.add_argument("--fix", action="store_true", help="Fix misaligned macros in-place") + parser.add_argument("--diff", action="store_true", help="Display unified diff of suggested fixes") + parser.add_argument("--check", action="store_true", help="Exit with code 1 if misaligned macros exist") + parser.add_argument("--mode", choices=["min", "max", "majority"], default="min", + help="Alignment mode: 'min' (max_len + pad), 'max' (max existing col), 'majority' (dominant col)") + parser.add_argument("--pad", type=int, default=2, help="Spaces between longest line and backslash (default: 2)") + parser.add_argument("--target-col", type=int, default=None, help="Force alignment to an exact column") + parser.add_argument("--all", action="store_true", help="Realign all macros even if already aligned (default: only misaligned)") + parser.add_argument("-q", "--quiet", action="store_true", help="Only output errors and diffs/summary") + + args = parser.parse_args() + + files = find_source_files(args.paths) + if not files: + logger.error("No C/C++ source files found.") + sys.exit(0) + + total_macros = 0 + total_misaligned = 0 + diffs: List[str] = [] + + for filepath in files: + num_macros, num_misaligned, diff_text = process_file(filepath, args) + total_macros += num_macros + total_misaligned += num_misaligned + if diff_text: + diffs.append(diff_text) + + if args.diff and diffs: + logger.info("\n--- Proposed Changes ---\n") + for d in diffs: + logger.info(d) + + logger.info(f"\nSummary: scanned {len(files)} files, {total_macros} multiline macros, {total_misaligned} misaligned.") + + if args.check and total_misaligned > 0: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/snapdragon/run.py b/scripts/snapdragon/run.py index 81eecd2e0..dc71d4a32 100755 --- a/scripts/snapdragon/run.py +++ b/scripts/snapdragon/run.py @@ -14,6 +14,42 @@ import logging logger = logging.getLogger("run") +MANAGED_ENV_NAMES = ( + "GGML_HEXAGON_DEVICES", + "GGML_HEXAGON_VERBOSE", + "GGML_HEXAGON_PROFILE", + "GGML_HEXAGON_NHVX", + "GGML_HEXAGON_NHMX", + "GGML_HEXAGON_HOSTBUF", + "GGML_HEXAGON_OPBATCH", + "GGML_HEXAGON_OPQUEUE", + "GGML_HEXAGON_OPPOLL", + "GGML_HEXAGON_OPFILTER", + "GGML_HEXAGON_OPFUSION", + "GGML_HEXAGON_VMEM", + "GGML_HEXAGON_MBUF", + "GGML_HEXAGON_MM_SELECT", + "GGML_HEXAGON_FA_SELECT", + "GGML_HEXAGON_AR_SELECT", + "GGML_HEXAGON_ETM", + "GGML_HEXAGON_ARCH", + "GGML_HEXAGON_OPTRACE", + "GGML_OPENCL_PLATFORM", + "GGML_OPENCL_DEVICE", + "GGML_OPENCL_OPFILTER", + "GGML_OPENCL_KERNEL_CACHE_DIR", + "GGML_OPENCL_KERNEL_CACHE_DEBUG", + "GGML_OPENCL_FA_TUNE", + "GGML_OPENCL_DISABLE_FUSION", + "GGML_OPENCL_ADRENO_XMEM_GEMM", + "GGML_OPENCL_ADRENO_USE_LARGE_BUFFER", + "GGML_SCHED_DEBUG", + "MTMD_BACKEND_DEVICE", + "D", + "DEVICE", +) + + def parse_target(target_str): if not target_str: return None, None @@ -38,6 +74,57 @@ def shlex_join(args_list): return " ".join(pipes.quote(x) for x in args_list) +def split_device_list(devices): + parts = [] + curr = [] + bracket_depth = 0 + + for ch in devices: + if ch == '[': + bracket_depth += 1 + curr.append(ch) + elif ch == ']': + if bracket_depth > 0: + bracket_depth -= 1 + curr.append(ch) + elif ch == ',' and bracket_depth == 0: + part = "".join(curr).strip() + if part: + parts.append(part) + curr = [] + else: + curr.append(ch) + + part = "".join(curr).strip() + if part: + parts.append(part) + + return parts + + +def device_arg_from_devices(devices): + if devices.isdigit(): + n = int(devices) + return ",".join(f"HTP{i}" for i in range(n)) + + names = [] + for part in split_device_list(devices): + if "[" in part: + part = part.split("[", 1)[0].strip() + if part: + names.append(part) + + return ",".join(names) + + +def normalize_cmd_device_args(cmd_args): + for i, arg in enumerate(cmd_args): + if arg == "--device" and i + 1 < len(cmd_args): + cmd_args[i + 1] = device_arg_from_devices(cmd_args[i + 1]) + elif arg.startswith("--device="): + cmd_args[i] = "--device=" + device_arg_from_devices(arg.split("=", 1)[1]) + + def main(): logging.basicConfig(level=logging.INFO, format='%(message)s') # Split arguments at '--' @@ -142,8 +229,6 @@ def main(): def set_env(env_name, opt_val): if opt_val is not None: env_vars[env_name] = str(opt_val) - elif env_name in os.environ: - env_vars[env_name] = os.environ[env_name] # Resolve and filter devices (HTP vs OpenCL) device_in_cmd = None @@ -166,7 +251,7 @@ def main(): hex_devices = devices_val cl_device = "" else: - parts = [p.strip() for p in devices_val.split(",")] + parts = split_device_list(devices_val) # Any device containing "htp" is Hexagon, rest is OpenCL hex_parts = [p for p in parts if "htp" in p.lower()] cl_parts = [ @@ -181,15 +266,13 @@ def main(): # Set Hexagon devices if hex_devices: env_vars["GGML_HEXAGON_DEVICES"] = hex_devices - elif "GGML_HEXAGON_DEVICES" in os.environ: - env_vars["GGML_HEXAGON_DEVICES"] = os.environ["GGML_HEXAGON_DEVICES"] + + normalize_cmd_device_args(cmd_args) # Set OpenCL device (unless overridden by --cl-device) final_cl_device = args.cl_device if args.cl_device is not None else cl_device if final_cl_device: env_vars["GGML_OPENCL_DEVICE"] = final_cl_device - elif "GGML_OPENCL_DEVICE" in os.environ: - env_vars["GGML_OPENCL_DEVICE"] = os.environ["GGML_OPENCL_DEVICE"] # Map shared & backend-specific parameters with correct overrides @@ -206,8 +289,6 @@ def main(): if args.cl_fa_tune or args.profile is not None: env_vars["GGML_OPENCL_FA_TUNE"] = "1" - elif "GGML_OPENCL_FA_TUNE" in os.environ: - env_vars["GGML_OPENCL_FA_TUNE"] = os.environ["GGML_OPENCL_FA_TUNE"] # Other Hexagon environment variables set_env("GGML_HEXAGON_NHVX", args.hex_nhvx) @@ -235,18 +316,12 @@ def main(): if args.cl_disable_fusion: env_vars["GGML_OPENCL_DISABLE_FUSION"] = "1" - elif "GGML_OPENCL_DISABLE_FUSION" in os.environ: - env_vars["GGML_OPENCL_DISABLE_FUSION"] = os.environ["GGML_OPENCL_DISABLE_FUSION"] if args.cl_adreno_xmem: env_vars["GGML_OPENCL_ADRENO_XMEM_GEMM"] = "1" - elif "GGML_OPENCL_ADRENO_XMEM_GEMM" in os.environ: - env_vars["GGML_OPENCL_ADRENO_XMEM_GEMM"] = os.environ["GGML_OPENCL_ADRENO_XMEM_GEMM"] if args.cl_adreno_large_buffer: env_vars["GGML_OPENCL_ADRENO_USE_LARGE_BUFFER"] = "1" - elif "GGML_OPENCL_ADRENO_USE_LARGE_BUFFER" in os.environ: - env_vars["GGML_OPENCL_ADRENO_USE_LARGE_BUFFER"] = os.environ["GGML_OPENCL_ADRENO_USE_LARGE_BUFFER"] if args.sched_debug: env_vars["GGML_SCHED_DEBUG"] = "2" @@ -288,15 +363,7 @@ def main(): has_b = any(arg == "-b" for arg in cmd_args) if not has_b: if args.devices: - if args.devices.isdigit(): - n = int(args.devices) - device_val = ",".join(f"HTP{i}" for i in range(n)) - else: - device_val = args.devices - elif "D" in os.environ: - device_val = os.environ["D"] - elif "DEVICE" in os.environ: - device_val = os.environ["DEVICE"] + device_val = device_arg_from_devices(args.devices) else: device_val = "HTP0" if device_val: @@ -305,17 +372,10 @@ def main(): has_device = any(arg.startswith("--device") for arg in cmd_args) if not has_device: if args.devices: - if args.devices.isdigit(): - n = int(args.devices) - device_val = ",".join(f"HTP{i}" for i in range(n)) - else: - device_val = args.devices - elif "D" in os.environ: - device_val = os.environ["D"] - elif "DEVICE" in os.environ: - device_val = os.environ["DEVICE"] + device_val = device_arg_from_devices(args.devices) else: device_val = "HTP0" + if device_val: cmd_args += ["--device", device_val] @@ -415,6 +475,8 @@ def main(): else: local_env["LD_LIBRARY_PATH"] = lib_dir + os.path.pathsep + local_env.get("LD_LIBRARY_PATH", "") + for k in MANAGED_ENV_NAMES: + local_env.pop(k, None) for k, v in env_vars.items(): local_env[k] = v From 3f5e94d7c2ab2267fe39852051777fe30c1f49ef Mon Sep 17 00:00:00 2001 From: Pascal Date: Sat, 12 Sep 2026 06:40:21 +0200 Subject: [PATCH 23/35] webgpu: align tensor bindings to the type block size (#28382) Walk the binding offset back until the distance to the tensor is a whole number of blocks, so block quantized views get a valid element offset in the shader. --- ggml/src/ggml-webgpu/ggml-webgpu.cpp | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp index 13db0b856..8b060c41a 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp @@ -374,20 +374,28 @@ static wgpu::Buffer ggml_webgpu_tensor_buf(const ggml_tensor * tensor) { return ctx->buffer; } +// Binding offset for a tensor: the largest aligned offset at or before the tensor whose +// distance to the tensor is a whole number of type blocks, so shaders can index the +// misalignment in elements even for block quantized types. +static size_t ggml_webgpu_tensor_align_offset(const ggml_tensor * t, size_t alignment) { + const size_t offset = ggml_webgpu_tensor_offset(t); + const size_t type_size = ggml_type_size(t->type); + size_t aligned = offset & ~(alignment - 1); + while ((offset - aligned) % type_size != 0) { + GGML_ASSERT(aligned >= alignment); + aligned -= alignment; + } + return aligned; +} + static size_t ggml_webgpu_tensor_misalignment(const ggml_tensor * t, size_t alignment) { - size_t offset = ggml_webgpu_tensor_offset(t); - return offset & (alignment - 1); + return ggml_webgpu_tensor_offset(t) - ggml_webgpu_tensor_align_offset(t, alignment); } static size_t ggml_webgpu_tensor_misalignment(webgpu_context & ctx, const ggml_tensor * t) { return ggml_webgpu_tensor_misalignment(t, ctx->global_ctx->capabilities.limits.minStorageBufferOffsetAlignment); } -static size_t ggml_webgpu_tensor_align_offset(const ggml_tensor * t, size_t alignment) { - size_t offset = ggml_webgpu_tensor_offset(t); - return offset & ~(alignment - 1); -} - static size_t ggml_webgpu_tensor_align_offset(webgpu_context & ctx, const ggml_tensor * t) { return ggml_webgpu_tensor_align_offset(t, ctx->global_ctx->capabilities.limits.minStorageBufferOffsetAlignment); } From 07fc97716fa3dab457797fcfe51705034ce8957a Mon Sep 17 00:00:00 2001 From: shaofeiqi Date: Fri, 11 Sep 2026 22:10:08 -0700 Subject: [PATCH 24/35] opencl: add bin kernel `kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin` (#28677) * opencl: add A8 Q4_K non-MoE binary kernel * opencl: fix layout compatibility * opencl: rename binary kernel selection helpers --------- Co-authored-by: Li He --- ggml/src/ggml-opencl/CMakeLists.txt | 1 + ggml/src/ggml-opencl/ggml-opencl.cpp | 249 +++++++++++++++++- .../gemv_noshuffle_q4_k_f32_32b_trans.cl | 134 ++++++++++ 3 files changed, 376 insertions(+), 8 deletions(-) create mode 100644 ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_32b_trans.cl diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 716577bb7..45a7075b2 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -185,6 +185,7 @@ set(GGML_OPENCL_KERNELS gemv_noshuffle_q4_k_f32_o4 gemv_noshuffle_q4_k_f32_tiled gemm_noshuffle_q4_k_f32 + gemv_noshuffle_q4_k_f32_32b_trans gemv_noshuffle_q6_k_f32 gemv_noshuffle_q6_k_f32_o4 gemv_noshuffle_q6_k_f32_tiled diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 231be2cf3..d6820b37d 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -1185,6 +1185,8 @@ struct ggml_backend_opencl_context { cl_kernel kernel_convert_block_q4_k_tiled_ns; // tiled-wide convert (opt-in) cl_kernel kernel_gemv_noshuffle_q4_k_f32_mc3; // multi-column (N=3) verify GEMV cl_kernel kernel_gemm_noshuffle_q4_k_f32; + cl_kernel kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin; + cl_kernel kernel_gemv_noshuffle_q4_k_f32_32b_trans; cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) dense prefill GEMM cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = nullptr; // dp4a dense prefill GEMM, weights via texture (X1 opt-in) cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q5_K prefill GEMM @@ -4260,6 +4262,43 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans = nullptr; + backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin = nullptr; + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E) { + { + std::string opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q4_k_f32_32b_trans.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32_32b_trans.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans = + clCreateKernel(prog, "gemv_noshuffle_q4_k_f32_32b_trans", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + if (use_adreno_bin_kernels(backend_ctx)) { + size_t bin_size = 0; + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_k_f32_32b_trans_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } + } + } + std::string CL_moe_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable " " -cl-fast-relaxed-math"; @@ -7722,6 +7761,7 @@ static void ggml_cl_moe_combine_fused(ggml_backend_t backend, const ggml_tensor } inline bool use_q4k_tiled(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); // defined below (used by the GLU-subgraph fuse check) +inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); // defined below static bool ggml_opencl_can_fuse(const ggml_backend_opencl_context * backend_ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list ops) { @@ -7776,6 +7816,10 @@ static bool ggml_opencl_can_fuse(const ggml_backend_opencl_context * backend_ctx if (use_q4k_tiled(backend_ctx, gate->src[0]) || use_q4k_tiled(backend_ctx, up->src[0])) { return false; } + // q4_K bin kernel requires 32b transposed layout, not compatible with the fused gemv + if (use_q4_k_bin_kernels(backend_ctx, gate->src[0]) || use_q4_k_bin_kernels(backend_ctx, up->src[0])) { + return false; + } // that noshuffle layout is only produced at set_tensor time when // use_adreno_kernels() accepts the weight (ne0 >= 512 && ne1 >= 512). // Smaller weights stay in the plain q4_K layout, which this kernel would @@ -8349,7 +8393,7 @@ inline bool enable_adreno_trans_weight_q5_K(const ggml_backend_opencl_context *b qh_img_width <= backend_ctx->image_max_buffer_size; } -inline bool use_q4_0_ila_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { +inline bool use_q4_0_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (!backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans || !backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin) { @@ -8442,6 +8486,21 @@ static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_backend_opencl_cont && tensor->ne[2] == 1 && tensor->ne[3] == 1; } +inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (!backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans || + !backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin) { + return false; + } + return (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0) && + !use_q4k_tiled(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *)dev->context; ggml_backend_opencl_context * backend_ctx = dev_ctx->backend_ctx; @@ -9625,7 +9684,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(K % 32 == 0); - if (use_q4_0_ila_kernels(backend_ctx, tensor)) { + if (use_q4_0_bin_kernels(backend_ctx, tensor)) { cl_int err; cl_image_format wimg_fmt; cl_image_desc wimg_desc; @@ -10576,8 +10635,25 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(K % 32 == 0); - // Transpose q, d, dm as ushort - transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + if (use_q4_k_bin_kernels(backend_ctx, tensor)) { + cl_int err; + cl_image_format wimg_fmt; + cl_image_desc wimg_desc; + + // transpose quants as 32-bit words (M-first) + GGML_ASSERT(M % 64 == 0); + transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/8, M); + + wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = (size_t)M * K / 8; + wimg_desc.buffer = extra->q; + CL_CHECK((extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + } else { + // Transpose q as ushort + transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + } transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/256, M); transpose_2d_as_16b(backend_ctx, extra->dm, extra->dm, size_dm, K/256, M); @@ -11180,7 +11256,7 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, buf_trans_d.allocate(backend_ctx->context, size_d); buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); - if (use_q4_0_ila_kernels(backend_ctx, tensor)) { + if (use_q4_0_bin_kernels(backend_ctx, tensor)) { transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K / 8); } else { transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K / 4); @@ -11855,7 +11931,11 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, buf_trans_s.allocate(backend_ctx->context, size_s); // Transpose q, d, dm, s back - transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); + if (use_q4_k_bin_kernels(backend_ctx, tensor)) { + transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/8); + } else { + transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); + } transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/256); transpose_2d_as_16b(backend_ctx, extra->dm, buf_trans_dm.buffer, size_dm, M, K/256); transpose_2d_as_8b (backend_ctx, extra->s, buf_trans_s.buffer, size_s, M, K/256*12, true, true); @@ -18639,9 +18719,9 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t static const bool q40_mc3 = (getenv("GGML_OPENCL_Q40_MC3") != nullptr); const bool use_q40_mc3 = q40_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); - const bool use_ila = use_q4_0_ila_kernels(backend_ctx, src0); + const bool use_bin = use_q4_0_bin_kernels(backend_ctx, src0); - if (use_ila) { + if (use_bin) { if (use_q40_mc3) { static bool warned = false; if (!warned) { @@ -20196,6 +20276,145 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t #endif } +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_mul_mat_q4_k_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q4_K * extra0_q4_k = (ggml_tensor_extra_cl_q4_K *)src0->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + + const int ne1 = dst->ne[1]; + + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); + + cl_context context = backend_ctx->context; + cl_kernel kernel; + + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; + + int M = ne01; + int N = ne1; + int K = ne00; + + if (ne1 == 1) { + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; + + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_RGBA, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); + + size_t local_work_size[3] = { 64, 8, 1 }; + size_t global_work_size[3] = { (size_t)ne01, 8, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + } else { + const int gemm_tile_n = 64; + int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n; + + cl_mem b_sub_buf = nullptr; + cl_mem b_padded = nullptr; + cl_mem b_buf = nullptr; + if (N_pad == N) { + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + b_buf = b_sub_buf; + } else { + CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, (size_t)K * N_pad * sizeof(float), NULL, &err), err)); + const float zero = 0.0f; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); + b_buf = b_padded; + } + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N_pad; + img_desc.buffer = b_buf; + cl_mem b_img; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + cl_mem d_sub_buf; + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub_buf; + cl_mem d_img; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uint), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &N)); + + size_t local_work_size[3] = { 64, 2, 2 }; + size_t m_tiles = (size_t)CEIL_DIV(M, 64); + size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_img)); + if (b_sub_buf) { + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } + if (b_padded) { + CL_CHECK(clReleaseMemObject(b_padded)); + } + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); + } +} +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); @@ -20248,6 +20467,20 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t // unified routes batched Q6_K lm_head to CPU). Per-layer mc3 is byte-identical. const bool use_mc3 = q4k_mc3 && (ne1 == 3) && (ne01 < 32768); + const bool use_bin = use_q4_k_bin_kernels(backend_ctx, src0); + + if (use_bin) { + if (use_mc3) { + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q4K_MC3 is bypassed by Q4_K binary kernels\n"); + warned = true; + } + } + ggml_cl_mul_mat_q4_k_f32_adreno_ila(backend, src0, src1, dst); + return; + } + if (ne1 == 1 || use_mc3) { cl_mem q_img = nullptr; cl_mem b_sub_buf = nullptr; diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_32b_trans.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_32b_trans.cl new file mode 100644 index 000000000..2dbd943fd --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_32b_trans.cl @@ -0,0 +1,134 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#define QK_K 256 +#define K_SCALE_SIZE 12 +#define N_SIMDGROUP 8 +#define SIMDGROUP_WIDTH 64 + +inline void get_scale_min_k4( + int j, + global const uchar * q, + uint stride, + uchar * d, + uchar * m +) { + if (j < 4) { + *d = q[j*stride] & 63; + *m = q[(j+4)*stride] & 63; + } else { + *d = (q[(j+4)*stride] & 0x0F) | ((q[(j-4)*stride] & 0xC0) >> 2); + *m = ((q[(j+4)*stride] >> 4) & 0x0F) | ((q[j*stride] & 0xC0) >> 2); + } +} + +static inline float8 q4_k_to_fp32_packed8(ushort2 q4x8, float scale, float minv) { + float8 fp32x8; + fp32x8.s0 = (q4x8.s0 & 0x000F) * scale - minv; + fp32x8.s1 = ((q4x8.s0 & 0x00F0) >> 4) * scale - minv; + fp32x8.s2 = ((q4x8.s0 & 0x0F00) >> 8) * scale - minv; + fp32x8.s3 = ((q4x8.s0 & 0xF000) >> 12) * scale - minv; + fp32x8.s4 = (q4x8.s1 & 0x000F) * scale - minv; + fp32x8.s5 = ((q4x8.s1 & 0x00F0) >> 4) * scale - minv; + fp32x8.s6 = ((q4x8.s1 & 0x0F00) >> 8) * scale - minv; + fp32x8.s7 = ((q4x8.s1 & 0xF000) >> 12) * scale - minv; + return fp32x8; +} + +__attribute__((qcom_reqd_sub_group_size("half"))) +__kernel void gemv_noshuffle_q4_k_f32_32b_trans( + read_only image1d_buffer_t src0_q, + __global half * src0_d, + __global half * src0_dm, + __global uchar * src0_s, + __read_only image1d_buffer_t src1, + __global float * dst, + ulong offsetd, + int ne00, + int ne01 +) { + uint i01 = get_global_id(0); + uint sgid = get_local_id(1); + uint slid = get_sub_group_local_id(); + + int num_subblocks = ne00 / 32; + + __private float sum = 0.0f; + + // Loop over sub-blocks of 32 elements, N_SIMDGROUP sub-blocks per iter + for (uint ib = sgid; ib < num_subblocks; ib += N_SIMDGROUP) { + uint sb = ib / 8; + uint j = ib % 8; + + // Load d and dmin for this super-block + half d_val = src0_d[sb * ne01 + i01]; + half dm_val = src0_dm[sb * ne01 + i01]; + + // Load sub-block scale and min. s is transposed [nb][12][M]; stride ne01 per code. + global const uchar * sc = src0_s + sb * K_SCALE_SIZE * ne01 + i01; + uchar sv, mn; + get_scale_min_k4(j, sc, ne01, &sv, &mn); + + float scale = (float)d_val * (float)sv; + float minv = (float)dm_val * (float)mn; + + // Load 4 uints of quants (32 nibbles = 32 elements), column-major stride ne01 + uint q_base = ib * ne01 * 4 + i01; + + uint4 regQ; + regQ.s0 = read_imageui(src0_q, q_base).x; + regQ.s1 = read_imageui(src0_q, q_base + ne01).x; + regQ.s2 = read_imageui(src0_q, q_base + ne01 * 2).x; + regQ.s3 = read_imageui(src0_q, q_base + ne01 * 3).x; + + // Load activations: 32 floats = 8 float4s + uint y_offset = ib * 8; + + float4 y_local = (slid < 8) ? read_imagef(src1, (y_offset + slid)) : (float4)0.0f; + float4 y0 = sub_group_broadcast(y_local, 0); + float4 y1 = sub_group_broadcast(y_local, 1); + float4 y2 = sub_group_broadcast(y_local, 2); + float4 y3 = sub_group_broadcast(y_local, 3); + float4 y4 = sub_group_broadcast(y_local, 4); + float4 y5 = sub_group_broadcast(y_local, 5); + float4 y6 = sub_group_broadcast(y_local, 6); + float4 y7 = sub_group_broadcast(y_local, 7); + + float8 fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s0), scale, minv); + float4 acc = y0 * fp32x8.lo; + acc += y1 * fp32x8.hi; + + fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s1), scale, minv); + acc += y2 * fp32x8.lo; + acc += y3 * fp32x8.hi; + + fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s2), scale, minv); + acc += y4 * fp32x8.lo; + acc += y5 * fp32x8.hi; + + fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s3), scale, minv); + acc += y6 * fp32x8.lo; + acc += y7 * fp32x8.hi; + + sum += ((acc.s0 + acc.s1) + (acc.s2 + acc.s3)); + } + + // reduction in local memory over N_SIMDGROUP subgroups + __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)]; + if (sgid > 0) { + reduceLM[SIMDGROUP_WIDTH * (sgid - 1) + slid] = sum; + } + barrier(CLK_LOCAL_MEM_FENCE); + if (sgid == 0) { + for (uint i = 0; i < N_SIMDGROUP - 1; ++i) { + sum += reduceLM[SIMDGROUP_WIDTH * i + slid]; + } + } + + // 1 output per thread in subgroup 0 + if (sgid == 0) { + dst = dst + (offsetd >> 2); + dst[i01] = sum; + } +} From 8a56aedd6143a014e25ec9f4295164f2f66ce30f Mon Sep 17 00:00:00 2001 From: Hongqiang Wang Date: Fri, 11 Sep 2026 22:11:12 -0700 Subject: [PATCH 25/35] opencl: fix several bugs where the backend aborts (#27630) --- ggml/src/ggml-opencl/ggml-opencl.cpp | 356 +++++++++++++++++---------- 1 file changed, 226 insertions(+), 130 deletions(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index d6820b37d..c107281a2 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -203,39 +203,67 @@ static ggml_cl_version get_opencl_platform_version(cl_platform_id platform) { return parse_cl_version(param_value); } +// Returns the DEVICE's OpenCL version. On an error returns ggml_cl_version with all zeroes. +static ggml_cl_version get_opencl_device_version(cl_device_id device) { + size_t param_size; + if (clGetDeviceInfo(device, CL_DEVICE_VERSION, 0, nullptr, ¶m_size) != CL_SUCCESS || !param_size) { + return {}; + } + std::unique_ptr param_storage(new char[param_size]); + if (clGetDeviceInfo(device, CL_DEVICE_VERSION, param_size, param_storage.get(), nullptr) != CL_SUCCESS) { + return {}; + } + + auto param_value = std::string_view(param_storage.get(), param_size); + const std::string version_prefix = "OpenCL "; // "OpenCL . " + if (param_value.find(version_prefix) != 0) { + return {}; + } + param_value.remove_prefix(version_prefix.length()); + return parse_cl_version(param_value); +} + // Return a version to use in OpenCL C compilation. On an error returns ggml_cl_version with all zeroes. static ggml_cl_version get_opencl_c_version(ggml_cl_version platform_version, cl_device_id device) { size_t param_size; #if CL_TARGET_OPENCL_VERSION >= 300 - if (platform_version.major >= 3) { - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, 0, nullptr, ¶m_size)); - if (!param_size) { - return {}; + // CL_DEVICE_OPENCL_C_ALL_VERSIONS is an OpenCL 3.0 *device* query, so gating it on the + // *platform* version is not enough: a 3.0 platform can expose 2.0 devices, where the + // query returns CL_INVALID_VALUE and the old CL_CHECK aborted during backend init. + // Gate on the device version, and treat a failure as "fall back to the legacy query" + // rather than fatal -- a device may advertise 3.0 and still refuse the property. + const ggml_cl_version device_version = get_opencl_device_version(device); + if (platform_version.major >= 3 && device_version.major >= 3) { + cl_int err = clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, 0, nullptr, ¶m_size); + if (err == CL_SUCCESS && param_size) { + std::unique_ptr versions(new cl_name_version[param_size]); + err = clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, param_size, versions.get(), nullptr); + if (err == CL_SUCCESS) { + unsigned versions_count = param_size / sizeof(cl_name_version); + + cl_version version_max = 0; + for (unsigned i = 0; i < versions_count; i++) { + version_max = std::max(versions[i].version, version_max); + } + + return { CL_VERSION_MAJOR(version_max), CL_VERSION_MINOR(version_max) }; + } } - - std::unique_ptr versions(new cl_name_version[param_size]); - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, param_size, versions.get(), nullptr)); - unsigned versions_count = param_size / sizeof(cl_name_version); - - cl_version version_max = 0; - for (unsigned i = 0; i < versions_count; i++) { - version_max = std::max(versions[i].version, version_max); - } - - return { CL_VERSION_MAJOR(version_max), CL_VERSION_MINOR(version_max) }; + // fall through to CL_DEVICE_OPENCL_C_VERSION below } #else GGML_UNUSED(platform_version); #endif // CL_TARGET_OPENCL_VERSION >= 300 - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, 0, nullptr, ¶m_size)); - if (!param_size) { + if (clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, 0, nullptr, ¶m_size) != CL_SUCCESS || !param_size) { return {}; } std::unique_ptr param_storage(new char[param_size]); - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, param_size, param_storage.get(), nullptr)); + if (clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, param_size, param_storage.get(), nullptr) != CL_SUCCESS) { + return {}; + } auto param_value = std::string_view(param_storage.get(), param_size); const std::string version_prefix = "OpenCL C "; // Suffix: "XX.YY " @@ -1115,6 +1143,18 @@ struct ggml_backend_opencl_context { } void enqueue_ndrange_kernel(cl_kernel kernel, cl_uint work_dim, size_t *global_work_size, size_t *local_work_size, const ggml_tensor * tensor) { + // From the spec on clEnqueueNDRangeKernel: + // If the device associated with command_queue is an OpenCL 2.1 or newer device, + // and global_work_size is NULL or the value in any passed dimension is zero, + // then the kernel command will trivially succeed after its event dependencies + // are satisfied and subsequently update its completion event. + // So this ensures such cases always return trivially without causing errors in + // case of an older device. + for (cl_uint i = 0; i < work_dim; i++) { + if (global_work_size[i] == 0) { + return; + } + } #ifdef GGML_OPENCL_PROFILING cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, work_dim, NULL, global_work_size, local_work_size, 0, NULL, &evt)); @@ -9459,6 +9499,96 @@ static enum ggml_status ggml_backend_opencl_buffer_init_tensor(ggml_backend_buff return GGML_STATUS_SUCCESS; } +// Allocate a temporary upload buffer of `nbytes` and populate it with `data` +// from host. On Adreno X1-85 the device-only pool intermittently fails to +// allocate at hundreds of MB once model weights fragment the heap (observed +// on Qwen3.5-9B output.weight Q6_K at 834 MB). Three-step retry: +// 1. CL_MEM_READ_WRITE alloc + clEnqueueWriteBuffer (normal fast path). +// 2. clFinish + retry (drains in-flight allocs that may be holding heap; +// mirrors the proven pattern at the FD-split partial buffer alloc). +// 3. CL_MEM_ALLOC_HOST_PTR + map(WRITE_INVALIDATE) + memcpy + unmap — +// different memory pool (host-pinned); true zero-copy on Adreno per +// QCOM guidance. (CL_MEM_USE_HOST_PTR is NOT zero-copy on Adreno: the +// driver triggers an internal copy because arbitrary host pages aren't +// guaranteed mappable/coherent, AND it draws from the same exhausted +// device pool — so it doesn't solve the problem.) +// Returns the ready-to-read buffer (caller must clReleaseMemObject) or NULL +// if all three strategies fail. The buffer is opaque to the caller — it can +// be passed as a kernel argument like any normal cl_mem. +static cl_mem ggml_cl_create_temp_upload_buffer( + cl_context context, cl_command_queue queue, + size_t nbytes, const void * data, + const char * tensor_name_for_log) +{ + cl_int err; + cl_mem buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); + if (err != CL_SUCCESS) { + clFinish(queue); + buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); + } + if (err == CL_SUCCESS) { + const cl_int werr = clEnqueueWriteBuffer(queue, buf, CL_TRUE, 0, nbytes, data, 0, NULL, NULL); + if (werr == CL_SUCCESS) { + return buf; + } + clReleaseMemObject(buf); + } + buf = clCreateBuffer(context, + CL_MEM_READ_ONLY | CL_MEM_ALLOC_HOST_PTR | CL_MEM_HOST_WRITE_ONLY, + nbytes, NULL, &err); + if (err != CL_SUCCESS) { + return NULL; + } + void * mapped = clEnqueueMapBuffer(queue, buf, CL_TRUE, + CL_MAP_WRITE_INVALIDATE_REGION, 0, nbytes, 0, NULL, NULL, &err); + if (err != CL_SUCCESS) { + clReleaseMemObject(buf); + return NULL; + } + memcpy(mapped, data, nbytes); + const cl_int uerr = clEnqueueUnmapMemObject(queue, buf, mapped, 0, NULL, NULL); + if (uerr != CL_SUCCESS) { + clReleaseMemObject(buf); + return NULL; + } + if (tensor_name_for_log) { + GGML_LOG_INFO("ggml_opencl: %s (%.1f MiB) — device alloc failed, using CL_MEM_ALLOC_HOST_PTR fallback\n", + tensor_name_for_log, nbytes / 1024.0 / 1024.0); + } + return buf; +} + +// Allocate a temporary download buffer of `nbytes`. The caller runs a kernel +// that writes into it, then reads it back to host via clEnqueueReadBuffer (or +// equivalent). Mirrors ggml_cl_create_temp_upload_buffer; the host-pinned +// fallback flags are flipped (CL_MEM_WRITE_ONLY | HOST_READ_ONLY) and the +// helper doesn't populate the buffer. +static cl_mem ggml_cl_create_temp_download_buffer( + cl_context context, cl_command_queue queue, + size_t nbytes, const char * tensor_name_for_log) +{ + cl_int err; + cl_mem buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); + if (err != CL_SUCCESS) { + clFinish(queue); + buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); + } + if (err == CL_SUCCESS) { + return buf; + } + buf = clCreateBuffer(context, + CL_MEM_WRITE_ONLY | CL_MEM_ALLOC_HOST_PTR | CL_MEM_HOST_READ_ONLY, + nbytes, NULL, &err); + if (err != CL_SUCCESS) { + return NULL; + } + if (tensor_name_for_log) { + GGML_LOG_INFO("ggml_opencl: %s download (%.1f MiB) — device alloc failed, using CL_MEM_ALLOC_HOST_PTR fallback\n", + tensor_name_for_log, nbytes / 1024.0 / 1024.0); + } + return buf; +} + static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer->buft->device->context; ggml_backend_opencl_context * backend_ctx = dev_ctx->backend_ctx; @@ -9567,12 +9697,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); // We consider the specified offset arg as always, although For weights // the offset arg should be 0 (we do not assert this). @@ -9730,12 +9856,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_d + size_m + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; @@ -9862,12 +9984,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_d + size_qs + size_qh == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; @@ -10026,12 +10144,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_d + size_m + size_qs + size_qh == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; @@ -10179,12 +10293,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_e + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); // The original tensor memory is divided into scales and quants, i.e., // we first store scales, then quants. @@ -10290,12 +10400,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); // The original tensor memory is divided into scales and quants, i.e., // we first store scales, then quants. @@ -10394,12 +10500,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; @@ -10478,12 +10580,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_d + size_dm + size_s + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "q4_K set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; @@ -10680,9 +10778,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, "Incorrect tensor size"); cl_int err; - cl_mem data_device; - CL_CHECK((data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, ggml_nbytes(tensor), NULL, &err), err)); - CL_CHECK(clEnqueueWriteBuffer(queue, data_device, CL_TRUE, 0, ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "q5_K set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; @@ -10868,9 +10965,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, "Incorrect tensor size"); cl_int err; - cl_mem data_device; - CL_CHECK((data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, ggml_nbytes(tensor), NULL, &err), err)); - CL_CHECK(clEnqueueWriteBuffer(queue, data_device, CL_TRUE, 0, ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "q6_K set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; @@ -11211,9 +11307,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, cl_int err; cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0_trans4_ns; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; @@ -11282,10 +11377,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, } #endif - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); @@ -11310,10 +11403,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1_trans4_ns; int ne00 = tensor->ne[0]; @@ -11385,10 +11476,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, } #endif - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); @@ -11416,9 +11505,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, if (use_adreno_moe_kernels(backend_ctx, tensor)) { cl_int err; // TODO: use ggml_cl_buffer to manage this temporary buffer - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q5_0_trans4_ns; @@ -11520,9 +11608,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, if (use_adreno_moe_kernels(backend_ctx, tensor)) { cl_int err; // TODO: use ggml_cl_buffer to manage this temporary buffer - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q5_1_trans4_ns; @@ -11627,10 +11714,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, if (tensor->type == GGML_TYPE_MXFP4) { ggml_tensor_extra_cl_mxfp4 * extra = (ggml_tensor_extra_cl_mxfp4 *)tensor->extra; - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { @@ -11692,10 +11777,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * extra_src = tensor->view_src != nullptr ? tensor->view_src : tensor; ggml_tensor_extra_cl_q8_0 * extra = (ggml_tensor_extra_cl_q8_0 *)extra_src->extra; - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (enable_adreno_trans_weight(backend_ctx, tensor)) { @@ -11748,10 +11831,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, if (tensor->type == GGML_TYPE_IQ4_NL) { ggml_tensor_extra_cl_iq4_nl * extra = (ggml_tensor_extra_cl_iq4_nl *)tensor->extra; - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_kernels(backend_ctx, tensor)) { @@ -11820,10 +11901,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, if (tensor->type == GGML_TYPE_Q4_K) { ggml_tensor_extra_cl_q4_K * extra = (ggml_tensor_extra_cl_q4_K *)tensor->extra; - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; @@ -11878,10 +11957,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, return; } if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q4_k_trans4_ns; @@ -11986,20 +12063,16 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, if (tensor->type == GGML_TYPE_Q5_K) { ggml_tensor_extra_cl_q5_K * extra = (ggml_tensor_extra_cl_q5_K *)tensor->extra; - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q5_k_trans4_ns; int ne00 = tensor->ne[0]; @@ -12159,10 +12232,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, return; } if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q6_k_trans4_ns; @@ -12249,10 +12320,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, } #endif // GGML_OPENCL_USE_ADRENO_KERNELS - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_uchar mask = 0xFF; cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); @@ -12380,6 +12449,21 @@ static ggml_backend_buffer_t ggml_backend_opencl_buffer_type_alloc_buffer(ggml_b cl_int err; cl_mem mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size, NULL, &err); + // On Adreno X1-85 the device pool intermittently fails at hundreds of MB + // once the heap fragments (e.g. graph-allocator compute-buffer reserve + // after model load). Four-step retry: + // 1. normal alloc (fast path) + // 2. clFinish + retry (drains in-flight allocs) + // 3. cl_qcom_large_buffer (X2-class driver only, OpenCL 3.0 only) + // 4. ALLOC_HOST_PTR (host-pinned pool) — last-resort fallback. This + // buffer backs compute scratch read/written by every kernel in the + // graph, so kernel accesses fall to host memory and runtime perf + // degrades meaningfully. Better than failing to load, but the user + // should see the warning and consider -ngl reduction. + if (err != CL_SUCCESS) { + clFinish(backend_ctx->queue); + mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size, NULL, &err); + } #if GGML_OPENCL_TARGET_VERSION >= 300 // clCreateBufferWithProperties and cl_mem_properties are OpenCL 3.0. Drivers older than // that do not export the symbol, so a build targeting them fails to link. The large @@ -12390,9 +12474,20 @@ static ggml_backend_buffer_t ggml_backend_opencl_buffer_type_alloc_buffer(ggml_b mem = clCreateBufferWithProperties(backend_ctx->context, props, CL_MEM_READ_WRITE, size, NULL, &err); } #endif + if (err != CL_SUCCESS) { + mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, size, NULL, &err); + if (err == CL_SUCCESS) { + GGML_LOG_WARN("%s: %.2f MiB allocated via CL_MEM_ALLOC_HOST_PTR fallback — " + "device pool exhausted; runtime perf will be degraded. " + "Consider lowering -ngl or context size.\n", + __func__, size / 1024.0 / 1024.0); + } + } if (err != CL_SUCCESS) { - GGML_LOG_INFO("%s: failed to allocate %.2f MiB\n", __func__, size / 1024.0 / 1024.0); + GGML_LOG_ERROR("%s: failed to allocate %.2f MiB (err=%d). " + "Consider reducing -ngl, lowering -c / -ub, or using quantized KV cache.\n", + __func__, size / 1024.0 / 1024.0, err); return nullptr; } @@ -13147,6 +13242,7 @@ static void ggml_cl_set_rows(ggml_backend_t backend, const ggml_tensor * src0, c (size_t)ne03}; size_t local_work_size[] = {(size_t)nth0, (size_t)rows_per_workgroup, 1}; + // ne01 == 0 makes global_work_size[0] zero here; enqueue_ndrange_kernel drops the empty range. backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } From c069aa7f5f2beeead1a3a8e9f71510f1b64d0725 Mon Sep 17 00:00:00 2001 From: Pascal Date: Sat, 12 Sep 2026 07:38:50 +0200 Subject: [PATCH 26/35] server: frame the router child state command as a whole line (#28747) The child writes its state commands on stdout while the logger writes on stderr, and both share a single pipe. The logger emits the trailing color reset after the newline of a debug, warn or error entry, so that escape sequence has no newline of its own and the router reads it glued in front of the next command. The line prefix check then fails and the command is forwarded as a log line instead of being handled, which leaves a finished download stuck in the downloading state. Writing the command with a leading newline closes the pending line so it always starts at a line boundary. --- tools/server/server-models.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index f1783c083..4984f1be6 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1786,7 +1786,10 @@ void server_child::notify_to_router(const std::string & state, const json & payl std::lock_guard lk(mtx_stdout); common_log_pause(common_log_main()); fflush(stdout); - fprintf(stdout, "%s%s\n", CMD_CHILD_TO_ROUTER_STATE, safe_json_to_str(data).c_str()); + // the router matches the command on a line prefix, so the leading newline + // closes whatever the logger left open on the shared pipe, down to the + // trailing color reset that carries no newline of its own + fprintf(stdout, "\n%s%s\n", CMD_CHILD_TO_ROUTER_STATE, safe_json_to_str(data).c_str()); fflush(stdout); common_log_resume(common_log_main()); } From f3a33dff26f5d5ba8fbf47a26d4857c6edfe69a8 Mon Sep 17 00:00:00 2001 From: Ed Addario <29247825+EAddario@users.noreply.github.com> Date: Sat, 12 Sep 2026 07:22:57 +0100 Subject: [PATCH 27/35] rpc : fix linking when compiling with BUILD_SHARED_LIBS=OFF (#28492) --- ggml/src/ggml-rpc/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-rpc/CMakeLists.txt b/ggml/src/ggml-rpc/CMakeLists.txt index af3bd0290..e3d0c9b4c 100644 --- a/ggml/src/ggml-rpc/CMakeLists.txt +++ b/ggml/src/ggml-rpc/CMakeLists.txt @@ -36,8 +36,10 @@ if (GGML_RPC_RDMA) target_compile_definitions(ggml-rpc PRIVATE GGML_RPC_RDMA) if (APPLE) # librdma.dylib only exists on macOS 26.2 and later. Link it weakly so a build made - # where it exists still loads where it does not; checked at runtime before use. - target_link_options(ggml-rpc PRIVATE "LINKER:-weak_library,${RDMA_LIB}") + # where it exists still loads where it does not; checked at runtime before use + # but with BUILD_SHARED_LIBS=OFF ggml-rpc is a static archive and never links + # so the librdma symbols used by transport-apple.cpp stay undefined. + target_link_options(ggml-rpc PUBLIC "LINKER:-weak_library,${RDMA_LIB}") target_compile_definitions(ggml-rpc PRIVATE GGML_RPC_RDMA_APPLE) target_sources(ggml-rpc PRIVATE transport-apple.cpp) else() From 2a3005c23f60cb38dab70b8ea2ddbd969bcf3e87 Mon Sep 17 00:00:00 2001 From: Michael Taylor <162068037+mctylr-gh@users.noreply.github.com> Date: Sat, 12 Sep 2026 04:05:38 -0300 Subject: [PATCH 28/35] syscl : Handle (fail gracefully) unsupported tq1_0 quants (#28681) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 4091f73a4..f225682f2 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -6326,7 +6326,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons return false; } - if (src0_type == GGML_TYPE_TQ2_0) { + if (src0_type == GGML_TYPE_TQ2_0 || src0_type == GGML_TYPE_TQ1_0) { return false; } @@ -6380,7 +6380,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons case GGML_OP_SET_ROWS: { - if (op->type == GGML_TYPE_TQ2_0) { + if (op->type == GGML_TYPE_TQ2_0 || op->type == GGML_TYPE_TQ1_0) { return false; } auto res = (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || @@ -6502,12 +6502,14 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons src1_type == GGML_TYPE_IQ3_S || src1_type == GGML_TYPE_IQ1_S || src1_type == GGML_TYPE_IQ1_M || - src1_type == GGML_TYPE_TQ2_0) { + src1_type == GGML_TYPE_TQ2_0 || + src1_type == GGML_TYPE_TQ1_0) { return false; } } - if (src0_type == GGML_TYPE_TQ2_0 || src1_type == GGML_TYPE_TQ2_0) { + if (src0_type == GGML_TYPE_TQ2_0 || src1_type == GGML_TYPE_TQ2_0 || + src0_type == GGML_TYPE_TQ1_0 || src1_type == GGML_TYPE_TQ1_0) { return false; } From 718f7b4175bf8b6af6f5eac09fee10754b3ecddd Mon Sep 17 00:00:00 2001 From: "Alessandro de Oliveira Faria (A.K.A.CABELO)" Date: Sat, 12 Sep 2026 04:15:08 -0300 Subject: [PATCH 29/35] vendor : update cpp-httplib to 0.56.0 (#28787) --- scripts/sync_vendor.py | 2 +- vendor/cpp-httplib/httplib.cpp | 296 ++++++++++++++++++++++++--------- vendor/cpp-httplib/httplib.h | 92 ++++++++-- 3 files changed, 298 insertions(+), 92 deletions(-) diff --git a/scripts/sync_vendor.py b/scripts/sync_vendor.py index 0170b5b16..a73ae1193 100755 --- a/scripts/sync_vendor.py +++ b/scripts/sync_vendor.py @@ -5,7 +5,7 @@ import os import sys import subprocess -HTTPLIB_VERSION = "refs/tags/v0.54.1" +HTTPLIB_VERSION = "refs/tags/v0.56.0" # used by examples/gguf-hash, these repos have no release tag, so we pin a commit XXHASH_COMMIT = "9f465f1ea932d6ad9a26cd77496311ffa544cd68" diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp index 7fd10b393..c82ff1e71 100644 --- a/vendor/cpp-httplib/httplib.cpp +++ b/vendor/cpp-httplib/httplib.cpp @@ -912,17 +912,42 @@ bool write_websocket_frame(Stream &strm, ws::Opcode opcode, namespace ws { namespace impl { -bool read_websocket_frame(Stream &strm, Opcode &opcode, - std::string &payload, bool &fin, - bool expect_masked, size_t max_len) { - // Read first 2 bytes +// Read exactly `size` bytes. Stream::read may return less than asked for -- it +// hands back whatever its buffer already holds -- so every multi-byte field has +// to loop. Reading a 2-byte header with a single read() fails whenever the +// header straddles the read buffer's boundary. +// +// Timeout is reported only when nothing at all was consumed. Once a byte has +// been taken the stream sits mid-field and cannot be resumed, so a timeout +// there is a failure like any other. (When read() fails it always records why, +// so the error belongs to this call and not to an earlier one.) +FrameRead read_exact(Stream &strm, void *buf, size_t size) { + auto p = static_cast(buf); + size_t total = 0; + while (total < size) { + auto n = strm.read(p + total, size - total); + if (n <= 0) { + auto timed_out = total == 0 && strm.get_error() == Error::Timeout; + return timed_out ? FrameRead::Timeout : FrameRead::Fail; + } + total += static_cast(n); + } + return FrameRead::Ok; +} + +FrameRead read_websocket_frame(Stream &strm, Opcode &opcode, + std::string &payload, bool &fin, + bool expect_masked, size_t max_len) { + // Read first 2 bytes. This is the only read that may report a timeout: it + // sits on a frame boundary, where nothing has been consumed yet. uint8_t header[2]; - if (strm.read(reinterpret_cast(header), 2) != 2) { return false; } + FrameRead first = read_exact(strm, header, 2); + if (first != FrameRead::Ok) { return first; } fin = (header[0] & 0x80) != 0; // RSV1, RSV2, RSV3 must be 0 when no extension is negotiated - if (header[0] & 0x70) { return false; } + if (header[0] & 0x70) { return FrameRead::Fail; } opcode = static_cast(header[0] & 0x0F); bool masked = (header[1] & 0x80) != 0; @@ -932,46 +957,44 @@ bool read_websocket_frame(Stream &strm, Opcode &opcode, // MUST have a payload length of 125 bytes or less bool is_control = (static_cast(opcode) & 0x08) != 0; if (is_control) { - if (!fin) { return false; } - if (payload_len > 125) { return false; } + if (!fin) { return FrameRead::Fail; } + if (payload_len > 125) { return FrameRead::Fail; } } - if (masked != expect_masked) { return false; } + if (masked != expect_masked) { return FrameRead::Fail; } // Extended payload length if (payload_len == 126) { uint8_t ext[2]; - if (strm.read(reinterpret_cast(ext), 2) != 2) { return false; } + if (read_exact(strm, ext, 2) != FrameRead::Ok) { return FrameRead::Fail; } payload_len = (static_cast(ext[0]) << 8) | ext[1]; } else if (payload_len == 127) { uint8_t ext[8]; - if (strm.read(reinterpret_cast(ext), 8) != 8) { return false; } + if (read_exact(strm, ext, 8) != FrameRead::Ok) { return FrameRead::Fail; } // RFC 6455 Section 5.2: the most significant bit MUST be 0 - if (ext[0] & 0x80) { return false; } + if (ext[0] & 0x80) { return FrameRead::Fail; } payload_len = 0; for (int i = 0; i < 8; i++) { payload_len = (payload_len << 8) | ext[i]; } } - if (payload_len > max_len) { return false; } + if (payload_len > max_len) { return FrameRead::Fail; } // Read mask key if present uint8_t mask_key[4] = {0}; if (masked) { - if (strm.read(reinterpret_cast(mask_key), 4) != 4) { return false; } + if (read_exact(strm, mask_key, 4) != FrameRead::Ok) { + return FrameRead::Fail; + } } // Read payload payload.resize(static_cast(payload_len)); - if (payload_len > 0) { - size_t total_read = 0; - while (total_read < payload_len) { - auto n = strm.read(&payload[total_read], - static_cast(payload_len - total_read)); - if (n <= 0) { return false; } - total_read += static_cast(n); - } + if (payload_len > 0 && + read_exact(strm, &payload[0], static_cast(payload_len)) != + FrameRead::Ok) { + return FrameRead::Fail; } // Unmask if needed @@ -981,7 +1004,7 @@ bool read_websocket_frame(Stream &strm, Opcode &opcode, } } - return true; + return FrameRead::Ok; } } // namespace impl @@ -1728,7 +1751,9 @@ ssize_t select_impl(socket_t sock, short events, time_t sec, pfd.events = events; pfd.revents = 0; - auto timeout = static_cast(sec * 1000 + usec / 1000); + // A negative timeout waits forever, poll's own convention. 0 keeps meaning + // "return immediately", which callers here rely on to probe a socket. + auto timeout = sec < 0 ? -1 : static_cast(sec * 1000 + usec / 1000); return handle_EINTR([&]() { return poll_wrapper(&pfd, 1, timeout); }); } @@ -1810,8 +1835,11 @@ private: bool ensure_readable(); socket_t sock_; - time_t read_timeout_sec_; - time_t read_timeout_usec_; + // Atomic because ws::WebSocket::set_read_timeout() reaches this from another + // thread while a read is in flight -- that is the point of it, for a caller + // holding one connection and wanting control back to send on it. + std::atomic read_timeout_sec_; + std::atomic read_timeout_usec_; time_t write_timeout_sec_; time_t write_timeout_usec_; time_t max_timeout_msec_; @@ -2204,12 +2232,10 @@ int getaddrinfo_with_timeout(const char *node, const char *service, // actually finish before letting the stack frame go. The trade-off is that // a wedged DNS server can hold this thread for the system resolver timeout // (~30s by default) past the caller's connection timeout. - struct gaicb request {}; + struct gaicb request{}; struct gaicb *requests[1] = {&request}; - struct sigevent sevp {}; - struct timespec timeout { - timeout_sec, 0 - }; + struct sigevent sevp{}; + struct timespec timeout{timeout_sec, 0}; request.ar_name = node; request.ar_service = service; @@ -2948,8 +2974,21 @@ EncodingType encoding_type(const Request &req, return best; } +// `content_type` is taken separately because a file-backed response has not +// been given one yet when its coding has to be decided. +EncodingType encoding_type(const Request &req, const Response &res, + const std::string &content_type) { + // The response already names a content coding of its own: a handler serving + // a body it encoded itself (pre-compressed static assets, say), or a mount + // point whose headers name the coding its files are stored in. Applying one + // on top of that would double-encode the body and append a second + // `Content-Encoding` field line. + if (res.has_header("Content-Encoding")) { return EncodingType::None; } + return encoding_type(req, content_type); +} + EncodingType encoding_type(const Request &req, const Response &res) { - return encoding_type(req, res.get_header_value("Content-Type")); + return encoding_type(req, res, res.get_header_value("Content-Type")); } std::unique_ptr make_compressor(EncodingType type) { @@ -3677,6 +3716,17 @@ bool is_chunked_transfer_encoding(const Headers &headers) { return case_ignore::equal(last_coding, "chunked"); } +bool has_conflicting_content_length(const Headers &headers) { + // RFC 9112 §6.3: a message carrying both Transfer-Encoding and a non-zero + // Content-Length is framed ambiguously. The body readers here delimit it by + // the transfer coding and drop Content-Length, while an intermediary may do + // the reverse, so the two disagree on where the body ends and a reused + // connection is desynchronised (request/response smuggling). Content-Length: + // 0 is tolerated for compatibility with existing peers. + return has_header(headers, "Transfer-Encoding") && + get_header_value_u64(headers, "Content-Length", 0, 0) > 0; +} + template bool prepare_content_receiver(T &x, int &status, ContentReceiverWithProgress receiver, @@ -4035,7 +4085,7 @@ void set_file_content_provider(Response &res, return true; }); - res.file_content_encoding_ = encoding; + res.content_coding_ = encoding; } template @@ -4361,13 +4411,20 @@ bool parse_range_header(const std::string &s, Ranges &ranges) try { ssize_t first = -1; if (!lhs.empty()) { - ssize_t v; - auto res = detail::from_chars(lhs.data(), lhs.data() + lhs.size(), v); - if (res.ec == std::errc{}) { first = v; } + // Reject an overflowing first-byte-pos; treating it as absent (-1) + // would turn the range into a suffix range. + auto res = + detail::from_chars(lhs.data(), lhs.data() + lhs.size(), first); + if (res.ec != std::errc{}) { + all_valid_ranges = false; + return; + } } ssize_t last = -1; if (!rhs.empty()) { + // An overflowing last-byte-pos is past any content length, so keeping + // -1 ("remainder", RFC 9110 14.1.2) is correct here. ssize_t v; auto res = detail::from_chars(rhs.data(), rhs.data() + rhs.size(), v); if (res.ec == std::errc{}) { last = v; } @@ -6902,7 +6959,7 @@ void Response::set_content(const char *s, size_t n, auto rng = headers.equal_range("Content-Type"); headers.erase(rng.first, rng.second); set_header("Content-Type", content_type); - file_content_encoding_ = detail::EncodingType::None; + content_coding_ = detail::EncodingType::None; } void Response::set_content(const std::string &s, @@ -6917,7 +6974,7 @@ void Response::set_content(std::string &&s, auto rng = headers.equal_range("Content-Type"); headers.erase(rng.first, rng.second); set_header("Content-Type", content_type); - file_content_encoding_ = detail::EncodingType::None; + content_coding_ = detail::EncodingType::None; } void Response::set_content_provider( @@ -6928,7 +6985,7 @@ void Response::set_content_provider( if (in_length > 0) { content_provider_ = std::move(provider); } content_provider_resource_releaser_ = std::move(resource_releaser); is_chunked_content_provider_ = false; - file_content_encoding_ = detail::EncodingType::None; + content_coding_ = detail::EncodingType::None; } void Response::set_content_provider( @@ -6939,7 +6996,7 @@ void Response::set_content_provider( content_provider_ = detail::ContentProviderAdapter(std::move(provider)); content_provider_resource_releaser_ = std::move(resource_releaser); is_chunked_content_provider_ = false; - file_content_encoding_ = detail::EncodingType::None; + content_coding_ = detail::EncodingType::None; } void Response::set_chunked_content_provider( @@ -6950,7 +7007,7 @@ void Response::set_chunked_content_provider( content_provider_ = detail::ContentProviderAdapter(std::move(provider)); content_provider_resource_releaser_ = std::move(resource_releaser); is_chunked_content_provider_ = true; - file_content_encoding_ = detail::EncodingType::None; + content_coding_ = detail::EncodingType::None; } void Response::set_file_content(const std::string &path, @@ -7991,12 +8048,19 @@ ssize_t WebSocketSSLStream::read(char *ptr, size_t size) { needs_readable || (err.code == tls::ErrorCode::SyscallError && WSAGetLastError() == WSAETIMEDOUT); #endif - if (!needs_readable && err.code != tls::ErrorCode::WantWrite) { return -1; } + if (!needs_readable && err.code != tls::ErrorCode::WantWrite) { + error_ = Error::Read; + return -1; + } if (!(needs_readable ? wait_readable() : wait_writable())) { error_ = Error::Timeout; return -1; } } + // Out of retries. Recording a reason matters: a caller that reads get_error() + // to tell a timeout from a close would otherwise see whatever the previous + // failure left behind (error_ is never cleared on success). + error_ = Error::Read; return -1; } @@ -8653,9 +8717,10 @@ Server::write_content_with_provider(Stream &strm, const Request &req, } } else { if (res.is_chunked_content_provider_) { - auto type = detail::encoding_type(req, res); - - auto compressor = detail::make_compressor(type); + // Use the coding `apply_ranges()` chose when it wrote the headers; + // re-negotiating here would disagree with them, e.g. once a handler's + // own Content-Encoding header suppresses the negotiation. + auto compressor = detail::make_compressor(res.content_coding_); if (!compressor) { compressor = detail::make_unique(); } @@ -8881,7 +8946,8 @@ bool Server::handle_file_request(Request &req, Response &res) { auto encoding = detail::EncodingType::None; if (static_file_compression_) { content_type = content_type_of(); - encoding = static_file_encoding(req, content_type, stat.size()); + encoding = + static_file_encoding(req, res, content_type, stat.size()); } // The ETag names the representation actually sent, so a client that @@ -9296,8 +9362,10 @@ bool Server::dispatch_request(Request &req, Response &res, // the ETag, which has to name the representation actually sent, and // `apply_static_file_compression()` go through this, so the two cannot drift // apart. -detail::EncodingType Server::static_file_encoding( - const Request &req, const std::string &content_type, size_t length) const { +detail::EncodingType +Server::static_file_encoding(const Request &req, const Response &res, + const std::string &content_type, + size_t length) const { if (!static_file_compression_) { return detail::EncodingType::None; } // Nothing to compress, and an empty file already answers with @@ -9322,14 +9390,14 @@ detail::EncodingType Server::static_file_encoding( return detail::EncodingType::None; } - return detail::encoding_type(req, content_type); + return detail::encoding_type(req, res, content_type); } // Compresses a file-backed content provider into `res.body` and takes over the // framing headers. Returns false when the response is left untouched. bool Server::apply_static_file_compression(const Request &req, Response &res) const { - auto type = res.file_content_encoding_; + auto type = res.content_coding_; if (type == detail::EncodingType::None || !res.content_provider_) { return false; } @@ -9353,7 +9421,7 @@ bool Server::apply_static_file_compression(const Request &req, res.content_provider_success_ = true; res.content_provider_ = nullptr; res.content_length_ = 0; - res.file_content_encoding_ = detail::EncodingType::None; + res.content_coding_ = detail::EncodingType::None; res.set_header("Content-Encoding", detail::encoding_name(type)); res.set_header("Vary", "Accept-Encoding"); @@ -9412,6 +9480,7 @@ void Server::apply_ranges(const Request &req, Response &res, if (res.content_provider_) { if (res.is_chunked_content_provider_) { res.set_header("Transfer-Encoding", "chunked"); + res.content_coding_ = type; if (type != detail::EncodingType::None) { res.set_header("Content-Encoding", detail::encoding_name(type)); res.set_header("Vary", "Accept-Encoding"); @@ -9568,8 +9637,8 @@ Server::process_request(Stream &strm, const std::string &remote_addr, // coding is not chunked, which leaves the body length undeterminable. The // latter must not fall through to the "no body" path, or the body bytes are // parsed as the next request on a persistent connection. - if (req.has_header("Transfer-Encoding") && - (req.get_header_value_u64("Content-Length") > 0 || + if (detail::has_conflicting_content_length(req.headers) || + (req.has_header("Transfer-Encoding") && !detail::is_chunked_transfer_encoding(req.headers))) { connection_closed = true; res.status = StatusCode::BadRequest_400; @@ -9734,7 +9803,7 @@ Server::process_request(Stream &strm, const std::string &remote_addr, auto ws_strm = std::unique_ptr(new detail::WebSocketSSLStream( strm.socket(), const_cast(req.ssl), - CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND, 0, + CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND, 0, write_timeout_sec_, write_timeout_usec_)); ws::WebSocket ws(std::move(ws_strm), req, true, websocket_ping_interval_sec_, @@ -9744,7 +9813,8 @@ Server::process_request(Stream &strm, const std::string &remote_addr, } #endif // Use WebSocket-specific read timeout instead of HTTP timeout - strm.set_read_timeout(CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND, 0); + strm.set_read_timeout(CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND, + 0); ws::WebSocket ws(strm, req, true, websocket_ping_interval_sec_, websocket_max_missed_pongs_); entry.handler(req, ws); @@ -9808,7 +9878,7 @@ Server::process_request(Stream &strm, const std::string &remote_addr, detail::set_file_content_provider( res, mm, content_type, - static_file_encoding(req, content_type, mm->size())); + static_file_encoding(req, res, content_type, mm->size())); } } @@ -10228,8 +10298,12 @@ Result ClientImpl::send_(Request &&req) { void ClientImpl::prepare_default_headers(Request &r, bool for_stream, const std::string &ct) { (void)for_stream; - for (const auto &header : default_headers_) { - if (!r.has_header(header.first)) { r.headers.insert(header); } + // Default headers are meant for the origin and may carry its credentials, so + // keep them off the CONNECT request the proxy reads. + if (r.method != "CONNECT") { + for (const auto &header : default_headers_) { + if (!r.has_header(header.first)) { r.headers.insert(header); } + } } // RFC 9110 5.3 recommends sending control data such as Host first, so @@ -10379,6 +10453,17 @@ ClientImpl::open_stream(const std::string &method, const std::string &path, return handle; } + // Same framing check as ClientImpl::process_request(). A HEAD or bodyless + // (204/304) response legitimately carries framing headers with no body. + if (method != "HEAD" && + handle.response->status != StatusCode::NoContent_204 && + handle.response->status != StatusCode::NotModified_304 && + detail::has_conflicting_content_length(handle.response->headers)) { + handle.error = Error::Read; + handle.response.reset(); + return handle; + } + handle.body_reader_.stream = handle.stream_; handle.body_reader_.payload_max_length = payload_max_length_; @@ -10910,24 +10995,24 @@ bool ClientImpl::write_request(Stream &strm, Request &req, } } - if (!basic_auth_password_.empty() || !basic_auth_username_.empty()) { - if (!req.has_header("Authorization")) { + // A CONNECT request is read by the proxy; everything sent through the tunnel + // it opens is read by the origin. Each credential goes only to its own hop. + auto is_connect = req.method == "CONNECT"; + + if (!is_connect && !req.has_header("Authorization")) { + if (!basic_auth_password_.empty() || !basic_auth_username_.empty()) { req.headers.insert(make_basic_authentication_header( basic_auth_username_, basic_auth_password_, false)); - } - } - - if (!bearer_token_auth_token_.empty()) { - if (!req.has_header("Authorization")) { + } else if (!bearer_token_auth_token_.empty()) { req.headers.insert(make_bearer_token_authentication_header( bearer_token_auth_token_, false)); } } - // Proxy-Authorization is only sent when the proxy is actually used for - // this target — otherwise NO_PROXY-matched requests would leak proxy - // credentials directly to the destination server. - if (is_proxy_enabled_for_host(host_)) { + // Proxy-Authorization is only sent when the proxy reads this message — + // otherwise NO_PROXY-matched requests, and requests inside a TLS tunnel, + // would leak proxy credentials to the destination server. + if (is_proxy_enabled_for_host(host_) && (!is_ssl() || is_connect)) { if (!proxy_basic_auth_username_.empty() && !proxy_basic_auth_password_.empty() && !req.has_header("Proxy-Authorization")) { @@ -11323,6 +11408,17 @@ bool ClientImpl::process_request(Stream &strm, Request &req, // Body if ((res.status != StatusCode::NoContent_204) && req.method != "HEAD" && req.method != "CONNECT") { + // Reject ambiguous framing (RFC 9112 §6.3). Unlike a request, a response + // whose final transfer coding is not chunked is not ambiguous: its body + // runs until the server closes the connection, so it is not rejected. + // HEAD/204 are excluded above and a 304 carries no body. + if (res.status != StatusCode::NotModified_304 && + detail::has_conflicting_content_length(res.headers)) { + error = Error::Read; + output_error_log(error, &req); + return false; + } + auto redirect = 300 < res.status && res.status < 400 && res.status != StatusCode::NotModified_304 && follow_location_; @@ -17562,8 +17658,16 @@ ReadResult WebSocket::read(std::string &msg) { std::string payload; bool fin; - if (!impl::read_websocket_frame(strm_, opcode, payload, fin, is_server_, - CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH)) { + impl::FrameRead r = + impl::read_websocket_frame(strm_, opcode, payload, fin, is_server_, + CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH); + // A timeout landed on a frame boundary: the connection is untouched and + // still usable, so hand control back without closing it. That is only + // useful to a caller who asked for the timeout; the compile-time default + // is a backstop against a peer gone quiet, and elapsing it closes the + // connection so a plain `while (ws.read(msg))` loop ends. + if (r == impl::FrameRead::Timeout && read_timeout_set_) { return Timeout; } + if (r != impl::FrameRead::Ok) { closed_ = true; return Fail; } @@ -17600,9 +17704,14 @@ ReadResult WebSocket::read(std::string &msg) { Opcode cont_opcode; std::string cont_payload; bool cont_fin; - if (!impl::read_websocket_frame( + // A timeout is not reportable here: half of a fragmented message is + // already in `msg` and read() has no way to resume it, so it is a + // failure like any other. Timeouts are only ever seen on a message + // boundary. + if (impl::read_websocket_frame( strm_, cont_opcode, cont_payload, cont_fin, is_server_, - CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH)) { + CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH) != + impl::FrameRead::Ok) { closed_ = true; return Fail; } @@ -17696,7 +17805,8 @@ void WebSocket::close(CloseStatus status, const std::string &reason) { Opcode op; std::string resp; bool fin; - while (impl::read_websocket_frame(strm_, op, resp, fin, is_server_, 125)) { + while (impl::read_websocket_frame(strm_, op, resp, fin, is_server_, 125) == + impl::FrameRead::Ok) { if (op == Opcode::Close) { break; } } } @@ -17741,6 +17851,15 @@ const Request &WebSocket::request() const { return req_; } bool WebSocket::is_open() const { return !closed_; } +void WebSocket::set_read_timeout(time_t sec, time_t usec) { + // 0 waits forever here, as it does for SO_RCVTIMEO. The stream waits with + // poll(), where 0 would instead mean "return immediately", so hand it the + // negative poll uses for an unbounded wait. + if (sec == 0 && usec == 0) { sec = -1; } + strm_.set_read_timeout(sec, usec); + read_timeout_set_ = true; +} + // WebSocketClient implementation WebSocketClient::WebSocketClient( const std::string &scheme_host_port_path, const Headers &headers) @@ -17843,6 +17962,16 @@ void WebSocketClient::shutdown_and_close() { bool WebSocketClient::create_stream(std::unique_ptr &strm, Error &error, int &ssl_error, uint64_t &ssl_backend_error) { + // A read timeout of 0 means "wait forever", the way SO_RCVTIMEO reads it. + // The streams wait with poll(), where 0 instead means "return immediately", + // so they are given the negative poll uses for an unbounded wait. + auto unbounded = read_timeout_sec_ == 0 && read_timeout_usec_ == 0; + time_t strm_read_sec = unbounded ? -1 : read_timeout_sec_; + time_t strm_read_usec = unbounded ? 0 : read_timeout_usec_; + // The handshake belongs to establishing the connection, so an unset read + // timeout leaves it bounded by the connection timeout instead of forever. + time_t hs_sec = unbounded ? connection_timeout_sec_ : read_timeout_sec_; + time_t hs_usec = unbounded ? connection_timeout_usec_ : read_timeout_usec_; #ifdef CPPHTTPLIB_SSL_ENABLED if (is_ssl_) { // A plain flag rather than SSLClient::load_certs()'s call_once: connect() @@ -17862,8 +17991,8 @@ bool WebSocketClient::create_stream(std::unique_ptr &strm, detail::ClientTlsSessionError tls_error; if (!detail::setup_client_tls_session(host_, tls_ctx_, tls_session_, sock_, server_certificate_verification_, - read_timeout_sec_, read_timeout_usec_, - &tls_error, options)) { + hs_sec, hs_usec, &tls_error, + options)) { error = tls_error.error; ssl_error = tls_error.ssl_error; ssl_backend_error = tls_error.backend_error; @@ -17871,17 +18000,19 @@ bool WebSocketClient::create_stream(std::unique_ptr &strm, } strm = std::unique_ptr(new detail::WebSocketSSLStream( - sock_, tls_session_, read_timeout_sec_, read_timeout_usec_, - write_timeout_sec_, write_timeout_usec_)); + sock_, tls_session_, strm_read_sec, strm_read_usec, write_timeout_sec_, + write_timeout_usec_)); return true; } #else (void)error; (void)ssl_error; (void)ssl_backend_error; + (void)hs_sec; + (void)hs_usec; #endif strm = std::unique_ptr( - new detail::SocketStream(sock_, read_timeout_sec_, read_timeout_usec_, + new detail::SocketStream(sock_, strm_read_sec, strm_read_usec, write_timeout_sec_, write_timeout_usec_)); return true; } @@ -17951,6 +18082,9 @@ Result WebSocketClient::connect() { ws_ = std::unique_ptr(new WebSocket(std::move(strm), req, false, websocket_ping_interval_sec_, websocket_max_missed_pongs_)); + // The stream was created with the timeout already; tell the WebSocket + // whether it came from the caller, so read() knows to report it as Timeout. + ws_->read_timeout_set_ = read_timeout_set_; return Result{Error::Success, upgrade.status, std::move(upgrade.headers)}; } @@ -17983,6 +18117,10 @@ const std::string &WebSocketClient::subprotocol() const { void WebSocketClient::set_read_timeout(time_t sec, time_t usec) { read_timeout_sec_ = sec; read_timeout_usec_ = usec; + read_timeout_set_ = true; + // The members above only seed the next connect(); read() consults the + // stream, so an already-open connection has to be told directly. + if (ws_) { ws_->set_read_timeout(sec, usec); } } void WebSocketClient::set_write_timeout(time_t sec, time_t usec) { diff --git a/vendor/cpp-httplib/httplib.h b/vendor/cpp-httplib/httplib.h index ca7c96a41..a3a2ff45a 100644 --- a/vendor/cpp-httplib/httplib.h +++ b/vendor/cpp-httplib/httplib.h @@ -8,8 +8,8 @@ #ifndef CPPHTTPLIB_HTTPLIB_H #define CPPHTTPLIB_HTTPLIB_H -#define CPPHTTPLIB_VERSION "0.54.1" -#define CPPHTTPLIB_VERSION_NUM "0x003601" +#define CPPHTTPLIB_VERSION "0.56.0" +#define CPPHTTPLIB_VERSION_NUM "0x003800" #ifdef _WIN32 #if defined(_WIN32_WINNT) && _WIN32_WINNT < 0x0A00 @@ -215,8 +215,36 @@ #define CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH 16777216 #endif -#ifndef CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND -#define CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND 300 +// One macro used to set the read timeout for both sides. They want different +// defaults: a client's read timeout is the caller's own tool (it waits forever +// until asked not to), while a server keeps a ceiling that reclaims a worker +// from a peer that has gone quiet. The old name still works and sets both. +#ifdef CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND +#pragma message( \ + "CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND is deprecated; define " \ + "CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND and/or " \ + "CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND instead") +#ifndef CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND \ + CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND +#endif +#ifndef CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND \ + CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND +#endif +#endif + +// 0 waits forever. A read timeout is how a caller gets control back to send on +// the same connection; it is not a liveness check (that is ping/pong). Only a +// timeout set at runtime through set_read_timeout() is reported as +// ws::Timeout; when one of these compile-time defaults elapses, read() returns +// ws::Fail and closes the connection. +#ifndef CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND 0 +#endif + +#ifndef CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND 300 #endif #ifndef CPPHTTPLIB_WEBSOCKET_CLOSE_TIMEOUT_SECOND @@ -1817,10 +1845,12 @@ struct Response { std::string file_content_path_; std::string file_content_content_type_; - // Content coding chosen for a file-backed content provider, decided once - // where the file is opened so that the ETag and the body cannot disagree. - // `EncodingType::None` for every other kind of response. - detail::EncodingType file_content_encoding_ = detail::EncodingType::None; + // Content coding chosen for the response body, decided once so that the + // headers and the body cannot disagree: where the file is opened for a + // file-backed content provider (keeping the ETag honest), and in + // `apply_ranges()` for a chunked content provider. `EncodingType::None` + // for every other kind of response. + detail::EncodingType content_coding_ = detail::EncodingType::None; }; enum class Error { @@ -2359,6 +2389,7 @@ private: bool parse_request_line(const char *s, Request &req) const; detail::EncodingType static_file_encoding(const Request &req, + const Response &res, const std::string &content_type, size_t length) const; bool apply_static_file_compression(const Request &req, Response &res) const; @@ -3663,6 +3694,9 @@ ssize_t read_socket(socket_t sock, void *ptr, size_t size, int flags); EncodingType encoding_type(const Request &req, const std::string &content_type); +EncodingType encoding_type(const Request &req, const Response &res, + const std::string &content_type); + EncodingType encoding_type(const Request &req, const Response &res); class BufferStream final : public Stream { @@ -4345,7 +4379,11 @@ enum class CloseStatus : uint16_t { InternalError = 1011, }; -enum ReadResult : int { Fail = 0, Text = 1, Binary = 2 }; +// Timeout is returned only when a read timeout was set and it elapsed before +// any byte of a frame arrived: nothing was consumed and the connection is +// still open, so the caller can send on it and read again. `msg` is left +// untouched, so a `while (ws.read(msg))` loop must not treat it as a message. +enum ReadResult : int { Fail = 0, Text = 1, Binary = 2, Timeout = 3 }; // Result of WebSocketClient::connect(). Truthy only when the WebSocket // upgrade handshake fully succeeded. On failure error() identifies the @@ -4405,6 +4443,18 @@ public: const Request &request() const; bool is_open() const; + // Bound how long read() waits before returning Timeout. 0 waits forever. + // A server handler owns its connection's timeout this way; a client sets it + // through WebSocketClient. Safe to call while another thread is in read(). + // + // Only a timeout set here is reported as Timeout. The compile-time default + // (CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND) is a backstop rather + // than a request for control, so when it elapses read() returns Fail and + // closes the connection, and `while (ws.read(msg))` ends as it always has. + void set_read_timeout(time_t sec, time_t usec = 0); + template + void set_read_timeout(const std::chrono::duration &duration); + private: friend class httplib::Server; friend class WebSocketClient; @@ -4440,6 +4490,10 @@ private: int max_missed_pongs_; int unacked_pings_ = 0; std::atomic closed_{false}; + // Set once the caller has bounded read() through set_read_timeout(). Until + // then the timeout in effect is the compile-time default, and elapsing it + // is a failure that closes the connection, not a Timeout. + std::atomic read_timeout_set_{false}; std::mutex write_mutex_; // Owned by whichever thread is parsing frames off strm_. Only one thread // may do so: read_websocket_frame() reads a payload until it has the whole @@ -4527,8 +4581,9 @@ private: bool is_valid_ = false; socket_t sock_ = INVALID_SOCKET; std::unique_ptr ws_; - time_t read_timeout_sec_ = CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND; + time_t read_timeout_sec_ = CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND; time_t read_timeout_usec_ = 0; + bool read_timeout_set_ = false; // see WebSocket::read_timeout_set_ time_t write_timeout_sec_ = CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_SECOND; time_t write_timeout_usec_ = CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_USECOND; time_t websocket_ping_interval_sec_ = @@ -4560,6 +4615,13 @@ private: #endif }; +template +inline void WebSocket::set_read_timeout( + const std::chrono::duration &duration) { + detail::duration_to_sec_and_usec( + duration, [&](time_t sec, time_t usec) { set_read_timeout(sec, usec); }); +} + template inline void WebSocketClient::set_read_timeout( const std::chrono::duration &duration) { @@ -4586,8 +4648,14 @@ namespace impl { bool is_valid_utf8(const std::string &s); -bool read_websocket_frame(Stream &strm, Opcode &opcode, std::string &payload, - bool &fin, bool expect_masked, size_t max_len); +// Three states, because a failure that consumed bytes and one that consumed +// none are not the same thing: the first has left the stream in the middle of +// a frame and the connection cannot be reused, the second can just be retried. +enum class FrameRead { Ok, Fail, Timeout }; + +FrameRead read_websocket_frame(Stream &strm, Opcode &opcode, + std::string &payload, bool &fin, + bool expect_masked, size_t max_len); } // namespace impl From e192abb406a35e9fbd2859892286144a6bda6ada Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Gallou=C3=ABt?= Date: Sat, 12 Sep 2026 11:23:54 +0200 Subject: [PATCH 30/35] server : add missing headers (#28795) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Adrien Gallouët --- tools/server/server-common.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index eade7db21..483391333 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -15,6 +15,8 @@ #include #include #include +#include +#include #ifdef _WIN32 // windows.h defines min and max as macros, which breaks std::min and std::max From c8edceb0615d859b2c0d9fa08c3ed07020ebf9b8 Mon Sep 17 00:00:00 2001 From: thelittlefireman <5165783+thelittlefireman@users.noreply.github.com> Date: Sat, 12 Sep 2026 11:26:53 +0200 Subject: [PATCH 31/35] ggml-cuda: hip add specific config table for AMD GCN (#27841) --- ggml/src/ggml-cuda/mmq-config-gcn.cuh | 281 ++++++++++++++++++++++++++ ggml/src/ggml-cuda/mmq.cuh | 8 +- 2 files changed, 288 insertions(+), 1 deletion(-) create mode 100644 ggml/src/ggml-cuda/mmq-config-gcn.cuh diff --git a/ggml/src/ggml-cuda/mmq-config-gcn.cuh b/ggml/src/ggml-cuda/mmq-config-gcn.cuh new file mode 100644 index 000000000..24af2ef2b --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-config-gcn.cuh @@ -0,0 +1,281 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_gcn(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 3, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index 24afedd14..6923f3510 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -218,6 +218,7 @@ struct ggml_cuda_mmq_config { #include "mmq-config-ampere.cuh" #include "mmq-config-blackwell.cuh" +#include "mmq-config-gcn.cuh" #include "mmq-config-cdna.cuh" #include "mmq-config-rdna2.cuh" #include "mmq-config-rdna3.cuh" @@ -228,6 +229,9 @@ struct ggml_cuda_mmq_config { static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type type, const int J, const bool fallback, const int cc) { if (GGML_CUDA_CC_IS_AMD(cc)) { + if (GGML_CUDA_CC_IS_GCN(cc)) { + return ggml_cuda_mmq_get_config_gcn(type, J, fallback); + } if (GGML_CUDA_CC_IS_CDNA(cc)) { return ggml_cuda_mmq_get_config_cdna(type, J, fallback); } @@ -256,7 +260,9 @@ static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type ty static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback) { #ifdef GGML_USE_HIP -#ifdef CDNA +#ifdef GCN + return ggml_cuda_mmq_get_config_gcn(type, J, fallback); +#elif defined(CDNA) return ggml_cuda_mmq_get_config_cdna(type, J, fallback); #elif defined(RDNA4) return ggml_cuda_mmq_get_config_rdna4(type, J, fallback); From 56381e407c0ccfb3a6f71e668a27a901001d22ce Mon Sep 17 00:00:00 2001 From: MiaoMing Chen Date: Sat, 12 Sep 2026 17:50:35 +0800 Subject: [PATCH 32/35] server : allow model downloads at model limit fix issue #26809 (#28530) --- tools/server/server-models.cpp | 3 ++- tools/server/tests/unit/test_router.py | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 4984f1be6..3d134acf3 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1129,7 +1129,8 @@ void server_models::load(const std::string & name, const load_options & opts) { // exceeding models_max. Without this, the window between unload_lru() // releasing its lock and this lock_guard acquiring allows multiple // threads to each observe capacity and all proceed to load. - if (base_params.models_max > 0) { + // Download workers do not use models_max slots. + if (opts.mode == SERVER_CHILD_MODE_NORMAL && base_params.models_max > 0) { size_t count_active = 0; for (const auto & m : mapping) { if (m.second.meta.is_running()) { diff --git a/tools/server/tests/unit/test_router.py b/tools/server/tests/unit/test_router.py index e4b7f9fe4..bae156517 100644 --- a/tools/server/tests/unit/test_router.py +++ b/tools/server/tests/unit/test_router.py @@ -540,13 +540,17 @@ def _wait_for_sse_event(collected: list, event_type: str, model: str, timeout: i def test_router_download_model(): - """Case 1: download a model, verify SSE events and GET /models.""" + """Case 1: download a model at the model limit, verify SSE events and GET /models.""" global server + server.models_max = 1 server.start() # Ensure the model is not present before we start server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}") + # A download worker must not consume or evict a model slot + _load_model_and_wait(MODEL_B, timeout=120) + sse_events: list = [] stop = threading.Event() sse_ready = threading.Event() @@ -580,6 +584,7 @@ def test_router_download_model(): # Model should now appear in GET /models ids = _get_model_ids(is_reload=False) assert MODEL_DOWNLOAD_ID in ids, f"{MODEL_DOWNLOAD_ID} not found in /models after download" + assert _get_model_status(MODEL_B) == "loaded" def test_router_delete_model(): From 3057bb66c86c46d5781e50e85462a760ba7d1feb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Gallou=C3=ABt?= Date: Sat, 12 Sep 2026 16:09:46 +0200 Subject: [PATCH 33/35] ui : add cache (#28802) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Adrien Gallouët --- scripts/ui-assets.cmake | 157 ++++++++++++++++++++++++++-------------- 1 file changed, 104 insertions(+), 53 deletions(-) diff --git a/scripts/ui-assets.cmake b/scripts/ui-assets.cmake index 402f95bd4..dec34ebda 100644 --- a/scripts/ui-assets.cmake +++ b/scripts/ui-assets.cmake @@ -21,6 +21,7 @@ set(DIST_DIR "${UI_BINARY_DIR}/dist") set(SRC_DIST_DIR "${UI_SOURCE_DIR}/dist") set(WORK_DIR "${UI_BINARY_DIR}/ui-src") set(STAMP_FILE "${UI_BINARY_DIR}/.ui-stamp") +set(EMBED_STAMP "${UI_BINARY_DIR}/.ui-embed.sha256") set(UI_CPP "${UI_BINARY_DIR}/ui.cpp") set(UI_H "${UI_BINARY_DIR}/ui.h") @@ -141,9 +142,59 @@ function(ui_validate_assets files in_dir) endfunction() # Generate ui.cpp/ui.h embedding every file of ${dist_dir} (empty table when -# it has no index.html). When LLAMA_UI_GZIP is enabled, assets are compressed -# first and served pre-gzipped (llama_ui_use_gzip()). +# it has no index.html), gzip-compressed when LLAMA_UI_GZIP is enabled. function(emit_files dist_dir) + set(UI_TEMPLATE_DIR "${LLAMA_SOURCE_DIR}/tools/ui") + + # Collect the asset list once and reuse it for the fingerprint, + # validation, compression and embedding. + set(assets "") + if(EXISTS "${dist_dir}/index.html") + file(GLOB_RECURSE assets + LIST_DIRECTORIES false + RELATIVE "${dist_dir}" + "${dist_dir}/*") + list(FILTER assets EXCLUDE REGEX "^_gzip/") + list(SORT assets) + endif() + + if(LLAMA_UI_GZIP AND NOT DEFINED ENV{SOURCE_DATE_EPOCH}) + # Zero the gzip header timestamp so identical inputs give identical + # bytes (and therefore stable ETags) on every machine. + set(ENV{SOURCE_DATE_EPOCH} 0) + endif() + + # Fingerprint of every input that determines ui.cpp/ui.h: compression + # settings, the asset tree (names + SHA-256) and this script + templates. + set(fp "${LLAMA_UI_GZIP}|$ENV{SOURCE_DATE_EPOCH}|${CMAKE_VERSION}\n") + foreach(f ${assets}) + file(SHA256 "${dist_dir}/${f}" h) + string(APPEND fp "${f} ${h}\n") + endforeach() + foreach(g + "${CMAKE_CURRENT_FUNCTION_LIST_FILE}" + "${UI_TEMPLATE_DIR}/ui.h.in" + "${UI_TEMPLATE_DIR}/ui.cpp.in") + file(SHA256 "${g}" h) + string(APPEND fp "gen ${h}\n") + endforeach() + string(SHA256 fingerprint "${fp}") + + if(EXISTS "${EMBED_STAMP}" AND EXISTS "${UI_CPP}" AND EXISTS "${UI_H}") + file(READ "${EMBED_STAMP}" fp_saved) + string(STRIP "${fp_saved}" fp_saved) + if(fp_saved STREQUAL "${fingerprint}") + message(STATUS "UI: assets unchanged, skipping embedding") + return() + endif() + endif() + + # Drop the old stamp up front so a crash mid-generation cannot leave + # outputs and stamp out of sync. + file(REMOVE "${EMBED_STAMP}") + + ui_validate_assets("${assets}" "${dist_dir}") + set(embed_dir "${dist_dir}") set(use_gzip FALSE) @@ -156,21 +207,11 @@ function(emit_files dist_dir) endif() if(LLAMA_UI_GZIP) # Compress every asset into a parallel _gzip/ tree under the build - # directory (never write into the source or dist tree); the - # structure stays the same: /abc/def --> /_gzip/abc/def. - # FORMAT raw produces a bare gzip stream (no archive container) - # that can be served with Content-Encoding: gzip. SOURCE_DATE_EPOCH - # zeroes the header timestamp so identical inputs give identical - # bytes (and therefore stable ETags) on every machine. - if(NOT DEFINED ENV{SOURCE_DATE_EPOCH}) - set(ENV{SOURCE_DATE_EPOCH} 0) - endif() + # directory, served with Content-Encoding: gzip. set(gzip_root "${UI_BINARY_DIR}/ui-gzip") set(gzip_dir "${gzip_root}/_gzip") file(REMOVE_RECURSE "${gzip_root}") - file(GLOB_RECURSE all_files RELATIVE "${dist_dir}" "${dist_dir}/*") - list(FILTER all_files EXCLUDE REGEX "^_gzip/") - foreach(f ${all_files}) + foreach(f IN LISTS assets) get_filename_component(asset_path "${dist_dir}/${f}" REALPATH) get_filename_component(dst_dir "${gzip_dir}/${f}" DIRECTORY) file(MAKE_DIRECTORY "${dst_dir}") @@ -187,21 +228,10 @@ function(emit_files dist_dir) endif() endif() - set(assets "") - if(EXISTS "${embed_dir}/index.html") - file(GLOB_RECURSE assets RELATIVE "${embed_dir}" "${embed_dir}/*") - list(FILTER assets EXCLUDE REGEX "^_gzip/") - list(SORT assets) - ui_validate_assets("${assets}" "${embed_dir}") - endif() - list(LENGTH assets n_assets) - # Only the per-asset data arrays and table rows are built here; all - # static C++ lives in the ui.h.in / ui.cpp.in templates. configure_file - # rewrites an output only when its contents change, so the library is - # not recompiled needlessly. @ONLY keeps ${...} in the content literal; - # mime types come from a fixed list. + # Per-asset arrays and table rows go into the ui.h.in / ui.cpp.in templates; + # configure_file only rewrites on content change, avoiding needless recompiles. set(ASSET_ARRAYS "") set(ASSET_TABLE "") set(idx 0) @@ -235,9 +265,11 @@ function(emit_files dist_dir) set(USE_GZIP true) endif() - set(UI_TEMPLATE_DIR "${LLAMA_SOURCE_DIR}/tools/ui") configure_file("${UI_TEMPLATE_DIR}/ui.h.in" "${UI_H}" @ONLY) configure_file("${UI_TEMPLATE_DIR}/ui.cpp.in" "${UI_CPP}" @ONLY) + + # Write the embed stamp last, after both generated files succeeded. + file(WRITE "${EMBED_STAMP}" "${fingerprint}") message(STATUS "UI: embedded ${n_assets} assets") endfunction() @@ -419,16 +451,8 @@ function(hf_download version out_var out_resolved) message(STATUS "UI: downloading from ${resolved}: ${base}/dist.tar.gz") - file(DOWNLOAD "${base}/dist.tar.gz?download=true" "${archive}" - STATUS status TIMEOUT 300 ${auth_headers} - ) - list(GET status 0 rc) - if(NOT rc EQUAL 0) - list(GET status 1 errmsg) - message(STATUS "UI: download dist.tar.gz from ${resolved} failed: ${errmsg}") - continue() - endif() - + # Fetch the checksum first: when the archive we already have matches + # it, the expensive download is skipped and only extraction repeats. file(DOWNLOAD "${base}/dist.tar.gz.sha256?download=true" "${archive}.sha256" STATUS status TIMEOUT 30 ${auth_headers} ) @@ -439,17 +463,44 @@ function(hf_download version out_var out_resolved) continue() endif() - # Validate sha256 checkums + # Validate the sha256 checksum: reject anything that is not a full + # 64-hex-digit digest before touching the archive. file(READ "${archive}.sha256" expected) string(REGEX MATCH "^[0-9a-fA-F]+" expected "${expected}") string(TOLOWER "${expected}" expected) - file(SHA256 "${archive}" actual) - if("${expected}" STREQUAL "" OR NOT "${actual}" STREQUAL "${expected}") - message(STATUS "UI: checksum mismatch for dist.tar.gz from ${resolved}") + string(LENGTH "${expected}" expected_len) + if(NOT expected_len EQUAL 64) + message(STATUS "UI: invalid checksum from ${resolved}") continue() endif() - # Clear DIST_DIR to remove stale files first + set(actual "") + if(EXISTS "${archive}") + file(SHA256 "${archive}" actual) + endif() + + if("${actual}" STREQUAL "${expected}") + message(STATUS "UI: local dist.tar.gz matches checksum from ${resolved}, skipping download") + else() + file(DOWNLOAD "${base}/dist.tar.gz?download=true" "${archive}" + STATUS status TIMEOUT 300 ${auth_headers} + ) + list(GET status 0 rc) + if(NOT rc EQUAL 0) + list(GET status 1 errmsg) + message(STATUS "UI: download dist.tar.gz from ${resolved} failed: ${errmsg}") + continue() + endif() + + file(SHA256 "${archive}" actual) + if(NOT "${actual}" STREQUAL "${expected}") + message(STATUS "UI: checksum mismatch for dist.tar.gz from ${resolved}") + continue() + endif() + endif() + + # Remove the stamp with the dist tree it describes, together. + file(REMOVE "${STAMP_FILE}") file(REMOVE_RECURSE "${DIST_DIR}") file(ARCHIVE_EXTRACT INPUT "${archive}" DESTINATION "${DIST_DIR}") @@ -495,27 +546,27 @@ endif() if(NOT provisioned AND HF_ENABLED) resolve_version(VERSION) + # Stamp a successful HF download: records bucket + requested version and + # lets later steps distinguish downloaded assets from locally built ones. + set(stamp_key "${HF_BUCKET}|${VERSION}") + set(stamp_ok FALSE) - if(EXISTS "${STAMP_FILE}" AND NOT "${VERSION}" STREQUAL "") + if(EXISTS "${STAMP_FILE}" AND EXISTS "${DIST_DIR}/index.html" AND NOT "${VERSION}" STREQUAL "") file(READ "${STAMP_FILE}" stamped) string(STRIP "${stamped}" stamped) - if("${stamped}" STREQUAL "${VERSION}") + if(stamped STREQUAL "${stamp_key}") set(stamp_ok TRUE) endif() endif() - set(have_assets FALSE) - if(EXISTS "${DIST_DIR}/index.html") - set(have_assets TRUE) - endif() - if(stamp_ok AND have_assets) - message(STATUS "UI: HF stamp '${stamped}' matches version, skipping HF fetch") + if(stamp_ok) + message(STATUS "UI: HF stamp matches '${stamp_key}', skipping HF fetch") set(provisioned TRUE) else() hf_download("${VERSION}" HF_OK HF_RESOLVED) if(HF_OK) - file(WRITE "${STAMP_FILE}" "${HF_RESOLVED}") - message(STATUS "UI: HF download succeeded, stamp updated (${HF_RESOLVED})") + file(WRITE "${STAMP_FILE}" "${stamp_key}") + message(STATUS "UI: HF download succeeded, stamp updated (${stamp_key}, resolved: ${HF_RESOLVED})") set(provisioned TRUE) else() message(STATUS "UI: HF download failed") From 737e0980fef1c2d573afedc8b00f7caf30617652 Mon Sep 17 00:00:00 2001 From: Pascal Date: Sat, 12 Sep 2026 22:47:14 +0200 Subject: [PATCH 34/35] cmake: leave the timestamp out of precompiled headers on clang (#28816) Clang stores the modification time of the precompiled header sources inside the header and refuses the header when they differ. A cached header restored from another checkout carries the timestamps of that checkout, so the build fails. The option covers the compilers ccache treats as MSVC while they are clang underneath, clang-cl and the Intel LLVM drivers. --- CMakeLists.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 86b09dfd4..4052fa3d6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -197,6 +197,16 @@ llama_option_depr(WARNING LLAMA_CURL) include("cmake/license.cmake") license_add_file("llama.cpp" "LICENSE") +# +# compile options +# + +# clang stores the modification time of the precompiled header sources inside the +# header and rejects it when they differ, so the timestamp is left out of it +add_compile_options( + "$<$:SHELL:-Xclang -fno-pch-timestamp>" + "$<$:SHELL:-Xclang -fno-pch-timestamp>") + # # 3rd-party # From ae9afff8d2c012ca760eb9c2adf41961cf6f6232 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sat, 12 Sep 2026 22:49:53 +0200 Subject: [PATCH 35/35] jinja : support dot property integer literals (#28817) --- common/jinja/runtime.cpp | 6 ++++++ tests/test-jinja.cpp | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/common/jinja/runtime.cpp b/common/jinja/runtime.cpp index 49354c7c9..252ab55de 100644 --- a/common/jinja/runtime.cpp +++ b/common/jinja/runtime.cpp @@ -842,6 +842,12 @@ value member_expression::execute_impl(context & ctx) { } else { property = this->property->execute(ctx); } + } else if (is_stmt(this->property)) { + // syntax: obj.index + property = mk_val(cast_stmt(this->property)->val); + if (property->as_int() < 0) { + throw std::runtime_error("Static member property cannot be negative"); + } } else { // syntax: obj.prop if (!is_stmt(this->property)) { diff --git a/tests/test-jinja.cpp b/tests/test-jinja.cpp index ab551d7b3..00de91ddf 100644 --- a/tests/test-jinja.cpp +++ b/tests/test-jinja.cpp @@ -398,6 +398,18 @@ static void test_expressions(testing & t) { "Bob" ); + test_template(t, "dot notation (integer property)", + "{{ {10: 'Bob'}.10 }}", + json::object(), + "Bob" + ); + + test_template(t, "dot notation (array index)", + "{{ user.10 }}", + {{"user", json::array({"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k"})}}, + "k" + ); + test_template(t, "negative float (not dot notation)", "{{ -1.0 }}", json::object(),