From 72797e89198ab564fd0e6baa54ab196e8dd1d884 Mon Sep 17 00:00:00 2001 From: Cordell Blanchard <55163549+CordellBlanchard@users.noreply.github.com> Date: Thu, 10 Sep 2026 03:06:07 -0400 Subject: [PATCH] vulkan : add command-buffer debug labels for GPU profilers (#28101) * vulkan : add command-buffer debug labels for GPU profilers Co-authored-by: gabby-zy Assisted-by: Claude Code * vulkan : close the queue debug label with the label struct --------- Co-authored-by: gabby-zy --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 135 +++++++++++++++++++++++++-- 1 file changed, 128 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 235b35c6c..cefe186fe 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -844,6 +844,8 @@ struct vk_device_struct { std::mutex compile_mutex; std::condition_variable compile_cv; + uint32_t debug_cmdbuf_idx {}; + vk::PhysicalDevice physical_device; vk::PhysicalDeviceProperties properties; std::string name; @@ -2209,6 +2211,8 @@ struct vk_context_struct { std::vector out_memcpys; std::vector memsets; + std::vector debug_labels; + vk_command_pool * p {}; }; typedef std::shared_ptr vk_context; @@ -8332,6 +8336,95 @@ template const T *push_constant_data(const std::array{1.0f, 1.0f, 1.0f, 1.0f}; + vk_instance.pfn_vkCmdBeginDebugUtilsLabelEXT(buf, reinterpret_cast(&label)); +} + +// no-op unless GGML_VK_DEBUG_MARKERS is set +struct ggml_vk_debug_label { + // at most one of these is set, depending on the scope the label was opened in + vk_context_struct * subctx {}; + vk_queue_handle * qhandle {}; + + // one region per dispatch, e.g. "matmul_q4_k_f32_f16acc_aligned_m (192,8,1)". + // RGP cannot recover the pipeline name on its own, it only has the hash + ggml_vk_debug_label(vk_context & ctx, const std::string & pipeline_name, uint32_t wg0, uint32_t wg1, uint32_t wg2) { + if (!vk_instance.debug_utils_support || ctx->s == nullptr) { + return; + } + begin(ctx, pipeline_name + " (" + std::to_string(wg0) + "," + std::to_string(wg1) + "," + std::to_string(wg2) + ")"); + } + + // one region per graph node + // fused nodes are joined with '+', e.g. "RMS_NORM+MUL+ROPE Qcur-19" + ggml_vk_debug_label(vk_context & ctx, const ggml_cgraph * cgraph, int node_idx, int n_fused) { + if (!vk_instance.debug_utils_support || ctx->s == nullptr) { + return; + } + std::string name = ggml_op_name(cgraph->nodes[node_idx]->op); + for (int i = 1; i <= n_fused; i++) { + name += "+"; + name += ggml_op_name(cgraph->nodes[node_idx + i]->op); + } + name += " "; + name += cgraph->nodes[node_idx]->name; + begin(ctx, name); + } + + // one region per graph evaluation, opened on the queue instead of a command buffer + // so it spans every submit the evaluation makes + ggml_vk_debug_label(vk_queue_handle * handle, const char * name) { + if (!vk_instance.debug_utils_support || handle == nullptr) { + return; + } + vk::DebugUtilsLabelEXT label = {}; + label.pLabelName = name; + label.color = std::array{1.0f, 1.0f, 1.0f, 1.0f}; + + qhandle = handle; + std::lock_guard guard(*qhandle); + vk_instance.pfn_vkQueueBeginDebugUtilsLabelEXT(qhandle->queue, reinterpret_cast(&label)); + } + + // call before the command buffer can end, the destructor covers the rest + void close() { + if (subctx != nullptr) { + // close on the current command buffer, which may differ from the one begin used + if (subctx->s != nullptr) { + vk_instance.pfn_vkCmdEndDebugUtilsLabelEXT(subctx->s->buffer->buf); + } + subctx->debug_labels.pop_back(); + subctx = nullptr; + } + if (qhandle != nullptr) { + std::lock_guard guard(*qhandle); + vk_instance.pfn_vkQueueEndDebugUtilsLabelEXT(qhandle->queue); + qhandle = nullptr; + } + } + + ~ggml_vk_debug_label() { + close(); + } + + ggml_vk_debug_label(const ggml_vk_debug_label &) = delete; + ggml_vk_debug_label & operator=(const ggml_vk_debug_label &) = delete; + +private: + // the constructors check this too, so the name is not built when markers are off + void begin(vk_context & ctx, const std::string & name) { + if (!vk_instance.debug_utils_support || ctx->s == nullptr) { + return; + } + subctx = ctx.get(); + subctx->debug_labels.push_back(name); + ggml_vk_cmd_label_begin(subctx->s->buffer->buf, subctx->debug_labels.back().c_str()); + } +}; + template static void ggml_vk_dispatch_pipeline(ggml_backend_vk_context* ctx, vk_context& subctx, vk_pipeline& pipeline, std::initializer_list const& descriptor_buffer_infos, const T &push_constants, std::array elements) { const uint32_t wg0 = CEIL_DIV(elements[0], pipeline->wg_denoms[0]); @@ -8361,7 +8454,10 @@ static void ggml_vk_dispatch_pipeline(ggml_backend_vk_context* ctx, vk_context& 0, { descriptor_set }, {}); - subctx->s->buffer->buf.dispatch(wg0, wg1, wg2); + { + ggml_vk_debug_label dbg(subctx, pipeline->name, wg0, wg1, wg2); + subctx->s->buffer->buf.dispatch(wg0, wg1, wg2); + } } static void ggml_vk_ctx_end(vk_context& ctx) { @@ -8370,6 +8466,15 @@ static void ggml_vk_ctx_end(vk_context& ctx) { return; } + // close open labels so this buffer is balanced; reopened in ggml_vk_ctx_begin + if (vk_instance.debug_utils_support) { + for (size_t i = 0; i < ctx->debug_labels.size(); i++) { + vk_instance.pfn_vkCmdEndDebugUtilsLabelEXT(ctx->s->buffer->buf); + } + // the enclosing per-command-buffer region + vk_instance.pfn_vkCmdEndDebugUtilsLabelEXT(ctx->s->buffer->buf); + } + ctx->s->buffer->buf.end(); ctx->s = nullptr; } @@ -8382,6 +8487,17 @@ static void ggml_vk_ctx_begin(vk_device& device, vk_context& subctx) { subctx->seqs.push_back({ ggml_vk_begin_submission(device, *subctx->p) }); subctx->s = subctx->seqs[subctx->seqs.size() - 1].data(); + + if (vk_instance.debug_utils_support) { + // outermost region, one per command buffer, so the gaps between submits stand out + const std::string name = "submit " + std::to_string(device->debug_cmdbuf_idx++); + ggml_vk_cmd_label_begin(subctx->s->buffer->buf, name.c_str()); + + // reopen labels left open when the previous command buffer was submitted + for (const std::string & label : subctx->debug_labels) { + ggml_vk_cmd_label_begin(subctx->s->buffer->buf, label.c_str()); + } + } } static vk_context ggml_vk_get_compute_ctx(ggml_backend_vk_context * ctx) { @@ -15971,6 +16087,9 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr } } + // closed explicitly below, and by the destructor on the early returns + ggml_vk_debug_label dbg(compute_ctx, cgraph, node_idx, ctx->num_additional_fused_ops); + switch (node->op) { case GGML_OP_REPEAT: ggml_vk_repeat(ctx, compute_ctx, src0, node); @@ -16375,6 +16494,9 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr return false; } + // the submit path below can end the command buffer, so close the region first + dbg.close(); + ctx->tensor_ctxs[node_idx] = compute_ctx; #if defined(GGML_VULKAN_CHECK_RESULTS) @@ -17841,14 +17963,13 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg ctx->device->diag_prev_end = -1; if (vk_instance.debug_utils_support) { - vk::DebugUtilsLabelEXT dul = {}; - dul.pLabelName = "ggml_backend_vk_graph_compute"; - dul.color = std::array{1.0f, 1.0f, 1.0f, 1.0f}; - - std::lock_guard guard(*ctx->device->compute_queue->handle); - vk_instance.pfn_vkQueueBeginDebugUtilsLabelEXT(ctx->device->compute_queue->handle->queue, reinterpret_cast(&dul)); + ctx->device->debug_cmdbuf_idx = 0; } + // queue scope, so it encloses every submit this evaluation makes. + // closed when the function returns + ggml_vk_debug_label queue_dbg(ctx->device->compute_queue->handle.get(), "ggml_backend_vk_graph_compute"); + ctx->prealloc_size_add_rms_partials_offset = 0; ctx->do_add_rms_partials = false; ctx->do_add_rms_partials_offset_calculation = false;