diff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txt index 2b60f427ad..1cddd6a2c0 100644 --- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt +++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt @@ -47,6 +47,7 @@ list(FIND HTP_HMX_VERSIONS ${DSP_VERSION} _hmx_idx) if (_hmx_idx GREATER_EQUAL 0) target_sources(${HTP_LIB} PRIVATE + hmx-worker.c hmx-matmul-ops.c ) diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c index ec191c1498..288a4caa6a 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c @@ -22,6 +22,7 @@ #include "htp-ctx.h" #include "htp-ops.h" +#include "hmx-worker.h" #include "hmx-utils.h" #include "hmx-ops.h" #include "hmx-profile.h" @@ -675,6 +676,39 @@ static void core_dot_chunk_fp16(__fp16 *output, const __fp16 *activation, const } } +// --- Async HMX matmul job (for pipeline overlap) --- + +typedef struct { + __fp16 *output; + const __fp16 *activation; + const __fp16 *weight; + const __fp16 *scales; + int n_row_tiles; + int n_col_tiles; + int n_dot_tiles; +} hmx_matmul_job_t; + +static void hmx_matmul_worker_fn(void *data) { + hmx_matmul_job_t *job = (hmx_matmul_job_t *) data; + core_dot_chunk_fp16(job->output, job->activation, job->weight, job->scales, + job->n_row_tiles, job->n_col_tiles, job->n_dot_tiles); +} + +static inline void hmx_matmul_job_init( + hmx_matmul_job_t *job, + __fp16 *output, const __fp16 *activation, const __fp16 *weight, const __fp16 *scales, + int n_row_tiles, int n_col_tiles, int n_dot_tiles) { + job->output = output; + job->activation = activation; + job->weight = weight; + job->scales = scales; + job->n_row_tiles = n_row_tiles; + job->n_col_tiles = n_col_tiles; + job->n_dot_tiles = n_dot_tiles; +} + +// --- End async HMX matmul job --- + static void transfer_output_chunk_fp16_to_fp32(float *restrict dst, const __fp16 *restrict vtcm_src, int n_rows, int n_cols, int n) { assert(n_cols % HMX_FP16_TILE_N_COLS == 0); const int n_col_tiles = n_cols / HMX_FP16_TILE_N_COLS; @@ -1256,9 +1290,8 @@ int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict ds use_pipeline ? "PIPELINE" : "SEQUENTIAL", m_chunk_n_rows, n_chunk_n_cols, (size_t)(vtcm_ptr - (uint8_t *)ctx->vtcm_base), vtcm_budget); - HAP_compute_res_hmx_lock(ctx->vtcm_rctx); - if (!use_pipeline) { + HAP_compute_res_hmx_lock(ctx->vtcm_rctx); for (size_t mr = 0; mr < m; mr += m_chunk_n_rows) { // transfer activation matrix chunk into VTCM size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); @@ -1318,20 +1351,23 @@ int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict ds TIMER_STOP(output_store); } } + HAP_compute_res_hmx_unlock(ctx->vtcm_rctx); } else { // 4-stage pipeline: DMA load (A), dequantize (B), HMX matmul (C), store (D) - // stage B and D (dequantize and store) are expected to be on the critical path + // HMX compute (C) runs on dedicated worker thread, overlapping with HVX stages (B, D). // A --> B: vtcm_qweight, 1 buffer // B --> C: vtcm_weight0/vtcm_weight1, 2 buffers // C --> D: vtcm_output0/vtcm_output1, 2 buffers - // - // LD ||A3| | B3 || - // MM || C2 || - // ST || D1 | || + // Async timeline (C overlaps B+D): + // main+HVX: [A0][Act][B0][A1][sub C0][B1‖C0][A2][wait,sub C1][D0+B2‖C1][wait,sub C2][D1‖C2][wait][D2] + // HMX worker: [████ C0 ████████][████ C1 ████████████][████ C2 ████████] int n_chunk_cnt = hmx_ceil_div(n, n_chunk_n_cols); + hmx_matmul_job_t job_slots[2]; // persistent double-buffered job descriptors + + hmx_worker_begin(ctx->hmx_worker); for (size_t mr = 0; mr < m; mr += m_chunk_n_rows) { const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); @@ -1352,31 +1388,33 @@ int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict ds transfer_activation_chunk_threaded(ctx, vtcm_activation, activation_chunk, n_rows, k, k); } - // prologue: B0, A1, C0, B1 + // prologue: B0, A1, submit C0 (async), B1 (overlaps C0) { - // B0 + // B0: wait for DMA, dequant weight chunk 0 dma_queue_pop(ctx->dma[0]); dequantize_x4x2_weight_chunk_to_fp16_tiles(ctx, vtcm_weight_bufs[0], vtcm_qweight, n_cols_A0, k, row_stride, weight_type); - // A1 + // A1: issue DMA for weight chunk 1 const size_t n_cols_A1 = hex_smin(n - 1 * n_chunk_n_cols, n_chunk_n_cols); if (1 < n_chunk_cnt) { const uint8_t *qweight_chunk_A1 = permuted_weight + n_chunk_n_cols * row_stride; dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_qweight, qweight_chunk_A1), row_stride, row_stride, row_stride, n_cols_A1); } - // C0 - core_dot_chunk_fp16((__fp16 *) vtcm_output_bufs[0], (__fp16 *) vtcm_activation, (__fp16 *) vtcm_weight_bufs[0], vtcm_scales, - hmx_ceil_div(n_rows, HMX_FP16_TILE_N_ROWS), hmx_ceil_div(n_cols_A0, HMX_FP16_TILE_N_COLS), k / HMX_FP16_TILE_N_ROWS); + // submit C0 (non-blocking — HMX worker executes in parallel) + hmx_matmul_job_init(&job_slots[0], + (__fp16 *) vtcm_output_bufs[0], (__fp16 *) vtcm_activation, (__fp16 *) vtcm_weight_bufs[0], vtcm_scales, + hmx_ceil_div(n_rows, HMX_FP16_TILE_N_ROWS), hmx_ceil_div(n_cols_A0, HMX_FP16_TILE_N_COLS), k / HMX_FP16_TILE_N_ROWS); + hmx_worker_submit(ctx->hmx_worker, hmx_matmul_worker_fn, &job_slots[0]); - // B1 + // B1: DMA pop + dequant (runs in parallel with C0 on HMX worker) if (1 < n_chunk_cnt) { dma_queue_pop(ctx->dma[0]); dequantize_x4x2_weight_chunk_to_fp16_tiles(ctx, vtcm_weight_bufs[1], vtcm_qweight, n_cols_A1, k, row_stride, weight_type); } } - // main loop + // main loop: wait C_i → submit C_{i+1} → D_i + B_{i+2} (parallel with C_{i+1}) for (int i = 0; i < n_chunk_cnt; ++i) { const size_t nc = i * n_chunk_n_cols; const size_t nc_p1 = nc + 1 * n_chunk_n_cols; @@ -1386,36 +1424,41 @@ int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict ds const size_t n_cols_p1 = hex_smin(n - nc_p1, n_chunk_n_cols); const size_t n_cols_p2 = hex_smin(n - nc_p2, n_chunk_n_cols); - // issue A_{i+2} + // issue A_{i+2}: DMA push (non-blocking) if (i + 2 < n_chunk_cnt) { const uint8_t *qweight_chunk_p2 = permuted_weight + nc_p2 * row_stride; dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_qweight, qweight_chunk_p2), row_stride, row_stride, row_stride, n_cols_p2); } - // wait for HMX (C_{i}) -- C_{i} is done + // wait C_i: block until prologue/previous C completes + hmx_worker_wait(ctx->hmx_worker); - // result of B_{i+1} (input of C_{i+1}) should be ready now - - // issue C_{i+1} + // submit C_{i+1} (non-blocking, overlaps with D_i + B_{i+2} below) + // job_slots[(i+1)%2] is safe: C_i just completed, freeing slot i%2's + // counterpart — and (i+1)%2 was last used by C_{i-1} which completed + // before C_i was submitted. if (i + 1 < n_chunk_cnt) { - core_dot_chunk_fp16((__fp16 *) vtcm_output_bufs[(i + 1) % 2], (__fp16 *) vtcm_activation, (__fp16 *) vtcm_weight_bufs[(i + 1) % 2], vtcm_scales, + hmx_matmul_job_init(&job_slots[(i + 1) % 2], + (__fp16 *) vtcm_output_bufs[(i + 1) % 2], (__fp16 *) vtcm_activation, + (__fp16 *) vtcm_weight_bufs[(i + 1) % 2], vtcm_scales, hmx_ceil_div(n_rows, HMX_FP16_TILE_N_ROWS), hmx_ceil_div(n_cols_p1, HMX_FP16_TILE_N_COLS), k / HMX_FP16_TILE_N_ROWS); + hmx_worker_submit(ctx->hmx_worker, hmx_matmul_worker_fn, &job_slots[(i + 1) % 2]); } - // compute D_{i} + // D_i: store output (multi-thread HVX, parallel with C_{i+1}) float *output_chunk = dst + (mr * n + nc); transfer_output_chunk_threaded(ctx, output_chunk, vtcm_output_bufs[i % 2], n_rows, n_cols, n); - // wait for DMA (A_{i+2}), compute B_{i+2} + // B_{i+2}: DMA pop + dequant (multi-thread HVX, parallel with C_{i+1}) if (i + 2 < n_chunk_cnt) { dma_queue_pop(ctx->dma[0]); dequantize_x4x2_weight_chunk_to_fp16_tiles(ctx, vtcm_weight_bufs[(i + 2) % 2], vtcm_qweight, n_cols_p2, k, row_stride, weight_type); } } } - } - HAP_compute_res_hmx_unlock(ctx->vtcm_rctx); + hmx_worker_end(ctx->hmx_worker); + } TIMER_STOP(total); diff --git a/ggml/src/ggml-hexagon/htp/hmx-worker.c b/ggml/src/ggml-hexagon/htp/hmx-worker.c new file mode 100644 index 0000000000..657d16b2a9 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hmx-worker.c @@ -0,0 +1,193 @@ +#include "hmx-worker.h" + +#include +#include +#include +#include + +#include +#include + +// --------------------------------------------------------------------------- +// Internal types +// --------------------------------------------------------------------------- + +enum hmx_worker_cmd { + HMX_WORKER_CMD_BEGIN, // acquire HMX lock + HMX_WORKER_CMD_JOB, // execute fn(data) + HMX_WORKER_CMD_END, // release HMX lock + HMX_WORKER_CMD_KILL, // exit thread +}; + +struct hmx_worker_context { + // Command channel: main thread → worker + atomic_uint cmd_seqn; // bumped by main thread for each command + enum hmx_worker_cmd cmd_type; + hmx_worker_fn_t fn; + void *data; + + // Completion channel: worker → main thread + atomic_uint done_seqn; // set to cmd_seqn when command completes + + // Configuration + uint32_t vtcm_rctx; + + // Thread resources + qurt_thread_t thread; + void *stack; // single allocation: stack + context +}; + +// --------------------------------------------------------------------------- +// Worker thread entry point +// --------------------------------------------------------------------------- + +static void hmx_worker_main(void *arg) { + struct hmx_worker_context *ctx = (struct hmx_worker_context *) arg; + + FARF(HIGH, "hmx-worker: thread started"); + + unsigned int prev_seqn = 0; + for (;;) { + unsigned int seqn = atomic_load_explicit(&ctx->cmd_seqn, memory_order_acquire); + if (seqn == prev_seqn) { + qurt_futex_wait(&ctx->cmd_seqn, prev_seqn); + continue; + } + prev_seqn = seqn; + + switch (ctx->cmd_type) { + case HMX_WORKER_CMD_BEGIN: + HAP_compute_res_hmx_lock(ctx->vtcm_rctx); + break; + + case HMX_WORKER_CMD_JOB: + ctx->fn(ctx->data); + break; + + case HMX_WORKER_CMD_END: + HAP_compute_res_hmx_unlock(ctx->vtcm_rctx); + break; + + case HMX_WORKER_CMD_KILL: + atomic_store_explicit(&ctx->done_seqn, seqn, memory_order_release); + qurt_futex_wake(&ctx->done_seqn, 1); + FARF(HIGH, "hmx-worker: thread stopped"); + return; + } + + atomic_store_explicit(&ctx->done_seqn, seqn, memory_order_release); + qurt_futex_wake(&ctx->done_seqn, 1); + } +} + +// --------------------------------------------------------------------------- +// Internal helpers +// --------------------------------------------------------------------------- + +// Issue a command to the worker (non-blocking). +static void hmx_worker_issue(struct hmx_worker_context *ctx, + enum hmx_worker_cmd type, + hmx_worker_fn_t fn, void *data) { + ctx->cmd_type = type; + ctx->fn = fn; + ctx->data = data; + atomic_fetch_add_explicit(&ctx->cmd_seqn, 1, memory_order_release); + qurt_futex_wake(&ctx->cmd_seqn, 1); +} + +// Block until the worker has completed the most recently issued command. +static void hmx_worker_drain(struct hmx_worker_context *ctx) { + unsigned int expected = atomic_load_explicit(&ctx->cmd_seqn, memory_order_acquire); + while (atomic_load_explicit(&ctx->done_seqn, memory_order_acquire) != expected) { + qurt_futex_wait(&ctx->done_seqn, + atomic_load_explicit(&ctx->done_seqn, memory_order_relaxed)); + } +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +#define LOWEST_USABLE_QURT_PRIO (254) + +AEEResult hmx_worker_init(hmx_worker_context_t *out, uint32_t stack_size, uint32_t vtcm_rctx) { + if (!out) { + return AEE_EBADPARM; + } + + // Single allocation: stack followed by context struct. + size_t total = stack_size + sizeof(struct hmx_worker_context); + unsigned char *blob = (unsigned char *) malloc(total); + if (!blob) { + FARF(ERROR, "hmx-worker: allocation failed (%zu bytes)", total); + return AEE_ENOMEMORY; + } + memset(blob, 0, total); + + struct hmx_worker_context *ctx = (struct hmx_worker_context *) (blob + stack_size); + ctx->stack = blob; + ctx->vtcm_rctx = vtcm_rctx; + atomic_init(&ctx->cmd_seqn, 0); + atomic_init(&ctx->done_seqn, 0); + + // Match caller thread priority (same pattern as worker-pool.c). + int prio = qurt_thread_get_priority(qurt_thread_get_id()); + if (prio < 1) prio = 1; + if (prio > LOWEST_USABLE_QURT_PRIO) prio = LOWEST_USABLE_QURT_PRIO; + + qurt_thread_attr_t attr; + qurt_thread_attr_init(&attr); + qurt_thread_attr_set_stack_addr(&attr, blob); + qurt_thread_attr_set_stack_size(&attr, stack_size); + qurt_thread_attr_set_priority(&attr, prio); + qurt_thread_attr_set_name(&attr, "hmx_worker"); + + int err = qurt_thread_create(&ctx->thread, &attr, hmx_worker_main, ctx); + if (err) { + FARF(ERROR, "hmx-worker: thread create failed (%d)", err); + free(blob); + return AEE_EQURTTHREADCREATE; + } + + *out = ctx; + return AEE_SUCCESS; +} + +void hmx_worker_release(hmx_worker_context_t ctx) { + if (!ctx) return; + + // Tell the worker to exit. + hmx_worker_issue(ctx, HMX_WORKER_CMD_KILL, NULL, NULL); + hmx_worker_drain(ctx); + + int status; + qurt_thread_join(ctx->thread, &status); + + free(ctx->stack); +} + +AEEResult hmx_worker_begin(hmx_worker_context_t ctx) { + hmx_worker_issue(ctx, HMX_WORKER_CMD_BEGIN, NULL, NULL); + hmx_worker_drain(ctx); // wait until HMX lock is acquired + return AEE_SUCCESS; +} + +AEEResult hmx_worker_submit(hmx_worker_context_t ctx, hmx_worker_fn_t fn, void *data) { + // Caller is expected to have called wait() for any previous job. + // Safety: drain any residual (should be instant in normal flow). + hmx_worker_drain(ctx); + hmx_worker_issue(ctx, HMX_WORKER_CMD_JOB, fn, data); + return AEE_SUCCESS; +} + +AEEResult hmx_worker_wait(hmx_worker_context_t ctx) { + hmx_worker_drain(ctx); + return AEE_SUCCESS; +} + +AEEResult hmx_worker_end(hmx_worker_context_t ctx) { + hmx_worker_drain(ctx); // ensure no in-flight job + hmx_worker_issue(ctx, HMX_WORKER_CMD_END, NULL, NULL); + hmx_worker_drain(ctx); // wait until HMX lock is released + return AEE_SUCCESS; +} diff --git a/ggml/src/ggml-hexagon/htp/hmx-worker.h b/ggml/src/ggml-hexagon/htp/hmx-worker.h new file mode 100644 index 0000000000..9c0477b974 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hmx-worker.h @@ -0,0 +1,54 @@ +#ifndef HMX_WORKER_H +#define HMX_WORKER_H + +// Async HMX worker: single dedicated thread for HMX compute, +// allowing the main thread to run HVX/DMA work in parallel. +// +// Lifecycle per matmul op: +// hmx_worker_begin — worker thread acquires HMX lock +// hmx_worker_submit — fire a job (non-blocking) +// hmx_worker_wait — block until current job completes +// ... — repeat submit/wait as needed +// hmx_worker_end — worker thread releases HMX lock +// +// Design: single-producer single-consumer, 1 in-flight job max. + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void (*hmx_worker_fn_t)(void *data); + +typedef struct hmx_worker_context *hmx_worker_context_t; + +// Create worker thread. Thread starts idle (no HMX lock held). +AEEResult hmx_worker_init(hmx_worker_context_t *ctx, uint32_t stack_size, uint32_t vtcm_rctx); + +// Destroy worker thread. Must not be called while a job is in-flight. +void hmx_worker_release(hmx_worker_context_t ctx); + +// Worker thread acquires HMX lock. Blocks until lock is held. +AEEResult hmx_worker_begin(hmx_worker_context_t ctx); + +// Submit a job (non-blocking). Caller must have called wait() for any +// previous job before submitting a new one. +// |data| must remain valid until the corresponding wait() returns. +AEEResult hmx_worker_submit(hmx_worker_context_t ctx, hmx_worker_fn_t fn, void *data); + +// Block until the current in-flight job completes. +// Returns immediately if no job is in-flight. +AEEResult hmx_worker_wait(hmx_worker_context_t ctx); + +// Ensure no in-flight job, then worker thread releases HMX lock. +// Blocks until unlock is complete. +AEEResult hmx_worker_end(hmx_worker_context_t ctx); + +#ifdef __cplusplus +} +#endif + +#endif /* HMX_WORKER_H */ diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.h index 4c36a6ea0c..7b09bb4f41 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ctx.h +++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h @@ -3,6 +3,7 @@ #include "hex-dma.h" #include "htp-ops.h" +#include "hmx-worker.h" #include "worker-pool.h" #include @@ -72,6 +73,10 @@ struct htp_context { atomic_bool vtcm_needs_release; struct htp_ops_context octx; + +#ifdef HTP_HAS_HMX + hmx_worker_context_t hmx_worker; // Async HMX worker for pipeline overlap +#endif }; int op_matmul(struct htp_ops_context * octx); diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index 8b34703942..b6ca127c1d 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -324,6 +324,14 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32 sess_id, uint64 dsp_que #ifdef HTP_HAS_HMX ctx->hmx_enabled = use_hmx; + ctx->hmx_worker = NULL; + if (use_hmx) { + AEEResult hmx_worker_err = hmx_worker_init(&ctx->hmx_worker, 8192, ctx->vtcm_rctx); + if (hmx_worker_err != AEE_SUCCESS) { + FARF(ERROR, "hmx_worker_init failed: %d", hmx_worker_err); + return hmx_worker_err; + } + } FARF(HIGH, "HMX %s (use_hmx=%d)", ctx->hmx_enabled ? "enabled" : "disabled", use_hmx); #endif @@ -389,7 +397,13 @@ AEEResult htp_iface_stop(remote_handle64 handle) { } #ifdef HTP_HAS_HMX - ctx->hmx_enabled = 0; + if (ctx->hmx_enabled) { + if (ctx->hmx_worker) { + hmx_worker_release(ctx->hmx_worker); + ctx->hmx_worker = NULL; + } + ctx->hmx_enabled = 0; + } #endif vtcm_free(ctx);