hexagon: add async HMX worker

Introduce hmx-worker (dedicated thread for HMX compute) to overlap HMX
matmul with HVX dequant/DMA stages in the pipeline path, replacing the
previous synchronous HMX calls that blocked the main thread.

ggml/src/ggml-hexagon/htp/CMakeLists.txt

@@ -47,6 +47,7 @@ list(FIND HTP_HMX_VERSIONS ${DSP_VERSION} _hmx_idx)
 
 if (_hmx_idx GREATER_EQUAL 0)
     target_sources(${HTP_LIB} PRIVATE
+        hmx-worker.c
         hmx-matmul-ops.c
     )
 

ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c

@@ -22,6 +22,7 @@
 #include "htp-ctx.h"
 #include "htp-ops.h"
 
+#include "hmx-worker.h"
 #include "hmx-utils.h"
 #include "hmx-ops.h"
 #include "hmx-profile.h"
@@ -675,6 +676,39 @@ static void core_dot_chunk_fp16(__fp16 *output, const __fp16 *activation, const
     }
 }
 
+// --- Async HMX matmul job (for pipeline overlap) ---
+
+typedef struct {
+    __fp16       *output;
+    const __fp16 *activation;
+    const __fp16 *weight;
+    const __fp16 *scales;
+    int           n_row_tiles;
+    int           n_col_tiles;
+    int           n_dot_tiles;
+} hmx_matmul_job_t;
+
+static void hmx_matmul_worker_fn(void *data) {
+    hmx_matmul_job_t *job = (hmx_matmul_job_t *) data;
+    core_dot_chunk_fp16(job->output, job->activation, job->weight, job->scales,
+                        job->n_row_tiles, job->n_col_tiles, job->n_dot_tiles);
+}
+
+static inline void hmx_matmul_job_init(
+    hmx_matmul_job_t *job,
+    __fp16 *output, const __fp16 *activation, const __fp16 *weight, const __fp16 *scales,
+    int n_row_tiles, int n_col_tiles, int n_dot_tiles) {
+    job->output      = output;
+    job->activation  = activation;
+    job->weight      = weight;
+    job->scales      = scales;
+    job->n_row_tiles = n_row_tiles;
+    job->n_col_tiles = n_col_tiles;
+    job->n_dot_tiles = n_dot_tiles;
+}
+
+// --- End async HMX matmul job ---
+
 static void transfer_output_chunk_fp16_to_fp32(float *restrict dst, const __fp16 *restrict vtcm_src, int n_rows, int n_cols, int n) {
     assert(n_cols % HMX_FP16_TILE_N_COLS == 0);
     const int n_col_tiles = n_cols / HMX_FP16_TILE_N_COLS;
@@ -1256,9 +1290,8 @@ int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict ds
          use_pipeline ? "PIPELINE" : "SEQUENTIAL", m_chunk_n_rows, n_chunk_n_cols,
          (size_t)(vtcm_ptr - (uint8_t *)ctx->vtcm_base), vtcm_budget);
 
-    HAP_compute_res_hmx_lock(ctx->vtcm_rctx);
-
     if (!use_pipeline) {
+        HAP_compute_res_hmx_lock(ctx->vtcm_rctx);
         for (size_t mr = 0; mr < m; mr += m_chunk_n_rows) {
             // transfer activation matrix chunk into VTCM
             size_t n_rows = hex_smin(m - mr, m_chunk_n_rows);
@@ -1318,20 +1351,23 @@ int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict ds
                 TIMER_STOP(output_store);
             }
         }
+        HAP_compute_res_hmx_unlock(ctx->vtcm_rctx);
     } else {
         // 4-stage pipeline: DMA load (A), dequantize (B), HMX matmul (C), store (D)
-        // stage B and D (dequantize and store) are expected to be on the critical path
+        // HMX compute (C) runs on dedicated worker thread, overlapping with HVX stages (B, D).
 
         // A --> B: vtcm_qweight, 1 buffer
         // B --> C: vtcm_weight0/vtcm_weight1, 2 buffers
         // C --> D: vtcm_output0/vtcm_output1, 2 buffers
 
-        //
-        // LD ||A3|  | B3 ||
-        // MM ||    C2    ||
-        // ST || D1 |     ||
+        // Async timeline (C overlaps B+D):
+        //   main+HVX:   [A0][Act][B0][A1][sub C0][B1‖C0][A2][wait,sub C1][D0+B2‖C1][wait,sub C2][D1‖C2][wait][D2]
+        //   HMX worker:                   [████ C0 ████████][████ C1 ████████████][████ C2 ████████]
 
         int n_chunk_cnt = hmx_ceil_div(n, n_chunk_n_cols);
+        hmx_matmul_job_t job_slots[2];  // persistent double-buffered job descriptors
+
+        hmx_worker_begin(ctx->hmx_worker);
         for (size_t mr = 0; mr < m; mr += m_chunk_n_rows) {
             const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows);
 
@@ -1352,31 +1388,33 @@ int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict ds
                 transfer_activation_chunk_threaded(ctx, vtcm_activation, activation_chunk, n_rows, k, k);
             }
 
-            // prologue: B0, A1, C0, B1
+            // prologue: B0, A1, submit C0 (async), B1 (overlaps C0)
             {
-                // B0
+                // B0: wait for DMA, dequant weight chunk 0
                 dma_queue_pop(ctx->dma[0]);
                 dequantize_x4x2_weight_chunk_to_fp16_tiles(ctx, vtcm_weight_bufs[0], vtcm_qweight, n_cols_A0, k, row_stride, weight_type);
 
-                // A1
+                // A1: issue DMA for weight chunk 1
                 const size_t n_cols_A1 = hex_smin(n - 1 * n_chunk_n_cols, n_chunk_n_cols);
                 if (1 < n_chunk_cnt) {
                     const uint8_t *qweight_chunk_A1 = permuted_weight + n_chunk_n_cols * row_stride;
                     dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_qweight, qweight_chunk_A1), row_stride, row_stride, row_stride, n_cols_A1);
                 }
 
-                // C0
-                core_dot_chunk_fp16((__fp16 *) vtcm_output_bufs[0], (__fp16 *) vtcm_activation, (__fp16 *) vtcm_weight_bufs[0], vtcm_scales,
-                         hmx_ceil_div(n_rows, HMX_FP16_TILE_N_ROWS), hmx_ceil_div(n_cols_A0, HMX_FP16_TILE_N_COLS), k / HMX_FP16_TILE_N_ROWS);
+                // submit C0 (non-blocking — HMX worker executes in parallel)
+                hmx_matmul_job_init(&job_slots[0],
+                    (__fp16 *) vtcm_output_bufs[0], (__fp16 *) vtcm_activation, (__fp16 *) vtcm_weight_bufs[0], vtcm_scales,
+                    hmx_ceil_div(n_rows, HMX_FP16_TILE_N_ROWS), hmx_ceil_div(n_cols_A0, HMX_FP16_TILE_N_COLS), k / HMX_FP16_TILE_N_ROWS);
+                hmx_worker_submit(ctx->hmx_worker, hmx_matmul_worker_fn, &job_slots[0]);
 
-                // B1
+                // B1: DMA pop + dequant (runs in parallel with C0 on HMX worker)
                 if (1 < n_chunk_cnt) {
                     dma_queue_pop(ctx->dma[0]);
                     dequantize_x4x2_weight_chunk_to_fp16_tiles(ctx, vtcm_weight_bufs[1], vtcm_qweight, n_cols_A1, k, row_stride, weight_type);
                 }
             }
 
-            // main loop
+            // main loop: wait C_i → submit C_{i+1} → D_i + B_{i+2} (parallel with C_{i+1})
             for (int i = 0; i < n_chunk_cnt; ++i) {
                 const size_t nc    = i * n_chunk_n_cols;
                 const size_t nc_p1 = nc + 1 * n_chunk_n_cols;
@@ -1386,36 +1424,41 @@ int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict ds
                 const size_t n_cols_p1 = hex_smin(n - nc_p1, n_chunk_n_cols);
                 const size_t n_cols_p2 = hex_smin(n - nc_p2, n_chunk_n_cols);
 
-                // issue A_{i+2}
+                // issue A_{i+2}: DMA push (non-blocking)
                 if (i + 2 < n_chunk_cnt) {
                     const uint8_t *qweight_chunk_p2 = permuted_weight + nc_p2 * row_stride;
                     dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_qweight, qweight_chunk_p2), row_stride, row_stride, row_stride, n_cols_p2);
                 }
 
-                // wait for HMX (C_{i}) -- C_{i} is done
+                // wait C_i: block until prologue/previous C completes
+                hmx_worker_wait(ctx->hmx_worker);
 
-                // result of B_{i+1} (input of C_{i+1}) should be ready now
-
-                // issue C_{i+1}
+                // submit C_{i+1} (non-blocking, overlaps with D_i + B_{i+2} below)
+                // job_slots[(i+1)%2] is safe: C_i just completed, freeing slot i%2's
+                // counterpart — and (i+1)%2 was last used by C_{i-1} which completed
+                // before C_i was submitted.
                 if (i + 1 < n_chunk_cnt) {
-                    core_dot_chunk_fp16((__fp16 *) vtcm_output_bufs[(i + 1) % 2], (__fp16 *) vtcm_activation, (__fp16 *) vtcm_weight_bufs[(i + 1) % 2], vtcm_scales,
+                    hmx_matmul_job_init(&job_slots[(i + 1) % 2],
+                        (__fp16 *) vtcm_output_bufs[(i + 1) % 2], (__fp16 *) vtcm_activation,
+                        (__fp16 *) vtcm_weight_bufs[(i + 1) % 2], vtcm_scales,
                         hmx_ceil_div(n_rows, HMX_FP16_TILE_N_ROWS), hmx_ceil_div(n_cols_p1, HMX_FP16_TILE_N_COLS), k / HMX_FP16_TILE_N_ROWS);
+                    hmx_worker_submit(ctx->hmx_worker, hmx_matmul_worker_fn, &job_slots[(i + 1) % 2]);
                 }
 
-                // compute D_{i}
+                // D_i: store output (multi-thread HVX, parallel with C_{i+1})
                 float *output_chunk = dst + (mr * n + nc);
                 transfer_output_chunk_threaded(ctx, output_chunk, vtcm_output_bufs[i % 2], n_rows, n_cols, n);
 
-                // wait for DMA (A_{i+2}), compute B_{i+2}
+                // B_{i+2}: DMA pop + dequant (multi-thread HVX, parallel with C_{i+1})
                 if (i + 2 < n_chunk_cnt) {
                     dma_queue_pop(ctx->dma[0]);
                     dequantize_x4x2_weight_chunk_to_fp16_tiles(ctx, vtcm_weight_bufs[(i + 2) % 2], vtcm_qweight, n_cols_p2, k, row_stride, weight_type);
                 }
             }
         }
-    }
 
-    HAP_compute_res_hmx_unlock(ctx->vtcm_rctx);
+        hmx_worker_end(ctx->hmx_worker);
+    }
 
     TIMER_STOP(total);
 

ggml/src/ggml-hexagon/htp/hmx-worker.c

@@ -0,0 +1,193 @@
+#include "hmx-worker.h"
+
+#include <qurt.h>
+#include <stdatomic.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <HAP_compute_res.h>
+#include <HAP_farf.h>
+
+// ---------------------------------------------------------------------------
+// Internal types
+// ---------------------------------------------------------------------------
+
+enum hmx_worker_cmd {
+    HMX_WORKER_CMD_BEGIN,  // acquire HMX lock
+    HMX_WORKER_CMD_JOB,   // execute fn(data)
+    HMX_WORKER_CMD_END,   // release HMX lock
+    HMX_WORKER_CMD_KILL,  // exit thread
+};
+
+struct hmx_worker_context {
+    // Command channel: main thread → worker
+    atomic_uint          cmd_seqn;   // bumped by main thread for each command
+    enum hmx_worker_cmd  cmd_type;
+    hmx_worker_fn_t      fn;
+    void                *data;
+
+    // Completion channel: worker → main thread
+    atomic_uint          done_seqn;  // set to cmd_seqn when command completes
+
+    // Configuration
+    uint32_t             vtcm_rctx;
+
+    // Thread resources
+    qurt_thread_t        thread;
+    void                *stack;      // single allocation: stack + context
+};
+
+// ---------------------------------------------------------------------------
+// Worker thread entry point
+// ---------------------------------------------------------------------------
+
+static void hmx_worker_main(void *arg) {
+    struct hmx_worker_context *ctx = (struct hmx_worker_context *) arg;
+
+    FARF(HIGH, "hmx-worker: thread started");
+
+    unsigned int prev_seqn = 0;
+    for (;;) {
+        unsigned int seqn = atomic_load_explicit(&ctx->cmd_seqn, memory_order_acquire);
+        if (seqn == prev_seqn) {
+            qurt_futex_wait(&ctx->cmd_seqn, prev_seqn);
+            continue;
+        }
+        prev_seqn = seqn;
+
+        switch (ctx->cmd_type) {
+        case HMX_WORKER_CMD_BEGIN:
+            HAP_compute_res_hmx_lock(ctx->vtcm_rctx);
+            break;
+
+        case HMX_WORKER_CMD_JOB:
+            ctx->fn(ctx->data);
+            break;
+
+        case HMX_WORKER_CMD_END:
+            HAP_compute_res_hmx_unlock(ctx->vtcm_rctx);
+            break;
+
+        case HMX_WORKER_CMD_KILL:
+            atomic_store_explicit(&ctx->done_seqn, seqn, memory_order_release);
+            qurt_futex_wake(&ctx->done_seqn, 1);
+            FARF(HIGH, "hmx-worker: thread stopped");
+            return;
+        }
+
+        atomic_store_explicit(&ctx->done_seqn, seqn, memory_order_release);
+        qurt_futex_wake(&ctx->done_seqn, 1);
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+// Issue a command to the worker (non-blocking).
+static void hmx_worker_issue(struct hmx_worker_context *ctx,
+                             enum hmx_worker_cmd type,
+                             hmx_worker_fn_t fn, void *data) {
+    ctx->cmd_type = type;
+    ctx->fn       = fn;
+    ctx->data     = data;
+    atomic_fetch_add_explicit(&ctx->cmd_seqn, 1, memory_order_release);
+    qurt_futex_wake(&ctx->cmd_seqn, 1);
+}
+
+// Block until the worker has completed the most recently issued command.
+static void hmx_worker_drain(struct hmx_worker_context *ctx) {
+    unsigned int expected = atomic_load_explicit(&ctx->cmd_seqn, memory_order_acquire);
+    while (atomic_load_explicit(&ctx->done_seqn, memory_order_acquire) != expected) {
+        qurt_futex_wait(&ctx->done_seqn,
+                        atomic_load_explicit(&ctx->done_seqn, memory_order_relaxed));
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+#define LOWEST_USABLE_QURT_PRIO (254)
+
+AEEResult hmx_worker_init(hmx_worker_context_t *out, uint32_t stack_size, uint32_t vtcm_rctx) {
+    if (!out) {
+        return AEE_EBADPARM;
+    }
+
+    // Single allocation: stack followed by context struct.
+    size_t total = stack_size + sizeof(struct hmx_worker_context);
+    unsigned char *blob = (unsigned char *) malloc(total);
+    if (!blob) {
+        FARF(ERROR, "hmx-worker: allocation failed (%zu bytes)", total);
+        return AEE_ENOMEMORY;
+    }
+    memset(blob, 0, total);
+
+    struct hmx_worker_context *ctx = (struct hmx_worker_context *) (blob + stack_size);
+    ctx->stack     = blob;
+    ctx->vtcm_rctx = vtcm_rctx;
+    atomic_init(&ctx->cmd_seqn,  0);
+    atomic_init(&ctx->done_seqn, 0);
+
+    // Match caller thread priority (same pattern as worker-pool.c).
+    int prio = qurt_thread_get_priority(qurt_thread_get_id());
+    if (prio < 1)                       prio = 1;
+    if (prio > LOWEST_USABLE_QURT_PRIO) prio = LOWEST_USABLE_QURT_PRIO;
+
+    qurt_thread_attr_t attr;
+    qurt_thread_attr_init(&attr);
+    qurt_thread_attr_set_stack_addr(&attr, blob);
+    qurt_thread_attr_set_stack_size(&attr, stack_size);
+    qurt_thread_attr_set_priority(&attr, prio);
+    qurt_thread_attr_set_name(&attr, "hmx_worker");
+
+    int err = qurt_thread_create(&ctx->thread, &attr, hmx_worker_main, ctx);
+    if (err) {
+        FARF(ERROR, "hmx-worker: thread create failed (%d)", err);
+        free(blob);
+        return AEE_EQURTTHREADCREATE;
+    }
+
+    *out = ctx;
+    return AEE_SUCCESS;
+}
+
+void hmx_worker_release(hmx_worker_context_t ctx) {
+    if (!ctx) return;
+
+    // Tell the worker to exit.
+    hmx_worker_issue(ctx, HMX_WORKER_CMD_KILL, NULL, NULL);
+    hmx_worker_drain(ctx);
+
+    int status;
+    qurt_thread_join(ctx->thread, &status);
+
+    free(ctx->stack);
+}
+
+AEEResult hmx_worker_begin(hmx_worker_context_t ctx) {
+    hmx_worker_issue(ctx, HMX_WORKER_CMD_BEGIN, NULL, NULL);
+    hmx_worker_drain(ctx);  // wait until HMX lock is acquired
+    return AEE_SUCCESS;
+}
+
+AEEResult hmx_worker_submit(hmx_worker_context_t ctx, hmx_worker_fn_t fn, void *data) {
+    // Caller is expected to have called wait() for any previous job.
+    // Safety: drain any residual (should be instant in normal flow).
+    hmx_worker_drain(ctx);
+    hmx_worker_issue(ctx, HMX_WORKER_CMD_JOB, fn, data);
+    return AEE_SUCCESS;
+}
+
+AEEResult hmx_worker_wait(hmx_worker_context_t ctx) {
+    hmx_worker_drain(ctx);
+    return AEE_SUCCESS;
+}
+
+AEEResult hmx_worker_end(hmx_worker_context_t ctx) {
+    hmx_worker_drain(ctx);  // ensure no in-flight job
+    hmx_worker_issue(ctx, HMX_WORKER_CMD_END, NULL, NULL);
+    hmx_worker_drain(ctx);  // wait until HMX lock is released
+    return AEE_SUCCESS;
+}

ggml/src/ggml-hexagon/htp/hmx-worker.h

@@ -0,0 +1,54 @@
+#ifndef HMX_WORKER_H
+#define HMX_WORKER_H
+
+// Async HMX worker: single dedicated thread for HMX compute,
+// allowing the main thread to run HVX/DMA work in parallel.
+//
+// Lifecycle per matmul op:
+//   hmx_worker_begin  — worker thread acquires HMX lock
+//   hmx_worker_submit — fire a job (non-blocking)
+//   hmx_worker_wait   — block until current job completes
+//   ...               — repeat submit/wait as needed
+//   hmx_worker_end    — worker thread releases HMX lock
+//
+// Design: single-producer single-consumer, 1 in-flight job max.
+
+#include <AEEStdDef.h>
+#include <AEEStdErr.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef void (*hmx_worker_fn_t)(void *data);
+
+typedef struct hmx_worker_context *hmx_worker_context_t;
+
+// Create worker thread.  Thread starts idle (no HMX lock held).
+AEEResult hmx_worker_init(hmx_worker_context_t *ctx, uint32_t stack_size, uint32_t vtcm_rctx);
+
+// Destroy worker thread.  Must not be called while a job is in-flight.
+void hmx_worker_release(hmx_worker_context_t ctx);
+
+// Worker thread acquires HMX lock.  Blocks until lock is held.
+AEEResult hmx_worker_begin(hmx_worker_context_t ctx);
+
+// Submit a job (non-blocking).  Caller must have called wait() for any
+// previous job before submitting a new one.
+// |data| must remain valid until the corresponding wait() returns.
+AEEResult hmx_worker_submit(hmx_worker_context_t ctx, hmx_worker_fn_t fn, void *data);
+
+// Block until the current in-flight job completes.
+// Returns immediately if no job is in-flight.
+AEEResult hmx_worker_wait(hmx_worker_context_t ctx);
+
+// Ensure no in-flight job, then worker thread releases HMX lock.
+// Blocks until unlock is complete.
+AEEResult hmx_worker_end(hmx_worker_context_t ctx);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* HMX_WORKER_H */

ggml/src/ggml-hexagon/htp/htp-ctx.h

@@ -3,6 +3,7 @@
 
 #include "hex-dma.h"
 #include "htp-ops.h"
+#include "hmx-worker.h"
 #include "worker-pool.h"
 
 #include <assert.h>
@@ -72,6 +73,10 @@ struct htp_context {
     atomic_bool            vtcm_needs_release;
 
     struct htp_ops_context octx;
+
+#ifdef HTP_HAS_HMX
+    hmx_worker_context_t   hmx_worker; // Async HMX worker for pipeline overlap
+#endif
 };
 
 int op_matmul(struct htp_ops_context * octx);

ggml/src/ggml-hexagon/htp/main.c

@@ -324,6 +324,14 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32 sess_id, uint64 dsp_que
 
 #ifdef HTP_HAS_HMX
     ctx->hmx_enabled = use_hmx;
+    ctx->hmx_worker  = NULL;
+    if (use_hmx) {
+        AEEResult hmx_worker_err = hmx_worker_init(&ctx->hmx_worker, 8192, ctx->vtcm_rctx);
+        if (hmx_worker_err != AEE_SUCCESS) {
+            FARF(ERROR, "hmx_worker_init failed: %d", hmx_worker_err);
+            return hmx_worker_err;
+        }
+    }
     FARF(HIGH, "HMX %s (use_hmx=%d)", ctx->hmx_enabled ? "enabled" : "disabled", use_hmx);
 #endif
 
@@ -389,7 +397,13 @@ AEEResult htp_iface_stop(remote_handle64 handle) {
     }
 
 #ifdef HTP_HAS_HMX
-    ctx->hmx_enabled = 0;
+    if (ctx->hmx_enabled) {
+        if (ctx->hmx_worker) {
+            hmx_worker_release(ctx->hmx_worker);
+            ctx->hmx_worker = NULL;
+        }
+        ctx->hmx_enabled = 0;
+    }
 #endif
 
     vtcm_free(ctx);