From 050dde50c9d70cf207db84f7224eedc491d817b2 Mon Sep 17 00:00:00 2001 From: Todor Boinovski Date: Mon, 7 Sep 2026 17:04:25 -0700 Subject: [PATCH] hexagon: add RELU and LEAKY_RELU ops (#28585) * hexagon: add RELU op * hexagon: add LEAKY_RELU op too --- ggml/src/ggml-hexagon/ggml-hexagon.cpp | 4 ++ ggml/src/ggml-hexagon/htp/htp-ops.h | 2 + ggml/src/ggml-hexagon/htp/hvx-arith.h | 89 ++++++++++++++++++++++++++ ggml/src/ggml-hexagon/htp/main.c | 2 + ggml/src/ggml-hexagon/htp/unary-ops.c | 47 +++++++++++++- ggml/src/ggml-hexagon/htp/unary-ops.h | 2 + tests/test-backend-ops.cpp | 5 ++ 7 files changed, 150 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 104201daf..a39df2a87 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -4979,6 +4979,7 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) { case GGML_OP_CONCAT: return HTP_OP_CONCAT; case GGML_OP_SCALE: return HTP_OP_SCALE; case GGML_OP_CLAMP: return HTP_OP_CLAMP; + case GGML_OP_LEAKY_RELU: return HTP_OP_LEAKY_RELU; case GGML_OP_SQR: return HTP_OP_SQR; case GGML_OP_SQRT: return HTP_OP_SQRT; case GGML_OP_LOG: return HTP_OP_UNARY_LOG; @@ -5006,6 +5007,7 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) { case GGML_UNARY_OP_SOFTPLUS: return HTP_OP_UNARY_SOFTPLUS; case GGML_UNARY_OP_TANH: return HTP_OP_UNARY_TANH; case GGML_UNARY_OP_ABS: return HTP_OP_UNARY_ABS; + case GGML_UNARY_OP_RELU: return HTP_OP_UNARY_RELU; default: break; } @@ -5871,6 +5873,7 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons case GGML_OP_RMS_NORM: case GGML_OP_SCALE: case GGML_OP_CLAMP: + case GGML_OP_LEAKY_RELU: supp = ggml_hexagon_supported_unary(sess, op); break; @@ -5899,6 +5902,7 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons case GGML_UNARY_OP_SILU: case GGML_UNARY_OP_GELU: case GGML_UNARY_OP_GELU_QUICK: + case GGML_UNARY_OP_RELU: supp = ggml_hexagon_supported_unary(sess, op); break; default: diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h index cf938f7ee..12a61b67f 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ops.h +++ b/ggml/src/ggml-hexagon/htp/htp-ops.h @@ -65,6 +65,7 @@ enum htp_op_code { HTP_OP_UNARY_TANH, HTP_OP_UNARY_ABS, HTP_OP_UNARY_LOG, + HTP_OP_UNARY_RELU, HTP_OP_GLU_SWIGLU, HTP_OP_GLU_SWIGLU_OAI, HTP_OP_GLU_GEGLU, @@ -93,6 +94,7 @@ enum htp_op_code { HTP_OP_NORM, HTP_OP_CONCAT, HTP_OP_CLAMP, + HTP_OP_LEAKY_RELU, HTP_OP_IM2COL, HTP_OP_FENCE, HTP_OP_ALLREDUCE, diff --git a/ggml/src/ggml-hexagon/htp/hvx-arith.h b/ggml/src/ggml-hexagon/htp/hvx-arith.h index 5ef746342..fe5477c1b 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-arith.h +++ b/ggml/src/ggml-hexagon/htp/hvx-arith.h @@ -308,6 +308,46 @@ static inline void hvx_min_scalar_f32(uint8_t * restrict dst, const uint8_t * re } } +// MAX Scalar variants + +#define HVX_OP_MAX_SCALAR(v) Q6_Vsf_vmax_VsfVsf(val_vec, v) + +static inline void hvx_max_scalar_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) { + const HVX_Vector val_vec = hvx_vec_splat_f32(val); + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(float), hvx_vec_store_a, HVX_OP_MAX_SCALAR); +} + +static inline void hvx_max_scalar_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) { + const HVX_Vector val_vec = hvx_vec_splat_f32(val); + assert((unsigned long) dst % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(float), hvx_vec_store_a, HVX_OP_MAX_SCALAR); +} + +static inline void hvx_max_scalar_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) { + const HVX_Vector val_vec = hvx_vec_splat_f32(val); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(float), hvx_vec_store_u, HVX_OP_MAX_SCALAR); +} + +static inline void hvx_max_scalar_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) { + const HVX_Vector val_vec = hvx_vec_splat_f32(val); + hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(float), hvx_vec_store_u, HVX_OP_MAX_SCALAR); +} + +static inline void hvx_max_scalar_f32(uint8_t * restrict dst, const uint8_t * restrict src, const float val, const int num_elems) { + if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { + hvx_max_scalar_f32_aa(dst, src, val, num_elems); + } else if (hex_is_aligned((void *) dst, 128)) { + hvx_max_scalar_f32_au(dst, src, val, num_elems); + } else if (hex_is_aligned((void *) src, 128)) { + hvx_max_scalar_f32_ua(dst, src, val, num_elems); + } else { + hvx_max_scalar_f32_uu(dst, src, val, num_elems); + } +} + // CLAMP Scalar variants #define HVX_OP_CLAMP_SCALAR(v) \ @@ -406,6 +446,53 @@ static inline void hvx_clamp_scalar_f16(uint8_t * restrict dst, const uint8_t * } } +#define HVX_OP_LEAKY_RELU_SCALAR(v) \ + ({ \ + HVX_VectorPred pred_neg = Q6_Q_vcmp_gt_VsfVsf(zero_vec, v); \ + HVX_Vector scaled = HVX_OP_MUL_F32(v, ns_vec); \ + Q6_V_vmux_QVV(pred_neg, scaled, v); \ + }) + +static inline void hvx_leaky_relu_scalar_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const float ns, uint32_t n) { + const HVX_Vector zero_vec = hvx_vec_splat_f32(0.0f); + const HVX_Vector ns_vec = hvx_vec_splat_f32(ns); + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(float), hvx_vec_store_a, HVX_OP_LEAKY_RELU_SCALAR); +} + +static inline void hvx_leaky_relu_scalar_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, const float ns, uint32_t n) { + const HVX_Vector zero_vec = hvx_vec_splat_f32(0.0f); + const HVX_Vector ns_vec = hvx_vec_splat_f32(ns); + assert((unsigned long) dst % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(float), hvx_vec_store_a, HVX_OP_LEAKY_RELU_SCALAR); +} + +static inline void hvx_leaky_relu_scalar_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, const float ns, uint32_t n) { + const HVX_Vector zero_vec = hvx_vec_splat_f32(0.0f); + const HVX_Vector ns_vec = hvx_vec_splat_f32(ns); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(float), hvx_vec_store_u, HVX_OP_LEAKY_RELU_SCALAR); +} + +static inline void hvx_leaky_relu_scalar_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, const float ns, uint32_t n) { + const HVX_Vector zero_vec = hvx_vec_splat_f32(0.0f); + const HVX_Vector ns_vec = hvx_vec_splat_f32(ns); + hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(float), hvx_vec_store_u, HVX_OP_LEAKY_RELU_SCALAR); +} + +static inline void hvx_leaky_relu_scalar_f32(uint8_t * restrict dst, const uint8_t * restrict src, const float ns, const int num_elems) { + if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { + hvx_leaky_relu_scalar_f32_aa(dst, src, ns, num_elems); + } else if (hex_is_aligned((void *) dst, 128)) { + hvx_leaky_relu_scalar_f32_au(dst, src, ns, num_elems); + } else if (hex_is_aligned((void *) src, 128)) { + hvx_leaky_relu_scalar_f32_ua(dst, src, ns, num_elems); + } else { + hvx_leaky_relu_scalar_f32_uu(dst, src, ns, num_elems); + } +} + // // Abs // @@ -627,8 +714,10 @@ static inline void hvx_sqr_f16(uint8_t * restrict dst, const uint8_t * restrict #undef HVX_OP_MUL_SCALAR_F16 #undef hvx_scalar_loop_body #undef HVX_OP_MIN_SCALAR +#undef HVX_OP_MAX_SCALAR #undef HVX_OP_CLAMP_SCALAR #undef HVX_OP_CLAMP_SCALAR_F16 +#undef HVX_OP_LEAKY_RELU_SCALAR #undef DEFINE_HVX_BINARY_OP_VARIANTS #undef HVX_BINARY_DISPATCHER #undef UNUSED diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index 3ab4613cf..be54d4fe9 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -771,6 +771,7 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_RMS_NORM_MUL: case HTP_OP_SCALE: case HTP_OP_CLAMP: + case HTP_OP_LEAKY_RELU: case HTP_OP_SQR: case HTP_OP_SQRT: case HTP_OP_UNARY_SOFTPLUS: @@ -782,6 +783,7 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_UNARY_TANH: case HTP_OP_UNARY_ABS: case HTP_OP_UNARY_LOG: + case HTP_OP_UNARY_RELU: case HTP_OP_L2_NORM: return op_unary(octx); diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c index 5e62b4a9b..7850ab27e 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.c +++ b/ggml/src/ggml-hexagon/htp/unary-ops.c @@ -156,6 +156,22 @@ static void clamp_f32(const float * restrict src, } } +static void leaky_relu_f32(const float * restrict src, + float * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float negative_slope = 0.f; + memcpy(&negative_slope, &op_params[0], sizeof(float)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_leaky_relu_scalar_f32(dst_local, src_local, negative_slope, ne0); + } +} + static void rms_norm_f32(const float * restrict src, float * restrict dst, const uint32_t num_rows, @@ -597,6 +613,20 @@ static void abs_f32(const float * restrict src, } } +static void relu_f32(const float * restrict src, + float * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_max_scalar_f32(dst_local, src_local, 0.0f, ne0); + } +} + static void log_f32(const float * restrict src, float * restrict dst, const uint32_t num_rows, @@ -774,6 +804,7 @@ DEFINE_UNARY_TASK(rms_norm, false, false, rms_norm_f32(src0_vtcm, dst_vtcm DEFINE_UNARY_TASK(rms_norm_mul, true, false, rms_norm_mul_f32(src0_vtcm, uctx->broadcast_weight ? (const float *) src1_vtcm_data : src1_vtcm, dst_vtcm, block_size, uctx)) DEFINE_UNARY_TASK(scale, false, false, scale_f32(src0_vtcm, dst_vtcm, block_size, uctx)) DEFINE_UNARY_TASK(clamp, false, false, clamp_f32(src0_vtcm, dst_vtcm, block_size, uctx)) +DEFINE_UNARY_TASK(leaky_relu, false, false, leaky_relu_f32(src0_vtcm, dst_vtcm, block_size, uctx)) DEFINE_UNARY_TASK(sqr, false, false, sqr_f32(src0_vtcm, dst_vtcm, block_size, uctx)) DEFINE_UNARY_TASK(sqrt, false, false, sqrt_f32(src0_vtcm, dst_vtcm, block_size, uctx)) DEFINE_UNARY_TASK(unary_neg, false, false, neg_f32(src0_vtcm, dst_vtcm, block_size, uctx)) @@ -785,6 +816,7 @@ DEFINE_UNARY_TASK(unary_softplus, false, false, softplus_f32(src0_vtcm, dst_vtcm DEFINE_UNARY_TASK(unary_tanh, false, false, tanh_f32(src0_vtcm, dst_vtcm, block_size, uctx)) DEFINE_UNARY_TASK(unary_abs, false, false, abs_f32(src0_vtcm, dst_vtcm, block_size, uctx)) DEFINE_UNARY_TASK(unary_log, false, false, log_f32(src0_vtcm, dst_vtcm, block_size, uctx)) +DEFINE_UNARY_TASK(unary_relu, false, false, relu_f32(src0_vtcm, dst_vtcm, block_size, uctx)) DEFINE_UNARY_TASK(l2_norm, false, false, l2_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx)) DEFINE_UNARY_TASK(tri, false, true, tri_f32(src0_vtcm, dst_vtcm, block_size, ir, uctx)) @@ -937,6 +969,12 @@ static inline void tile_clamp_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, hvx_clamp_scalar_f32(dst_vtcm, src_vtcm, min, max, tw); } +static inline void tile_leaky_relu_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw, const int32_t * op_params) { + float negative_slope = 0.f; + memcpy(&negative_slope, &op_params[0], sizeof(float)); + hvx_leaky_relu_scalar_f32(dst_vtcm, src_vtcm, negative_slope, tw); +} + static inline void tile_unary_softplus_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw) { const float * restrict sf = (const float *) src_vtcm; float * restrict df = (float *) dst_vtcm; @@ -1035,6 +1073,7 @@ static inline void tri_apply_tile_f32(const uint8_t * restrict src, uint8_t * re DEFINE_UNARY_TILED_TASK(scale, false, tile_scale_f32(dst_vtcm, src_vtcm, tw, op_params)) DEFINE_UNARY_TILED_TASK(clamp, false, tile_clamp_f32(dst_vtcm, src_vtcm, tw, op_params)) +DEFINE_UNARY_TILED_TASK(leaky_relu, false, tile_leaky_relu_f32(dst_vtcm, src_vtcm, tw, op_params)) DEFINE_UNARY_TILED_TASK(sqr, false, hvx_sqr_f32_aa(dst_vtcm, src_vtcm, tw)) DEFINE_UNARY_TILED_TASK(sqrt, false, hvx_sqrt_f32_aa(dst_vtcm, src_vtcm, tw)) DEFINE_UNARY_TILED_TASK(unary_neg, false, hvx_scale_f32_aa(dst_vtcm, src_vtcm, tw, -1.0f)) @@ -1046,6 +1085,7 @@ DEFINE_UNARY_TILED_TASK(unary_softplus, false, tile_unary_softplus_f32(dst_vtcm, DEFINE_UNARY_TILED_TASK(unary_tanh, false, hvx_tanh_f32_aa(dst_vtcm, src_vtcm, tw)) DEFINE_UNARY_TILED_TASK(unary_abs, false, hvx_abs_f32_aa(dst_vtcm, src_vtcm, tw)) DEFINE_UNARY_TILED_TASK(unary_log, false, hvx_log_f32_aa(dst_vtcm, src_vtcm, tw)) +DEFINE_UNARY_TILED_TASK(unary_relu, false, hvx_max_scalar_f32(dst_vtcm, src_vtcm, 0.0f, tw)) DEFINE_UNARY_TILED_TASK(tri, true, tri_apply_tile_f32(src_vtcm, dst_vtcm, tw, col, i01, ne0, tri_ttype)) static int execute_op_unary(struct htp_ops_context * octx) { @@ -1064,6 +1104,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_RMS_NORM_MUL: op_type = "rmsnorm-mul-f32"; break; case HTP_OP_SCALE: op_type = is_f16 ? "scale-f16" : "scale-f32"; break; case HTP_OP_CLAMP: op_type = is_f16 ? "clamp-f16" : "clamp-f32"; break; + case HTP_OP_LEAKY_RELU: op_type = "leaky-relu-f32"; break; case HTP_OP_SQR: op_type = is_f16 ? "sqr-f16" : "sqr-f32"; break; case HTP_OP_SQRT: op_type = is_f16 ? "sqrt-f16" : "sqrt-f32"; break; case HTP_OP_UNARY_NEG: op_type = "neg-f32"; break; @@ -1075,9 +1116,9 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_UNARY_TANH: op_type = "tanh-f32"; break; case HTP_OP_UNARY_ABS: op_type = is_f16 ? "abs-f16" : "abs-f32"; break; case HTP_OP_UNARY_LOG: op_type = is_f16 ? "log-f16" : "log-f32"; break; + case HTP_OP_UNARY_RELU: op_type = "relu-f32"; break; case HTP_OP_L2_NORM: op_type = is_f16 ? "l2norm-f16" : "l2norm-f32"; break; case HTP_OP_TRI: op_type = "tri-f32"; break; - default: FARF(ERROR, "Unsupported unary Op %u\n", octx->op); return HTP_STATUS_NO_SUPPORT; @@ -1190,6 +1231,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { switch (octx->op) { case HTP_OP_SCALE: task_func = unary_task_f32_tiled_scale; break; case HTP_OP_CLAMP: task_func = unary_task_f32_tiled_clamp; break; + case HTP_OP_LEAKY_RELU: task_func = unary_task_f32_tiled_leaky_relu; break; case HTP_OP_SQR: task_func = unary_task_f32_tiled_sqr; break; case HTP_OP_SQRT: task_func = unary_task_f32_tiled_sqrt; break; case HTP_OP_UNARY_NEG: task_func = unary_task_f32_tiled_unary_neg; break; @@ -1201,6 +1243,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_UNARY_TANH: task_func = unary_task_f32_tiled_unary_tanh; break; case HTP_OP_UNARY_ABS: task_func = unary_task_f32_tiled_unary_abs; break; case HTP_OP_UNARY_LOG: task_func = unary_task_f32_tiled_unary_log; break; + case HTP_OP_UNARY_RELU: task_func = unary_task_f32_tiled_unary_relu; break; case HTP_OP_TRI: task_func = unary_task_f32_tiled_tri; break; default: break; } @@ -1224,6 +1267,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_RMS_NORM_MUL: task_func = unary_task_f32_rms_norm_mul; break; case HTP_OP_SCALE: task_func = unary_task_f32_scale; break; case HTP_OP_CLAMP: task_func = unary_task_f32_clamp; break; + case HTP_OP_LEAKY_RELU: task_func = unary_task_f32_leaky_relu; break; case HTP_OP_SQR: task_func = unary_task_f32_sqr; break; case HTP_OP_SQRT: task_func = unary_task_f32_sqrt; break; case HTP_OP_UNARY_NEG: task_func = unary_task_f32_unary_neg; break; @@ -1235,6 +1279,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_UNARY_TANH: task_func = unary_task_f32_unary_tanh; break; case HTP_OP_UNARY_ABS: task_func = unary_task_f32_unary_abs; break; case HTP_OP_UNARY_LOG: task_func = unary_task_f32_unary_log; break; + case HTP_OP_UNARY_RELU: task_func = unary_task_f32_unary_relu; break; case HTP_OP_L2_NORM: task_func = unary_task_f32_l2_norm; break; case HTP_OP_TRI: task_func = unary_task_f32_tri; break; default: break; diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.h index 116a591c2..e410d7fd8 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.h +++ b/ggml/src/ggml-hexagon/htp/unary-ops.h @@ -42,6 +42,7 @@ _Static_assert(sizeof(struct htp_unary_kernel_params) <= 128, "htp_unary_kernel_ static inline bool htp_op_is_unary(uint32_t opcode) { switch (opcode) { case HTP_OP_CLAMP: + case HTP_OP_LEAKY_RELU: case HTP_OP_NORM: case HTP_OP_RMS_NORM: case HTP_OP_RMS_NORM_MUL: @@ -57,6 +58,7 @@ static inline bool htp_op_is_unary(uint32_t opcode) { case HTP_OP_UNARY_TANH: case HTP_OP_UNARY_ABS: case HTP_OP_UNARY_LOG: + case HTP_OP_UNARY_RELU: case HTP_OP_L2_NORM: case HTP_OP_TRI: return true; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 3342253d0..c335e793b 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10839,6 +10839,11 @@ static std::vector> make_test_cases_perf() { GGML_TYPE_F32, {n_kv, 512, 64, 1}, false, {2, 1, 0, 3})); } + // LEAKY_RELU at FFN activation width, for direct comparison with RELU + for (int64_t n_tokens : {512, 2048}) { + test_cases.emplace_back(new test_leaky_relu(GGML_TYPE_F32, { 17408, n_tokens, 1, 1 }, 0.1f)); + } + // Conv2d: K=CRS=NPQ=4096 matmul performance uint32_t iwh_idx = 0; uint32_t kwh_idx = 1;