From f027c4f1b025e05d6a2fc3b741047bda07b85ef7 Mon Sep 17 00:00:00 2001 From: cqderek Date: Thu, 3 Sep 2026 03:59:36 +0800 Subject: [PATCH] ggml-hexagon: add F16 support for unary ops (#28228) Extend the HTP backend's F16 unary op coverage to include ABS on top of the existing NORM/RMS_NORM/L2_NORM/SCALE/CLAMP/SQR/SQRT set. - Add hvx_abs_f16_{aa,au,ua,uu} + dispatcher in hvx-arith.h, mirroring the sqr_f16 kernel structure and using the existing hvx_vec_abs_f16() sign-bit-clear helper - Add abs_f16() row-wise dispatch and DEFINE_UNARY_TASK_F16(unary_abs, ...) in unary-ops.c, wired into execute_op_unary()'s op_type/task_func switches - Register HTP_OP_UNARY_ABS in htp_op_is_unary() (unary-ops.h) so that ggml_hexagon_precompute_unary_params() fills kernel_params (n_threads, VTCM layout) for ABS nodes -- required for the F16 path to function - Narrow the F16 GGML_OP_UNARY gate in ggml_hexagon_supported_unary() (ggml-hexagon.cpp) to allow GGML_UNARY_OP_ABS specifically, instead of rejecting all GGML_OP_UNARY ops for F16 - Merge the separate execute_op_unary_f32()/execute_op_unary_f16() functions into a single execute_op_unary(), branching on an is_f16 flag for the parts that actually differ by type (elem_size, the early F16 op-support check, and which task_func table to use) while keeping the F32-only tiled/RMS_NORM_MUL paths intact -- per review feedback to avoid duplicating the shared VTCM/DMA plumbing Verified on-device (QRD8850, Hexagon v81) via test-backend-ops -o ABS: 8/8 passing (F16 + F32, HTP0, no CPU fallback). Regression-checked SQR/CLAMP/SQRT (F16+F32) and NORM/RMS_NORM/L2_NORM/SCALE (F32; their F16 paths have no CPU reference kernel in test-backend-ops and cannot be correctness-tested there independent of this change). --- ggml/src/ggml-hexagon/ggml-hexagon.cpp | 38 +++- ggml/src/ggml-hexagon/htp/hvx-arith.h | 171 +++++++++++++++- ggml/src/ggml-hexagon/htp/hvx-log.h | 29 +++ ggml/src/ggml-hexagon/htp/hvx-norm.h | 197 +++++++++++++++++++ ggml/src/ggml-hexagon/htp/hvx-scale.h | 66 +++++++ ggml/src/ggml-hexagon/htp/hvx-sqrt.h | 63 ++++++ ggml/src/ggml-hexagon/htp/unary-ops.c | 260 ++++++++++++++++++++++--- ggml/src/ggml-hexagon/htp/unary-ops.h | 17 +- 8 files changed, 797 insertions(+), 44 deletions(-) diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 04fb9a223..104201daf 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -4005,8 +4005,10 @@ static void ggml_hexagon_precompute_unary_params( kparams->n_threads = n_threads; - const size_t src0_data_row_size = src0->ne[0] * sizeof(float); - const size_t dst_data_row_size = dst->ne[0] * sizeof(float); + const size_t elem_size = ggml_type_size(src0->type); + + const size_t src0_data_row_size = src0->ne[0] * elem_size; + const size_t dst_data_row_size = dst->ne[0] * ggml_type_size(dst->type); const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128); const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, 128); @@ -4020,7 +4022,7 @@ static void ggml_hexagon_precompute_unary_params( if (op == HTP_OP_RMS_NORM_MUL) { GGML_ASSERT(src1 != nullptr); - src1_data_row_size = src1->ne[0] * sizeof(float); + src1_data_row_size = src1->ne[0] * ggml_type_size(src1->type); src1_row_size_aligned = hex_round_up(src1_data_row_size, 128); broadcast_weight = (src1->ne[1] * src1->ne[2] * src1->ne[3] == 1); } @@ -4034,7 +4036,7 @@ static void ggml_hexagon_precompute_unary_params( htp_unary_vtcm_layout_build(&L, op, src0->ne[0], dst->ne[0], op == HTP_OP_RMS_NORM_MUL ? src1->ne[0] : 0, - broadcast_weight, n_threads, sess->vtcm_size, + broadcast_weight, n_threads, sess->vtcm_size, elem_size, &col_tile, &vtcm_row_per_thread); kparams->col_tile = col_tile; @@ -4451,15 +4453,39 @@ static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * ses const struct ggml_tensor * src0 = op->src[0]; const struct ggml_tensor * dst = op; - if (src0->type != GGML_TYPE_F32) { + if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) { return false; } - if (dst->type != GGML_TYPE_F32) { + if (dst->type != src0->type) { return false; } if (!ggml_is_contiguous_rows(src0)) { return false; } + + // F16 device kernels only cover this explicit whitelist (must stay in sync with + // the is_f16 whitelist in execute_op_unary(), unary-ops.c). + if (src0->type == GGML_TYPE_F16) { + switch (op->op) { + case GGML_OP_NORM: + case GGML_OP_RMS_NORM: + case GGML_OP_L2_NORM: + case GGML_OP_SCALE: + case GGML_OP_CLAMP: + case GGML_OP_SQR: + case GGML_OP_SQRT: + case GGML_OP_LOG: + break; + case GGML_OP_UNARY: + if (ggml_get_unary_op(op) != GGML_UNARY_OP_ABS) { + return false; + } + break; + default: + return false; + } + } + if (!ggml_are_same_shape(src0, dst)) { return false; } diff --git a/ggml/src/ggml-hexagon/htp/hvx-arith.h b/ggml/src/ggml-hexagon/htp/hvx-arith.h index 765c35776..5ef746342 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-arith.h +++ b/ggml/src/ggml-hexagon/htp/hvx-arith.h @@ -358,6 +358,54 @@ static inline void hvx_clamp_scalar_f32(uint8_t * restrict dst, const uint8_t * } } +#define HVX_OP_CLAMP_SCALAR_F16(v) \ + ({ \ + HVX_VectorPred pred_cap_right = Q6_Q_vcmp_gt_VhfVhf(v, max_vec); \ + HVX_VectorPred pred_cap_left = Q6_Q_vcmp_gt_VhfVhf(min_vec, v); \ + HVX_Vector tmp = Q6_V_vmux_QVV(pred_cap_right, max_vec, v); \ + Q6_V_vmux_QVV(pred_cap_left, min_vec, tmp); \ + }) + +static inline void hvx_clamp_scalar_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) { + const HVX_Vector min_vec = hvx_vec_splat_f16(min); + const HVX_Vector max_vec = hvx_vec_splat_f16(max); + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_CLAMP_SCALAR_F16); +} + +static inline void hvx_clamp_scalar_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) { + const HVX_Vector min_vec = hvx_vec_splat_f16(min); + const HVX_Vector max_vec = hvx_vec_splat_f16(max); + assert((unsigned long) dst % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_CLAMP_SCALAR_F16); +} + +static inline void hvx_clamp_scalar_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) { + const HVX_Vector min_vec = hvx_vec_splat_f16(min); + const HVX_Vector max_vec = hvx_vec_splat_f16(max); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_CLAMP_SCALAR_F16); +} + +static inline void hvx_clamp_scalar_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) { + const HVX_Vector min_vec = hvx_vec_splat_f16(min); + const HVX_Vector max_vec = hvx_vec_splat_f16(max); + hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_CLAMP_SCALAR_F16); +} + +static inline void hvx_clamp_scalar_f16(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, const int num_elems) { + if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { + hvx_clamp_scalar_f16_aa(dst, src, min, max, num_elems); + } else if (hex_is_aligned((void *) dst, 128)) { + hvx_clamp_scalar_f16_au(dst, src, min, max, num_elems); + } else if (hex_is_aligned((void *) src, 128)) { + hvx_clamp_scalar_f16_ua(dst, src, min, max, num_elems); + } else { + hvx_clamp_scalar_f16_uu(dst, src, min, max, num_elems); + } +} + // // Abs // @@ -386,11 +434,69 @@ static inline void hvx_abs_f32_aa(uint8_t * restrict dst, const uint8_t * restri } } +#define hvx_abs_f16_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t elem_size = sizeof(_Float16); \ + const uint32_t epv = 128 / elem_size; \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = hvx_vec_abs_f16(vsrc[i]); \ + } \ + if (nloe) { \ + HVX_Vector v = hvx_vec_abs_f16(vsrc[i]); \ + vec_store((void *) &vdst[i], nloe * elem_size, v); \ + } \ + } while(0) + +static inline void hvx_abs_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_abs_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); +} + +static inline void hvx_abs_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + hvx_abs_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a); +} + +static inline void hvx_abs_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) src % 128 == 0); + hvx_abs_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u); +} + +static inline void hvx_abs_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + hvx_abs_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); +} + +static inline void hvx_abs_f16(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t num_elems) { + if (hex_is_aligned((void *) dst, 128)) { + if (hex_is_aligned((void *) src, 128)) { + hvx_abs_f16_aa(dst, src, num_elems); + } else { + hvx_abs_f16_au(dst, src, num_elems); + } + } else { + if (hex_is_aligned((void *) src, 128)) { + hvx_abs_f16_ua(dst, src, num_elems); + } else { + hvx_abs_f16_uu(dst, src, num_elems); + } + } +} + // // Square // -#define hvx_sqr_f32_loop_body(dst_type, src_type, vec_store) \ +#define hvx_sqr_f32_loop_body(dst_type, src_type, vec_store) \ do { \ dst_type * restrict vdst = (dst_type *) dst; \ src_type * restrict vsrc = (src_type *) src; \ @@ -404,10 +510,10 @@ static inline void hvx_abs_f32_aa(uint8_t * restrict dst, const uint8_t * restri \ _Pragma("unroll(4)") \ for (; i < nvec; i++) { \ - vdst[i] = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \ + vdst[i] = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \ } \ if (nloe) { \ - HVX_Vector v = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \ + HVX_Vector v = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \ vec_store((void *) &vdst[i], nloe * elem_size, v); \ } \ } while(0) @@ -448,6 +554,64 @@ static inline void hvx_sqr_f32(uint8_t * restrict dst, const uint8_t * restrict } } +#define hvx_sqr_f16_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t elem_size = sizeof(_Float16); \ + const uint32_t epv = 128 / elem_size; \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = HVX_OP_MUL_F16(vsrc[i], vsrc[i]); \ + } \ + if (nloe) { \ + HVX_Vector v = HVX_OP_MUL_F16(vsrc[i], vsrc[i]); \ + vec_store((void *) &vdst[i], nloe * elem_size, v); \ + } \ + } while(0) + +static inline void hvx_sqr_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_sqr_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); +} + +static inline void hvx_sqr_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + hvx_sqr_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a); +} + +static inline void hvx_sqr_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) src % 128 == 0); + hvx_sqr_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u); +} + +static inline void hvx_sqr_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + hvx_sqr_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); +} + +static inline void hvx_sqr_f16(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t num_elems) { + if (hex_is_aligned((void *) dst, 128)) { + if (hex_is_aligned((void *) src, 128)) { + hvx_sqr_f16_aa(dst, src, num_elems); + } else { + hvx_sqr_f16_au(dst, src, num_elems); + } + } else { + if (hex_is_aligned((void *) src, 128)) { + hvx_sqr_f16_ua(dst, src, num_elems); + } else { + hvx_sqr_f16_uu(dst, src, num_elems); + } + } +} + #undef HVX_OP_ADD_F32 #undef HVX_OP_SUB_F32 #undef HVX_OP_MUL_F32 @@ -464,6 +628,7 @@ static inline void hvx_sqr_f32(uint8_t * restrict dst, const uint8_t * restrict #undef hvx_scalar_loop_body #undef HVX_OP_MIN_SCALAR #undef HVX_OP_CLAMP_SCALAR +#undef HVX_OP_CLAMP_SCALAR_F16 #undef DEFINE_HVX_BINARY_OP_VARIANTS #undef HVX_BINARY_DISPATCHER #undef UNUSED diff --git a/ggml/src/ggml-hexagon/htp/hvx-log.h b/ggml/src/ggml-hexagon/htp/hvx-log.h index a209f88d5..491041d5a 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-log.h +++ b/ggml/src/ggml-hexagon/htp/hvx-log.h @@ -86,4 +86,33 @@ static inline void hvx_log_f32_aa(uint8_t * restrict dst, const uint8_t * restri } } +// Compute log(x) for f16 by promoting to f32, applying hvx_vec_log_f32, and narrowing back. +static inline void hvx_log_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; + + const uint32_t nvec = n / VLEN_FP16; + const uint32_t nloe = n % VLEN_FP16; + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); + HVX_Vector r0 = hvx_vec_log_f32(Q6_V_lo_W(p)); + HVX_Vector r1 = hvx_vec_log_f32(Q6_V_hi_W(p)); + vdst[i] = hvx_vec_f32_to_f16(r0, r1); + } + if (nloe) { + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); + HVX_Vector r0 = hvx_vec_log_f32(Q6_V_lo_W(p)); + HVX_Vector r1 = hvx_vec_log_f32(Q6_V_hi_W(p)); + HVX_Vector v = hvx_vec_f32_to_f16(r0, r1); + hvx_vec_store_a((void *) &vdst[i], nloe * SIZEOF_FP16, v); + } +} + #endif /* HVX_LOG_H */ diff --git a/ggml/src/ggml-hexagon/htp/hvx-norm.h b/ggml/src/ggml-hexagon/htp/hvx-norm.h index a8645e412..7ea945a33 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-norm.h +++ b/ggml/src/ggml-hexagon/htp/hvx-norm.h @@ -254,4 +254,201 @@ static inline void hvx_fast_l2_norm_f32(const uint8_t * restrict src, } } +// F16 norm kernels: reduce and scale in f32 (via promote/narrow), matching the +// precision-preserving pattern used by the flash-attn f16 kernels. + +static inline void hvx_fast_rms_norm_f16(const uint8_t * restrict src, + uint8_t * restrict dst, + const int num_elems, + float epsilon) { + + const HVX_Vector * restrict v_src = (HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + + const int nvec = num_elems / VLEN_FP16; // number of full f16 vectors + const int nloe = num_elems % VLEN_FP16; // leftover elements + + HVX_Vector sum_v = Q6_V_vsplat_R(0x00000000); + HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]); + HVX_Vector p0 = Q6_V_lo_W(p); + HVX_Vector p1 = Q6_V_hi_W(p); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0)); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1)); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_VectorPair p = hvx_vec_f16_to_f32(v1); + HVX_Vector p0 = Q6_V_lo_W(p); + HVX_Vector p1 = Q6_V_hi_W(p); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0)); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1)); + } + + sum_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v)); + + HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems); + HVX_Vector denom_v = hvx_vec_inverse_f32(t_v); + HVX_Vector mean_v = Q6_Vqf32_vmpy_VsfVsf(sum_v, denom_v); + HVX_Vector mean_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(mean_v, epsilon_v); + + HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v)); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]); + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v)); + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v)); + v_dst[i] = hvx_vec_f32_to_f16(r0, r1); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_VectorPair p = hvx_vec_f16_to_f32(v1); + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v)); + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v)); + HVX_Vector result = hvx_vec_f32_to_f16(r0, r1); + hvx_vec_store_a(&v_dst[nvec], nloe * SIZEOF_FP16, result); + } +} + +static inline void hvx_fast_norm_f16(const uint8_t * restrict src, + uint8_t * restrict dst, + const int num_elems, + float epsilon) { + + const HVX_Vector * restrict v_src = (HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + + const int nvec = num_elems / VLEN_FP16; + const int nloe = num_elems % VLEN_FP16; + + HVX_Vector sum_sq_v = Q6_V_vsplat_R(0x00000000); + HVX_Vector sum_x_v = Q6_V_vsplat_R(0x00000000); + HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]); + HVX_Vector p0 = Q6_V_lo_W(p); + HVX_Vector p1 = Q6_V_hi_W(p); + sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0)); + sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1)); + sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p0, Q6_V_vzero())); + sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p1, Q6_V_vzero())); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_VectorPair p = hvx_vec_f16_to_f32(v1); + HVX_Vector p0 = Q6_V_lo_W(p); + HVX_Vector p1 = Q6_V_hi_W(p); + sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0)); + sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1)); + sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p0, Q6_V_vzero())); + sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p1, Q6_V_vzero())); + } + + sum_sq_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_sq_v)); + sum_x_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_x_v)); + + HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems); + HVX_Vector denom_v = hvx_vec_inverse_f32(t_v); + HVX_Vector mean_sq_v = Q6_Vqf32_vmpy_VsfVsf(sum_sq_v, denom_v); + HVX_Vector mean_x_v = Q6_Vqf32_vmpy_VsfVsf(sum_x_v, denom_v); + HVX_Vector mean_x_sq_v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(mean_x_v), Q6_Vsf_equals_Vqf32(mean_x_v)); + HVX_Vector var_v = Q6_Vqf32_vsub_Vqf32Vqf32(mean_sq_v, mean_x_sq_v); + HVX_Vector var_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(var_v, epsilon_v); + + HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(var_epsilon_v)); + HVX_Vector mean_x_b = hvx_vec_repl_f32(Q6_Vsf_equals_Vqf32(mean_x_v)); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]); + HVX_Vector d0 = Q6_Vqf32_vsub_VsfVsf(Q6_V_lo_W(p), mean_x_b); + HVX_Vector d1 = Q6_Vqf32_vsub_VsfVsf(Q6_V_hi_W(p), mean_x_b); + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d0), scale_v)); + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d1), scale_v)); + v_dst[i] = hvx_vec_f32_to_f16(r0, r1); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_VectorPair p = hvx_vec_f16_to_f32(v1); + HVX_Vector d0 = Q6_Vqf32_vsub_VsfVsf(Q6_V_lo_W(p), mean_x_b); + HVX_Vector d1 = Q6_Vqf32_vsub_VsfVsf(Q6_V_hi_W(p), mean_x_b); + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d0), scale_v)); + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d1), scale_v)); + HVX_Vector result = hvx_vec_f32_to_f16(r0, r1); + hvx_vec_store_a(&v_dst[nvec], nloe * SIZEOF_FP16, result); + } +} + +static inline void hvx_fast_l2_norm_f16(const uint8_t * restrict src, + uint8_t * restrict dst, + const int num_elems, + float epsilon) { + + const HVX_Vector * restrict v_src = (HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + + const int nvec = num_elems / VLEN_FP16; + const int nloe = num_elems % VLEN_FP16; + + HVX_Vector sum_v = hvx_vec_splat_f32(0.0f); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]); + HVX_Vector p0 = Q6_V_lo_W(p); + HVX_Vector p1 = Q6_V_hi_W(p); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0)); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1)); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_VectorPair p = hvx_vec_f16_to_f32(v1); + HVX_Vector p0 = Q6_V_lo_W(p); + HVX_Vector p1 = Q6_V_hi_W(p); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0)); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1)); + } + + HVX_Vector sum_sf = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v)); + HVX_Vector rsqrt_v = hvx_vec_rsqrt_f32(sum_sf); + HVX_Vector sqrt_v = hvx_vec_inverse_f32(rsqrt_v); + HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon); + HVX_Vector denom_v = Q6_Vsf_vmax_VsfVsf(sqrt_v, epsilon_v); + HVX_Vector scale_v = hvx_vec_inverse_f32(denom_v); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]); + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v)); + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v)); + v_dst[i] = hvx_vec_f32_to_f16(r0, r1); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_VectorPair p = hvx_vec_f16_to_f32(v1); + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v)); + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v)); + HVX_Vector result = hvx_vec_f32_to_f16(r0, r1); + hvx_vec_store_a(&v_dst[nvec], nloe * SIZEOF_FP16, result); + } +} + #endif // HVX_NORM_H diff --git a/ggml/src/ggml-hexagon/htp/hvx-scale.h b/ggml/src/ggml-hexagon/htp/hvx-scale.h index c65c98639..9b1a28f52 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-scale.h +++ b/ggml/src/ggml-hexagon/htp/hvx-scale.h @@ -130,4 +130,70 @@ static inline void hvx_scale_offset_f32(uint8_t * restrict dst, const uint8_t * } } +// Scale+offset computed by promoting f16 -> f32, then narrowing the result back to f16. +#define hvx_scale_offset_f16_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + HVX_Vector vs = hvx_vec_splat_f32(scale); \ + HVX_Vector vo = hvx_vec_splat_f32(offset); \ + \ + const uint32_t nvec = n / VLEN_FP16; \ + const uint32_t nloe = n % VLEN_FP16; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; ++i) { \ + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \ + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), vs), vo)); \ + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), vs), vo)); \ + vdst[i] = hvx_vec_f32_to_f16(r0, r1); \ + } \ + if (nloe) { \ + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \ + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), vs), vo)); \ + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), vs), vo)); \ + HVX_Vector v = hvx_vec_f32_to_f16(r0, r1); \ + vec_store((void *) &vdst[i], nloe * SIZEOF_FP16, v); \ + } \ + } while(0) + +static inline void hvx_scale_offset_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) { + assert((size_t) dst % 128 == 0); + assert((size_t) src % 128 == 0); + hvx_scale_offset_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); +} + +static inline void hvx_scale_offset_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) { + assert((size_t) dst % 128 == 0); + hvx_scale_offset_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a); +} + +static inline void hvx_scale_offset_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) { + assert((size_t) src % 128 == 0); + hvx_scale_offset_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u); +} + +static inline void hvx_scale_offset_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) { + hvx_scale_offset_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); +} + +static inline void hvx_scale_offset_f16(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) { + if (((size_t) dst & 127) == 0) { + if (((size_t) src & 127) == 0) { + hvx_scale_offset_f16_aa(dst, src, n, scale, offset); + } else { + hvx_scale_offset_f16_au(dst, src, n, scale, offset); + } + } else { + if (((size_t) src & 127) == 0) { + hvx_scale_offset_f16_ua(dst, src, n, scale, offset); + } else { + hvx_scale_offset_f16_uu(dst, src, n, scale, offset); + } + } +} + #endif // HVX_SCALE_H diff --git a/ggml/src/ggml-hexagon/htp/hvx-sqrt.h b/ggml/src/ggml-hexagon/htp/hvx-sqrt.h index e31a1006d..abdded5ce 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +++ b/ggml/src/ggml-hexagon/htp/hvx-sqrt.h @@ -123,4 +123,67 @@ static inline void hvx_sqrt_f32(uint8_t * restrict dst, const uint8_t * restrict } } +// Compute sqrt(x) for f16 by promoting to f32, applying hvx_vec_rsqrt_f32, and narrowing back. +#define hvx_sqrt_f16_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t nvec = n / VLEN_FP16; \ + const uint32_t nloe = n % VLEN_FP16; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \ + HVX_Vector r0 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_lo_W(p)), Q6_V_lo_W(p)); \ + HVX_Vector r1 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_hi_W(p)), Q6_V_hi_W(p)); \ + vdst[i] = hvx_vec_f32_to_f16(r0, r1); \ + } \ + if (nloe) { \ + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \ + HVX_Vector r0 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_lo_W(p)), Q6_V_lo_W(p)); \ + HVX_Vector r1 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_hi_W(p)), Q6_V_hi_W(p)); \ + HVX_Vector v = hvx_vec_f32_to_f16(r0, r1); \ + vec_store((void *) &vdst[i], nloe * SIZEOF_FP16, v); \ + } \ + } while(0) + +static inline void hvx_sqrt_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_sqrt_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); +} + +static inline void hvx_sqrt_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + hvx_sqrt_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a); +} + +static inline void hvx_sqrt_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) src % 128 == 0); + hvx_sqrt_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u); +} + +static inline void hvx_sqrt_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + hvx_sqrt_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); +} + +static inline void hvx_sqrt_f16(uint8_t * restrict dst, const uint8_t * restrict src, const int num_elems) { + if ((unsigned long) dst % 128 == 0) { + if ((unsigned long) src % 128 == 0) { + hvx_sqrt_f16_aa(dst, src, num_elems); + } else { + hvx_sqrt_f16_au(dst, src, num_elems); + } + } else { + if ((unsigned long) src % 128 == 0) { + hvx_sqrt_f16_ua(dst, src, num_elems); + } else { + hvx_sqrt_f16_uu(dst, src, num_elems); + } + } +} + #endif /* HVX_SQRT_H */ diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c index 1a632bf56..5e62b4a9b 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.c +++ b/ggml/src/ggml-hexagon/htp/unary-ops.c @@ -234,6 +234,146 @@ static void sqrt_f32(const float * restrict src, } } +static void scale_f16(const _Float16 * restrict src, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float scale = 0.f; + float bias = 0.f; + memcpy(&scale, &op_params[0], sizeof(float)); + memcpy(&bias, &op_params[1], sizeof(float)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_scale_offset_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0, scale, bias); + } +} + +static void clamp_f16(const _Float16 * restrict src, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float min = 0.f; + float max = 0.f; + memcpy(&min, &op_params[0], sizeof(float)); + memcpy(&max, &op_params[1], sizeof(float)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_clamp_scalar_f16(dst_local, src_local, (_Float16) min, (_Float16) max, ne0); + } +} + +static void rms_norm_f16(const _Float16 * restrict src, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float epsilon = 0.f; + memcpy(&epsilon, op_params, sizeof(float)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_fast_rms_norm_f16((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon); + } +} + +static void norm_f16(const _Float16 * restrict src, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float epsilon = 0.f; + memcpy(&epsilon, op_params, sizeof(float)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_fast_norm_f16((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon); + } +} + +static void sqr_f16(const _Float16 * restrict src, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_sqr_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0); + } +} + +static void sqrt_f16(const _Float16 * restrict src, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_sqrt_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0); + } +} + +static void abs_f16(const _Float16 * restrict src, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_abs_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0); + } +} + +static void log_f16(const _Float16 * restrict src, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_log_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0); + } +} + +static void l2_norm_f16(const _Float16 * restrict src, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float epsilon = 0.f; + memcpy(&epsilon, op_params, sizeof(float)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_f = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_f = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_fast_l2_norm_f16((const uint8_t *)src_f, (uint8_t *)dst_f, ne0, epsilon); + } +} + static void neg_f32(const float * restrict src, float * restrict dst, const uint32_t num_rows, @@ -471,8 +611,8 @@ static void log_f32(const float * restrict src, } } -#define DEFINE_UNARY_TASK(NAME, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) \ -static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * data) { \ +#define DEFINE_UNARY_TASK_IMPL(NAME, TYPE, SUFFIX, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) \ +static void unary_task_##SUFFIX##_##NAME(unsigned int nth, unsigned int ith, void * data) { \ const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; \ struct htp_ops_context * octx = uctx->octx; \ const struct htp_tensor * src = octx->src[0]; \ @@ -536,7 +676,7 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat const uint32_t dst_max_block = block_dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1); \ const uint32_t BLOCK = MIN(src0_max_block, dst_max_block); \ if (BLOCK == 0) { \ - FARF(ERROR, "unary-f32 : current VTCM reservation %zu is too small, needed at least %zu\n", \ + FARF(ERROR, "unary-" #SUFFIX " : current VTCM reservation %zu is too small, needed at least %zu\n", \ uctx->vtcm_src0_size_per_thread, src0_row_size_aligned); \ return; \ } \ @@ -578,11 +718,11 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_src0_contig, block_dst_contig, \ ne01, div_ne01); \ \ - float * dst_vtcm = (float *) dma_queue_pop(dma_queue).src; \ - float * src0_vtcm = (float *) dma_queue_pop(dma_queue).dst; \ - float * src1_vtcm = NULL; \ + TYPE * dst_vtcm = (TYPE *) dma_queue_pop(dma_queue).src; \ + TYPE * src0_vtcm = (TYPE *) dma_queue_pop(dma_queue).dst; \ + TYPE * src1_vtcm = NULL; \ if ((IS_RMS_NORM_MUL) && !uctx->broadcast_weight) { \ - src1_vtcm = (float *) dma_queue_pop(dma_queue).dst; \ + src1_vtcm = (TYPE *) dma_queue_pop(dma_queue).dst; \ } \ \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ @@ -625,6 +765,10 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat dma_queue_flush(dma_queue); \ } +// F32 unary task: row-block DMA/VTCM plumbing, float-typed VTCM buffers. +#define DEFINE_UNARY_TASK(NAME, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) \ + DEFINE_UNARY_TASK_IMPL(NAME, float, f32, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) + DEFINE_UNARY_TASK(norm, false, false, norm_f32(src0_vtcm, dst_vtcm, block_size, uctx)) DEFINE_UNARY_TASK(rms_norm, false, false, rms_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx)) DEFINE_UNARY_TASK(rms_norm_mul, true, false, rms_norm_mul_f32(src0_vtcm, uctx->broadcast_weight ? (const float *) src1_vtcm_data : src1_vtcm, dst_vtcm, block_size, uctx)) @@ -644,6 +788,18 @@ DEFINE_UNARY_TASK(unary_log, false, false, log_f32(src0_vtcm, dst_vtcm, blo DEFINE_UNARY_TASK(l2_norm, false, false, l2_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx)) DEFINE_UNARY_TASK(tri, false, true, tri_f32(src0_vtcm, dst_vtcm, block_size, ir, uctx)) +// F16 unary tasks: same DMA/VTCM plumbing as DEFINE_UNARY_TASK, but VTCM buffers are +// _Float16-typed. None of the current F16 ops need RMS_NORM_MUL or TRI support. +DEFINE_UNARY_TASK_IMPL(norm, _Float16, f16, false, false, norm_f16(src0_vtcm, dst_vtcm, block_size, uctx)) +DEFINE_UNARY_TASK_IMPL(rms_norm, _Float16, f16, false, false, rms_norm_f16(src0_vtcm, dst_vtcm, block_size, uctx)) +DEFINE_UNARY_TASK_IMPL(scale, _Float16, f16, false, false, scale_f16(src0_vtcm, dst_vtcm, block_size, uctx)) +DEFINE_UNARY_TASK_IMPL(clamp, _Float16, f16, false, false, clamp_f16(src0_vtcm, dst_vtcm, block_size, uctx)) +DEFINE_UNARY_TASK_IMPL(sqr, _Float16, f16, false, false, sqr_f16(src0_vtcm, dst_vtcm, block_size, uctx)) +DEFINE_UNARY_TASK_IMPL(sqrt, _Float16, f16, false, false, sqrt_f16(src0_vtcm, dst_vtcm, block_size, uctx)) +DEFINE_UNARY_TASK_IMPL(l2_norm, _Float16, f16, false, false, l2_norm_f16(src0_vtcm, dst_vtcm, block_size, uctx)) +DEFINE_UNARY_TASK_IMPL(unary_abs, _Float16, f16, false, false, abs_f16(src0_vtcm, dst_vtcm, block_size, uctx)) +DEFINE_UNARY_TASK_IMPL(unary_log, _Float16, f16, false, false, log_f16(src0_vtcm, dst_vtcm, block_size, uctx)) + // Apply a pointwise unary op to one column tile that is already in VTCM. #define DEFINE_UNARY_TILED_TASK(NAME, IS_TRI, CORE_TILE_EXPR) \ static void unary_task_f32_tiled_##NAME(unsigned int nth, unsigned int ith, void * data) { \ @@ -892,50 +1048,76 @@ DEFINE_UNARY_TILED_TASK(unary_abs, false, hvx_abs_f32_aa(dst_vtcm, src_vtcm DEFINE_UNARY_TILED_TASK(unary_log, false, hvx_log_f32_aa(dst_vtcm, src_vtcm, tw)) DEFINE_UNARY_TILED_TASK(tri, true, tri_apply_tile_f32(src_vtcm, dst_vtcm, tw, col, i01, ne0, tri_ttype)) -static int execute_op_unary_f32(struct htp_ops_context * octx) { +static int execute_op_unary(struct htp_ops_context * octx) { int err = HTP_STATUS_OK; const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; + const bool is_f16 = (src0->type == HTP_TYPE_F16); + const char * op_type = NULL; switch (octx->op) { - case HTP_OP_NORM: op_type = "norm-f32"; break; - case HTP_OP_RMS_NORM: op_type = "rmsnorm-f32"; break; - case HTP_OP_RMS_NORM_MUL: op_type = "rmsnorm-mul-f32"; break; - case HTP_OP_SCALE: op_type = "scale-f32"; break; - case HTP_OP_CLAMP: op_type = "clamp-f32"; break; - case HTP_OP_SQR: op_type = "sqr-f32"; break; - case HTP_OP_SQRT: op_type = "sqrt-f32"; break; - case HTP_OP_UNARY_NEG: op_type = "neg-f32"; break; - case HTP_OP_UNARY_EXP: op_type = "exp-f32"; break; - case HTP_OP_UNARY_SIGMOID: op_type = "sigmoid-f32"; break; - case HTP_OP_UNARY_SILU: op_type = "silu-f32"; break; - case HTP_OP_UNARY_GELU: op_type = "gelu-f32"; break; - case HTP_OP_UNARY_SOFTPLUS: op_type = "softplus-f32"; break; - case HTP_OP_UNARY_TANH: op_type = "tanh-f32"; break; - case HTP_OP_UNARY_ABS: op_type = "abs-f32"; break; - case HTP_OP_UNARY_LOG: op_type = "log-f32"; break; - case HTP_OP_L2_NORM: op_type = "l2norm-f32"; break; - case HTP_OP_TRI: op_type = "tri-f32"; break; + case HTP_OP_NORM: op_type = is_f16 ? "norm-f16" : "norm-f32"; break; + case HTP_OP_RMS_NORM: op_type = is_f16 ? "rmsnorm-f16" : "rmsnorm-f32"; break; + case HTP_OP_RMS_NORM_MUL: op_type = "rmsnorm-mul-f32"; break; + case HTP_OP_SCALE: op_type = is_f16 ? "scale-f16" : "scale-f32"; break; + case HTP_OP_CLAMP: op_type = is_f16 ? "clamp-f16" : "clamp-f32"; break; + case HTP_OP_SQR: op_type = is_f16 ? "sqr-f16" : "sqr-f32"; break; + case HTP_OP_SQRT: op_type = is_f16 ? "sqrt-f16" : "sqrt-f32"; break; + case HTP_OP_UNARY_NEG: op_type = "neg-f32"; break; + case HTP_OP_UNARY_EXP: op_type = "exp-f32"; break; + case HTP_OP_UNARY_SIGMOID: op_type = "sigmoid-f32"; break; + case HTP_OP_UNARY_SILU: op_type = "silu-f32"; break; + case HTP_OP_UNARY_GELU: op_type = "gelu-f32"; break; + case HTP_OP_UNARY_SOFTPLUS: op_type = "softplus-f32"; break; + case HTP_OP_UNARY_TANH: op_type = "tanh-f32"; break; + case HTP_OP_UNARY_ABS: op_type = is_f16 ? "abs-f16" : "abs-f32"; break; + case HTP_OP_UNARY_LOG: op_type = is_f16 ? "log-f16" : "log-f32"; break; + case HTP_OP_L2_NORM: op_type = is_f16 ? "l2norm-f16" : "l2norm-f32"; break; + case HTP_OP_TRI: op_type = "tri-f32"; break; default: FARF(ERROR, "Unsupported unary Op %u\n", octx->op); return HTP_STATUS_NO_SUPPORT; } + // F16 only has row-block kernels for this subset of ops (see the dispatch switch + // below) - reject everything else up front, before touching kparams/VTCM. + if (is_f16) { + switch (octx->op) { + case HTP_OP_NORM: + case HTP_OP_RMS_NORM: + case HTP_OP_SCALE: + case HTP_OP_CLAMP: + case HTP_OP_SQR: + case HTP_OP_SQRT: + case HTP_OP_L2_NORM: + case HTP_OP_UNARY_ABS: + case HTP_OP_UNARY_LOG: + break; + default: + FARF(ERROR, "unary-%s: not supported for F16\n", op_type); + return HTP_STATUS_NO_SUPPORT; + } + } + const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params; const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; const uint32_t n_threads = kparams->n_threads; - const size_t src0_data_row_size = src0->ne[0] * sizeof(float); - const size_t dst_data_row_size = dst->ne[0] * sizeof(float); + const size_t elem_size = is_f16 ? sizeof(_Float16) : sizeof(float); + + const size_t src0_data_row_size = src0->ne[0] * elem_size; + const size_t dst_data_row_size = dst->ne[0] * elem_size; const size_t src0_row_size_aligned = kparams->src0_row_size_aligned; const size_t dst_row_size_aligned = kparams->dst_row_size_aligned; + // Always 0 for F16 - htp_unary_vtcm_layout_build() keeps F16 on the row-block path, + // since only F32 has unary_task_f32_tiled_* kernels. const uint32_t col_tile = kparams->col_tile; size_t src1_data_row_size = 0; @@ -943,6 +1125,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) { bool broadcast_weight = kparams->broadcast_weight; const struct htp_tensor * src1 = NULL; + // RMS_NORM_MUL fusion is F32-only (its weight tensor is always F32; see + // try_fuse_node()'s type guard), so this never triggers when is_f16 is true. if (octx->op == HTP_OP_RMS_NORM_MUL) { src1 = octx->src[1]; src1_data_row_size = src1->ne[0] * sizeof(float); @@ -987,7 +1171,7 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) { .block = kparams->block, .nc = src0->ne[0], - .col_tile = (uint32_t) kparams->col_tile, + .col_tile = col_tile, .broadcast_weight = broadcast_weight, .vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, 0), @@ -1020,6 +1204,19 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) { case HTP_OP_TRI: task_func = unary_task_f32_tiled_tri; break; default: break; } + } else if (is_f16) { + switch (octx->op) { + case HTP_OP_NORM: task_func = unary_task_f16_norm; break; + case HTP_OP_RMS_NORM: task_func = unary_task_f16_rms_norm; break; + case HTP_OP_SCALE: task_func = unary_task_f16_scale; break; + case HTP_OP_CLAMP: task_func = unary_task_f16_clamp; break; + case HTP_OP_SQR: task_func = unary_task_f16_sqr; break; + case HTP_OP_SQRT: task_func = unary_task_f16_sqrt; break; + case HTP_OP_L2_NORM: task_func = unary_task_f16_l2_norm; break; + case HTP_OP_UNARY_ABS: task_func = unary_task_f16_unary_abs; break; + case HTP_OP_UNARY_LOG: task_func = unary_task_f16_unary_log; break; + default: break; + } } else { switch (octx->op) { case HTP_OP_NORM: task_func = unary_task_f32_norm; break; @@ -1047,7 +1244,7 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) { if (task_func) { worker_pool_run_func(octx->ctx->worker_pool, task_func, &uctx, n_threads); } else { - FARF(ERROR, "execute_op_unary_f32: task function is NULL for op %d\n", octx->op); + FARF(ERROR, "execute_op_unary: task function is NULL for op %d\n", octx->op); err = HTP_STATUS_NO_SUPPORT; } } @@ -1058,7 +1255,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) { int op_unary(struct htp_ops_context * octx) { switch (octx->src[0]->type) { case HTP_TYPE_F32: - return execute_op_unary_f32(octx); + case HTP_TYPE_F16: + return execute_op_unary(octx); default: return HTP_STATUS_NO_SUPPORT; diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.h index 458218ff4..116a591c2 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.h +++ b/ggml/src/ggml-hexagon/htp/unary-ops.h @@ -85,17 +85,19 @@ static inline void htp_unary_vtcm_layout_build( bool broadcast_weight, uint32_t n_threads, size_t vtcm_size, + size_t elem_size, uint32_t * out_col_tile, uint32_t * out_vtcm_row_per_thread ) { - const size_t src0_data_row_size = ne00 * sizeof(float); - const size_t dst_data_row_size = ne10 * sizeof(float); + const size_t src0_data_row_size = ne00 * elem_size; + const size_t dst_data_row_size = ne10 * elem_size; const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128); const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, 128); size_t src1_row_size_aligned = 0; if (op == HTP_OP_RMS_NORM_MUL) { + // RMS_NORM_MUL fusion is F32-only; its weight tensor is always F32. const size_t src1_data_row_size = ne11 * sizeof(float); src1_row_size_aligned = hex_round_up(src1_data_row_size, 128); } @@ -125,12 +127,19 @@ static inline void htp_unary_vtcm_layout_build( const bool is_reduction = (op == HTP_OP_NORM || op == HTP_OP_RMS_NORM || op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_L2_NORM); + // The tiled fallback path below only has F32 task functions (unary_task_f32_tiled_*); + // F16 has no tiled kernels, so it must stay on the row-block path like reduction ops. + // NOTE: if F16 ends up with vtcm_row_per_thread == 0 here (row too large for the VTCM + // budget), execute_op_unary() will see BLOCK == 0 and skip computation for that op + // (logged via FARF(ERROR, ...)) since there is no F16 tiled fallback. This is a known + // limitation; supporting it would require adding F16 tiled kernels. + const bool is_f16 = (elem_size == sizeof(_Float16)); uint32_t col_tile = 0; - if (vtcm_row_per_thread == 0 && !is_reduction) { + if (vtcm_row_per_thread == 0 && !is_reduction && !is_f16) { const size_t per_thread_budget = vtcm_size / n_threads; const size_t col_tile_bytes = hex_align_down(per_thread_budget / 4, 128); - col_tile = (uint32_t) (col_tile_bytes / sizeof(float)); + col_tile = (uint32_t) (col_tile_bytes / elem_size); L->src0_bytes = col_tile_bytes * 2; L->dst_bytes = col_tile_bytes * 2;