Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-09 06:19:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

ggml-hexagon: add F16 support for unary ops (#28228)

Browse Source 
Extend the HTP backend's F16 unary op coverage to include ABS on top
of the existing NORM/RMS_NORM/L2_NORM/SCALE/CLAMP/SQR/SQRT set.

- Add hvx_abs_f16_{aa,au,ua,uu} + dispatcher in hvx-arith.h, mirroring
  the sqr_f16 kernel structure and using the existing hvx_vec_abs_f16()
  sign-bit-clear helper
- Add abs_f16() row-wise dispatch and DEFINE_UNARY_TASK_F16(unary_abs, ...)
  in unary-ops.c, wired into execute_op_unary()'s op_type/task_func
  switches
- Register HTP_OP_UNARY_ABS in htp_op_is_unary() (unary-ops.h) so that
  ggml_hexagon_precompute_unary_params() fills kernel_params (n_threads,
  VTCM layout) for ABS nodes -- required for the F16 path to function
- Narrow the F16 GGML_OP_UNARY gate in ggml_hexagon_supported_unary()
  (ggml-hexagon.cpp) to allow GGML_UNARY_OP_ABS specifically, instead of
  rejecting all GGML_OP_UNARY ops for F16
- Merge the separate execute_op_unary_f32()/execute_op_unary_f16()
  functions into a single execute_op_unary(), branching on an is_f16
  flag for the parts that actually differ by type (elem_size, the
  early F16 op-support check, and which task_func table to use) while
  keeping the F32-only tiled/RMS_NORM_MUL paths intact -- per review
  feedback to avoid duplicating the shared VTCM/DMA plumbing

Verified on-device (QRD8850, Hexagon v81) via test-backend-ops -o ABS:
8/8 passing (F16 + F32, HTP0, no CPU fallback). Regression-checked
SQR/CLAMP/SQRT (F16+F32) and NORM/RMS_NORM/L2_NORM/SCALE (F32; their F16
paths have no CPU reference kernel in test-backend-ops and cannot be
correctness-tested there independent of this change).
This commit is contained in:
cqderek
2026-09-03 03:59:36 +08:00
committed by GitHub
parent 7339054744
commit f027c4f1b0
8 changed files with 797 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-hexagon/ggml-hexagon.cpp
+32 -6
View File
@@ -4005,8 +4005,10 @@ static void ggml_hexagon_precompute_unary_params(
kparams->n_threads = n_threads;
const size_t src0_data_row_size = src0->ne[0] * sizeof(float);
const size_t dst_data_row_size = dst->ne[0] * sizeof(float);
const size_t elem_size = ggml_type_size(src0->type);
const size_t src0_data_row_size = src0->ne[0] * elem_size;
const size_t dst_data_row_size = dst->ne[0] * ggml_type_size(dst->type);
const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128);
const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, 128);
@@ -4020,7 +4022,7 @@ static void ggml_hexagon_precompute_unary_params(
if (op == HTP_OP_RMS_NORM_MUL) {
GGML_ASSERT(src1 != nullptr);
src1_data_row_size = src1->ne[0] * sizeof(float);
src1_data_row_size = src1->ne[0] * ggml_type_size(src1->type);
src1_row_size_aligned = hex_round_up(src1_data_row_size, 128);
broadcast_weight = (src1->ne[1] * src1->ne[2] * src1->ne[3] == 1);
}
@@ -4034,7 +4036,7 @@ static void ggml_hexagon_precompute_unary_params(
htp_unary_vtcm_layout_build(&L, op, src0->ne[0], dst->ne[0],
op == HTP_OP_RMS_NORM_MUL ? src1->ne[0] : 0,
broadcast_weight, n_threads, sess->vtcm_size,
broadcast_weight, n_threads, sess->vtcm_size, elem_size,
&col_tile, &vtcm_row_per_thread);
kparams->col_tile = col_tile;
@@ -4451,15 +4453,39 @@ static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * ses
const struct ggml_tensor * src0 = op->src[0];
const struct ggml_tensor * dst = op;
if (src0->type != GGML_TYPE_F32) {
if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) {
return false;
}
if (dst->type != GGML_TYPE_F32) {
if (dst->type != src0->type) {
return false;
}
if (!ggml_is_contiguous_rows(src0)) {
return false;
}
// F16 device kernels only cover this explicit whitelist (must stay in sync with
// the is_f16 whitelist in execute_op_unary(), unary-ops.c).
if (src0->type == GGML_TYPE_F16) {
switch (op->op) {
case GGML_OP_NORM:
case GGML_OP_RMS_NORM:
case GGML_OP_L2_NORM:
case GGML_OP_SCALE:
case GGML_OP_CLAMP:
case GGML_OP_SQR:
case GGML_OP_SQRT:
case GGML_OP_LOG:
break;
case GGML_OP_UNARY:
if (ggml_get_unary_op(op) != GGML_UNARY_OP_ABS) {
return false;
}
break;
default:
return false;
}
}
if (!ggml_are_same_shape(src0, dst)) {
return false;
}
ggml/src/ggml-hexagon/htp/hvx-arith.h
+168 -3
View File
@@ -358,6 +358,54 @@ static inline void hvx_clamp_scalar_f32(uint8_t * restrict dst, const uint8_t *
}
}
#define HVX_OP_CLAMP_SCALAR_F16(v) \
({ \
HVX_VectorPred pred_cap_right = Q6_Q_vcmp_gt_VhfVhf(v, max_vec); \
HVX_VectorPred pred_cap_left = Q6_Q_vcmp_gt_VhfVhf(min_vec, v); \
HVX_Vector tmp = Q6_V_vmux_QVV(pred_cap_right, max_vec, v); \
Q6_V_vmux_QVV(pred_cap_left, min_vec, tmp); \
})
static inline void hvx_clamp_scalar_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) {
const HVX_Vector min_vec = hvx_vec_splat_f16(min);
const HVX_Vector max_vec = hvx_vec_splat_f16(max);
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_CLAMP_SCALAR_F16);
}
static inline void hvx_clamp_scalar_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) {
const HVX_Vector min_vec = hvx_vec_splat_f16(min);
const HVX_Vector max_vec = hvx_vec_splat_f16(max);
assert((unsigned long) dst % 128 == 0);
hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_CLAMP_SCALAR_F16);
}
static inline void hvx_clamp_scalar_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) {
const HVX_Vector min_vec = hvx_vec_splat_f16(min);
const HVX_Vector max_vec = hvx_vec_splat_f16(max);
assert((unsigned long) src % 128 == 0);
hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_CLAMP_SCALAR_F16);
}
static inline void hvx_clamp_scalar_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) {
const HVX_Vector min_vec = hvx_vec_splat_f16(min);
const HVX_Vector max_vec = hvx_vec_splat_f16(max);
hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_CLAMP_SCALAR_F16);
}
static inline void hvx_clamp_scalar_f16(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, const int num_elems) {
if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) {
hvx_clamp_scalar_f16_aa(dst, src, min, max, num_elems);
} else if (hex_is_aligned((void *) dst, 128)) {
hvx_clamp_scalar_f16_au(dst, src, min, max, num_elems);
} else if (hex_is_aligned((void *) src, 128)) {
hvx_clamp_scalar_f16_ua(dst, src, min, max, num_elems);
} else {
hvx_clamp_scalar_f16_uu(dst, src, min, max, num_elems);
}
}
//
// Abs
//
@@ -386,11 +434,69 @@ static inline void hvx_abs_f32_aa(uint8_t * restrict dst, const uint8_t * restri
}
}
#define hvx_abs_f16_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t elem_size = sizeof(_Float16); \
const uint32_t epv = 128 / elem_size; \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = hvx_vec_abs_f16(vsrc[i]); \
} \
if (nloe) { \
HVX_Vector v = hvx_vec_abs_f16(vsrc[i]); \
vec_store((void *) &vdst[i], nloe * elem_size, v); \
} \
} while(0)
static inline void hvx_abs_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_abs_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
static inline void hvx_abs_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
hvx_abs_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
}
static inline void hvx_abs_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) src % 128 == 0);
hvx_abs_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
}
static inline void hvx_abs_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_abs_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
static inline void hvx_abs_f16(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t num_elems) {
if (hex_is_aligned((void *) dst, 128)) {
if (hex_is_aligned((void *) src, 128)) {
hvx_abs_f16_aa(dst, src, num_elems);
} else {
hvx_abs_f16_au(dst, src, num_elems);
}
} else {
if (hex_is_aligned((void *) src, 128)) {
hvx_abs_f16_ua(dst, src, num_elems);
} else {
hvx_abs_f16_uu(dst, src, num_elems);
}
}
}
//
// Square
//
#define hvx_sqr_f32_loop_body(dst_type, src_type, vec_store) \
#define hvx_sqr_f32_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
@@ -404,10 +510,10 @@ static inline void hvx_abs_f32_aa(uint8_t * restrict dst, const uint8_t * restri
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \
vdst[i] = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \
} \
if (nloe) { \
HVX_Vector v = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \
HVX_Vector v = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \
vec_store((void *) &vdst[i], nloe * elem_size, v); \
} \
} while(0)
@@ -448,6 +554,64 @@ static inline void hvx_sqr_f32(uint8_t * restrict dst, const uint8_t * restrict
}
}
#define hvx_sqr_f16_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t elem_size = sizeof(_Float16); \
const uint32_t epv = 128 / elem_size; \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = HVX_OP_MUL_F16(vsrc[i], vsrc[i]); \
} \
if (nloe) { \
HVX_Vector v = HVX_OP_MUL_F16(vsrc[i], vsrc[i]); \
vec_store((void *) &vdst[i], nloe * elem_size, v); \
} \
} while(0)
static inline void hvx_sqr_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_sqr_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
static inline void hvx_sqr_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
hvx_sqr_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
}
static inline void hvx_sqr_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) src % 128 == 0);
hvx_sqr_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
}
static inline void hvx_sqr_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_sqr_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
static inline void hvx_sqr_f16(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t num_elems) {
if (hex_is_aligned((void *) dst, 128)) {
if (hex_is_aligned((void *) src, 128)) {
hvx_sqr_f16_aa(dst, src, num_elems);
} else {
hvx_sqr_f16_au(dst, src, num_elems);
}
} else {
if (hex_is_aligned((void *) src, 128)) {
hvx_sqr_f16_ua(dst, src, num_elems);
} else {
hvx_sqr_f16_uu(dst, src, num_elems);
}
}
}
#undef HVX_OP_ADD_F32
#undef HVX_OP_SUB_F32
#undef HVX_OP_MUL_F32
@@ -464,6 +628,7 @@ static inline void hvx_sqr_f32(uint8_t * restrict dst, const uint8_t * restrict
#undef hvx_scalar_loop_body
#undef HVX_OP_MIN_SCALAR
#undef HVX_OP_CLAMP_SCALAR
#undef HVX_OP_CLAMP_SCALAR_F16
#undef DEFINE_HVX_BINARY_OP_VARIANTS
#undef HVX_BINARY_DISPATCHER
#undef UNUSED
ggml/src/ggml-hexagon/htp/hvx-log.h
+29
View File
@@ -86,4 +86,33 @@ static inline void hvx_log_f32_aa(uint8_t * restrict dst, const uint8_t * restri
}
}
// Compute log(x) for f16 by promoting to f32, applying hvx_vec_log_f32, and narrowing back.
static inline void hvx_log_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
HVX_Vector * restrict vsrc = (HVX_Vector *) src;
const uint32_t nvec = n / VLEN_FP16;
const uint32_t nloe = n % VLEN_FP16;
uint32_t i = 0;
_Pragma("unroll(4)")
for (; i < nvec; i++) {
HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]);
HVX_Vector r0 = hvx_vec_log_f32(Q6_V_lo_W(p));
HVX_Vector r1 = hvx_vec_log_f32(Q6_V_hi_W(p));
vdst[i] = hvx_vec_f32_to_f16(r0, r1);
}
if (nloe) {
HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]);
HVX_Vector r0 = hvx_vec_log_f32(Q6_V_lo_W(p));
HVX_Vector r1 = hvx_vec_log_f32(Q6_V_hi_W(p));
HVX_Vector v = hvx_vec_f32_to_f16(r0, r1);
hvx_vec_store_a((void *) &vdst[i], nloe * SIZEOF_FP16, v);
}
}
#endif /* HVX_LOG_H */
ggml/src/ggml-hexagon/htp/hvx-norm.h
+197
View File
@@ -254,4 +254,201 @@ static inline void hvx_fast_l2_norm_f32(const uint8_t * restrict src,
}
}
// F16 norm kernels: reduce and scale in f32 (via promote/narrow), matching the
// precision-preserving pattern used by the flash-attn f16 kernels.
static inline void hvx_fast_rms_norm_f16(const uint8_t * restrict src,
uint8_t * restrict dst,
const int num_elems,
float epsilon) {
const HVX_Vector * restrict v_src = (HVX_Vector *) src;
HVX_Vector * restrict v_dst = (HVX_Vector *) dst;
const int nvec = num_elems / VLEN_FP16; // number of full f16 vectors
const int nloe = num_elems % VLEN_FP16; // leftover elements
HVX_Vector sum_v = Q6_V_vsplat_R(0x00000000);
HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]);
HVX_Vector p0 = Q6_V_lo_W(p);
HVX_Vector p1 = Q6_V_hi_W(p);
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0));
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1));
}
if (nloe > 0) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16);
HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
HVX_VectorPair p = hvx_vec_f16_to_f32(v1);
HVX_Vector p0 = Q6_V_lo_W(p);
HVX_Vector p1 = Q6_V_hi_W(p);
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0));
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1));
}
sum_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v));
HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems);
HVX_Vector denom_v = hvx_vec_inverse_f32(t_v);
HVX_Vector mean_v = Q6_Vqf32_vmpy_VsfVsf(sum_v, denom_v);
HVX_Vector mean_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(mean_v, epsilon_v);
HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v));
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]);
HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v));
HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v));
v_dst[i] = hvx_vec_f32_to_f16(r0, r1);
}
if (nloe > 0) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16);
HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
HVX_VectorPair p = hvx_vec_f16_to_f32(v1);
HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v));
HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v));
HVX_Vector result = hvx_vec_f32_to_f16(r0, r1);
hvx_vec_store_a(&v_dst[nvec], nloe * SIZEOF_FP16, result);
}
}
static inline void hvx_fast_norm_f16(const uint8_t * restrict src,
uint8_t * restrict dst,
const int num_elems,
float epsilon) {
const HVX_Vector * restrict v_src = (HVX_Vector *) src;
HVX_Vector * restrict v_dst = (HVX_Vector *) dst;
const int nvec = num_elems / VLEN_FP16;
const int nloe = num_elems % VLEN_FP16;
HVX_Vector sum_sq_v = Q6_V_vsplat_R(0x00000000);
HVX_Vector sum_x_v = Q6_V_vsplat_R(0x00000000);
HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]);
HVX_Vector p0 = Q6_V_lo_W(p);
HVX_Vector p1 = Q6_V_hi_W(p);
sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0));
sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1));
sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p0, Q6_V_vzero()));
sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p1, Q6_V_vzero()));
}
if (nloe > 0) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16);
HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
HVX_VectorPair p = hvx_vec_f16_to_f32(v1);
HVX_Vector p0 = Q6_V_lo_W(p);
HVX_Vector p1 = Q6_V_hi_W(p);
sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0));
sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1));
sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p0, Q6_V_vzero()));
sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p1, Q6_V_vzero()));
}
sum_sq_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_sq_v));
sum_x_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_x_v));
HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems);
HVX_Vector denom_v = hvx_vec_inverse_f32(t_v);
HVX_Vector mean_sq_v = Q6_Vqf32_vmpy_VsfVsf(sum_sq_v, denom_v);
HVX_Vector mean_x_v = Q6_Vqf32_vmpy_VsfVsf(sum_x_v, denom_v);
HVX_Vector mean_x_sq_v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(mean_x_v), Q6_Vsf_equals_Vqf32(mean_x_v));
HVX_Vector var_v = Q6_Vqf32_vsub_Vqf32Vqf32(mean_sq_v, mean_x_sq_v);
HVX_Vector var_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(var_v, epsilon_v);
HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(var_epsilon_v));
HVX_Vector mean_x_b = hvx_vec_repl_f32(Q6_Vsf_equals_Vqf32(mean_x_v));
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]);
HVX_Vector d0 = Q6_Vqf32_vsub_VsfVsf(Q6_V_lo_W(p), mean_x_b);
HVX_Vector d1 = Q6_Vqf32_vsub_VsfVsf(Q6_V_hi_W(p), mean_x_b);
HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d0), scale_v));
HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d1), scale_v));
v_dst[i] = hvx_vec_f32_to_f16(r0, r1);
}
if (nloe > 0) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16);
HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
HVX_VectorPair p = hvx_vec_f16_to_f32(v1);
HVX_Vector d0 = Q6_Vqf32_vsub_VsfVsf(Q6_V_lo_W(p), mean_x_b);
HVX_Vector d1 = Q6_Vqf32_vsub_VsfVsf(Q6_V_hi_W(p), mean_x_b);
HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d0), scale_v));
HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d1), scale_v));
HVX_Vector result = hvx_vec_f32_to_f16(r0, r1);
hvx_vec_store_a(&v_dst[nvec], nloe * SIZEOF_FP16, result);
}
}
static inline void hvx_fast_l2_norm_f16(const uint8_t * restrict src,
uint8_t * restrict dst,
const int num_elems,
float epsilon) {
const HVX_Vector * restrict v_src = (HVX_Vector *) src;
HVX_Vector * restrict v_dst = (HVX_Vector *) dst;
const int nvec = num_elems / VLEN_FP16;
const int nloe = num_elems % VLEN_FP16;
HVX_Vector sum_v = hvx_vec_splat_f32(0.0f);
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]);
HVX_Vector p0 = Q6_V_lo_W(p);
HVX_Vector p1 = Q6_V_hi_W(p);
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0));
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1));
}
if (nloe > 0) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16);
HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
HVX_VectorPair p = hvx_vec_f16_to_f32(v1);
HVX_Vector p0 = Q6_V_lo_W(p);
HVX_Vector p1 = Q6_V_hi_W(p);
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0));
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1));
}
HVX_Vector sum_sf = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v));
HVX_Vector rsqrt_v = hvx_vec_rsqrt_f32(sum_sf);
HVX_Vector sqrt_v = hvx_vec_inverse_f32(rsqrt_v);
HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);
HVX_Vector denom_v = Q6_Vsf_vmax_VsfVsf(sqrt_v, epsilon_v);
HVX_Vector scale_v = hvx_vec_inverse_f32(denom_v);
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]);
HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v));
HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v));
v_dst[i] = hvx_vec_f32_to_f16(r0, r1);
}
if (nloe > 0) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16);
HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
HVX_VectorPair p = hvx_vec_f16_to_f32(v1);
HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v));
HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v));
HVX_Vector result = hvx_vec_f32_to_f16(r0, r1);
hvx_vec_store_a(&v_dst[nvec], nloe * SIZEOF_FP16, result);
}
}
#endif // HVX_NORM_H
ggml/src/ggml-hexagon/htp/hvx-scale.h
+66
View File
@@ -130,4 +130,70 @@ static inline void hvx_scale_offset_f32(uint8_t * restrict dst, const uint8_t *
}
}
// Scale+offset computed by promoting f16 -> f32, then narrowing the result back to f16.
#define hvx_scale_offset_f16_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
HVX_Vector vs = hvx_vec_splat_f32(scale); \
HVX_Vector vo = hvx_vec_splat_f32(offset); \
\
const uint32_t nvec = n / VLEN_FP16; \
const uint32_t nloe = n % VLEN_FP16; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; ++i) { \
HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \
HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), vs), vo)); \
HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), vs), vo)); \
vdst[i] = hvx_vec_f32_to_f16(r0, r1); \
} \
if (nloe) { \
HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \
HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), vs), vo)); \
HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), vs), vo)); \
HVX_Vector v = hvx_vec_f32_to_f16(r0, r1); \
vec_store((void *) &vdst[i], nloe * SIZEOF_FP16, v); \
} \
} while(0)
static inline void hvx_scale_offset_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {
assert((size_t) dst % 128 == 0);
assert((size_t) src % 128 == 0);
hvx_scale_offset_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
static inline void hvx_scale_offset_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {
assert((size_t) dst % 128 == 0);
hvx_scale_offset_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
}
static inline void hvx_scale_offset_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {
assert((size_t) src % 128 == 0);
hvx_scale_offset_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
}
static inline void hvx_scale_offset_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {
hvx_scale_offset_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
static inline void hvx_scale_offset_f16(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {
if (((size_t) dst & 127) == 0) {
if (((size_t) src & 127) == 0) {
hvx_scale_offset_f16_aa(dst, src, n, scale, offset);
} else {
hvx_scale_offset_f16_au(dst, src, n, scale, offset);
}
} else {
if (((size_t) src & 127) == 0) {
hvx_scale_offset_f16_ua(dst, src, n, scale, offset);
} else {
hvx_scale_offset_f16_uu(dst, src, n, scale, offset);
}
}
}
#endif // HVX_SCALE_H
ggml/src/ggml-hexagon/htp/hvx-sqrt.h
+63
View File
@@ -123,4 +123,67 @@ static inline void hvx_sqrt_f32(uint8_t * restrict dst, const uint8_t * restrict
}
}
// Compute sqrt(x) for f16 by promoting to f32, applying hvx_vec_rsqrt_f32, and narrowing back.
#define hvx_sqrt_f16_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t nvec = n / VLEN_FP16; \
const uint32_t nloe = n % VLEN_FP16; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \
HVX_Vector r0 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_lo_W(p)), Q6_V_lo_W(p)); \
HVX_Vector r1 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_hi_W(p)), Q6_V_hi_W(p)); \
vdst[i] = hvx_vec_f32_to_f16(r0, r1); \
} \
if (nloe) { \
HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \
HVX_Vector r0 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_lo_W(p)), Q6_V_lo_W(p)); \
HVX_Vector r1 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_hi_W(p)), Q6_V_hi_W(p)); \
HVX_Vector v = hvx_vec_f32_to_f16(r0, r1); \
vec_store((void *) &vdst[i], nloe * SIZEOF_FP16, v); \
} \
} while(0)
static inline void hvx_sqrt_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_sqrt_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
static inline void hvx_sqrt_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
hvx_sqrt_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
}
static inline void hvx_sqrt_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) src % 128 == 0);
hvx_sqrt_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
}
static inline void hvx_sqrt_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_sqrt_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
static inline void hvx_sqrt_f16(uint8_t * restrict dst, const uint8_t * restrict src, const int num_elems) {
if ((unsigned long) dst % 128 == 0) {
if ((unsigned long) src % 128 == 0) {
hvx_sqrt_f16_aa(dst, src, num_elems);
} else {
hvx_sqrt_f16_au(dst, src, num_elems);
}
} else {
if ((unsigned long) src % 128 == 0) {
hvx_sqrt_f16_ua(dst, src, num_elems);
} else {
hvx_sqrt_f16_uu(dst, src, num_elems);
}
}
}
#endif /* HVX_SQRT_H */
ggml/src/ggml-hexagon/htp/unary-ops.c
+229 -31
View File
@@ -234,6 +234,146 @@ static void sqrt_f32(const float * restrict src,
}
}
static void scale_f16(const _Float16 * restrict src,
_Float16 * restrict dst,
const uint32_t num_rows,
const struct htp_unary_context * uctx) {
htp_unary_op_preamble;
float scale = 0.f;
float bias = 0.f;
memcpy(&scale, &op_params[0], sizeof(float));
memcpy(&bias, &op_params[1], sizeof(float));
for (uint32_t ir = 0; ir < num_rows; ir++) {
const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
hvx_scale_offset_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0, scale, bias);
}
}
static void clamp_f16(const _Float16 * restrict src,
_Float16 * restrict dst,
const uint32_t num_rows,
const struct htp_unary_context * uctx) {
htp_unary_op_preamble;
float min = 0.f;
float max = 0.f;
memcpy(&min, &op_params[0], sizeof(float));
memcpy(&max, &op_params[1], sizeof(float));
for (uint32_t ir = 0; ir < num_rows; ir++) {
const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
hvx_clamp_scalar_f16(dst_local, src_local, (_Float16) min, (_Float16) max, ne0);
}
}
static void rms_norm_f16(const _Float16 * restrict src,
_Float16 * restrict dst,
const uint32_t num_rows,
const struct htp_unary_context * uctx) {
htp_unary_op_preamble;
float epsilon = 0.f;
memcpy(&epsilon, op_params, sizeof(float));
for (uint32_t ir = 0; ir < num_rows; ir++) {
const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
hvx_fast_rms_norm_f16((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon);
}
}
static void norm_f16(const _Float16 * restrict src,
_Float16 * restrict dst,
const uint32_t num_rows,
const struct htp_unary_context * uctx) {
htp_unary_op_preamble;
float epsilon = 0.f;
memcpy(&epsilon, op_params, sizeof(float));
for (uint32_t ir = 0; ir < num_rows; ir++) {
const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
hvx_fast_norm_f16((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon);
}
}
static void sqr_f16(const _Float16 * restrict src,
_Float16 * restrict dst,
const uint32_t num_rows,
const struct htp_unary_context * uctx) {
htp_unary_op_preamble;
for (uint32_t ir = 0; ir < num_rows; ir++) {
const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
hvx_sqr_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0);
}
}
static void sqrt_f16(const _Float16 * restrict src,
_Float16 * restrict dst,
const uint32_t num_rows,
const struct htp_unary_context * uctx) {
htp_unary_op_preamble;
for (uint32_t ir = 0; ir < num_rows; ir++) {
const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
hvx_sqrt_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0);
}
}
static void abs_f16(const _Float16 * restrict src,
_Float16 * restrict dst,
const uint32_t num_rows,
const struct htp_unary_context * uctx) {
htp_unary_op_preamble;
for (uint32_t ir = 0; ir < num_rows; ir++) {
const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
hvx_abs_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0);
}
}
static void log_f16(const _Float16 * restrict src,
_Float16 * restrict dst,
const uint32_t num_rows,
const struct htp_unary_context * uctx) {
htp_unary_op_preamble;
for (uint32_t ir = 0; ir < num_rows; ir++) {
const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
hvx_log_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0);
}
}
static void l2_norm_f16(const _Float16 * restrict src,
_Float16 * restrict dst,
const uint32_t num_rows,
const struct htp_unary_context * uctx) {
htp_unary_op_preamble;
float epsilon = 0.f;
memcpy(&epsilon, op_params, sizeof(float));
for (uint32_t ir = 0; ir < num_rows; ir++) {
const uint8_t * restrict src_f = (const uint8_t *)src + (ir * src0_row_size_aligned);
uint8_t * restrict dst_f = (uint8_t *)dst + (ir * dst_row_size_aligned);
hvx_fast_l2_norm_f16((const uint8_t *)src_f, (uint8_t *)dst_f, ne0, epsilon);
}
}
static void neg_f32(const float * restrict src,
float * restrict dst,
const uint32_t num_rows,
@@ -471,8 +611,8 @@ static void log_f32(const float * restrict src,
}
}
#define DEFINE_UNARY_TASK(NAME, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) \
static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * data) { \
#define DEFINE_UNARY_TASK_IMPL(NAME, TYPE, SUFFIX, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) \
static void unary_task_##SUFFIX##_##NAME(unsigned int nth, unsigned int ith, void * data) { \
const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; \
struct htp_ops_context * octx = uctx->octx; \
const struct htp_tensor * src = octx->src[0]; \
@@ -536,7 +676,7 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat
const uint32_t dst_max_block = block_dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1); \
const uint32_t BLOCK = MIN(src0_max_block, dst_max_block); \
if (BLOCK == 0) { \
FARF(ERROR, "unary-f32 : current VTCM reservation %zu is too small, needed at least %zu\n", \
FARF(ERROR, "unary-" #SUFFIX " : current VTCM reservation %zu is too small, needed at least %zu\n", \
uctx->vtcm_src0_size_per_thread, src0_row_size_aligned); \
return; \
} \
@@ -578,11 +718,11 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat
const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_src0_contig, block_dst_contig, \
ne01, div_ne01); \
\
float * dst_vtcm = (float *) dma_queue_pop(dma_queue).src; \
float * src0_vtcm = (float *) dma_queue_pop(dma_queue).dst; \
float * src1_vtcm = NULL; \
TYPE * dst_vtcm = (TYPE *) dma_queue_pop(dma_queue).src; \
TYPE * src0_vtcm = (TYPE *) dma_queue_pop(dma_queue).dst; \
TYPE * src1_vtcm = NULL; \
if ((IS_RMS_NORM_MUL) && !uctx->broadcast_weight) { \
src1_vtcm = (float *) dma_queue_pop(dma_queue).dst; \
src1_vtcm = (TYPE *) dma_queue_pop(dma_queue).dst; \
} \
\
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \
@@ -625,6 +765,10 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat
dma_queue_flush(dma_queue); \
}
// F32 unary task: row-block DMA/VTCM plumbing, float-typed VTCM buffers.
#define DEFINE_UNARY_TASK(NAME, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) \
DEFINE_UNARY_TASK_IMPL(NAME, float, f32, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR)
DEFINE_UNARY_TASK(norm, false, false, norm_f32(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK(rms_norm, false, false, rms_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK(rms_norm_mul, true, false, rms_norm_mul_f32(src0_vtcm, uctx->broadcast_weight ? (const float *) src1_vtcm_data : src1_vtcm, dst_vtcm, block_size, uctx))
@@ -644,6 +788,18 @@ DEFINE_UNARY_TASK(unary_log, false, false, log_f32(src0_vtcm, dst_vtcm, blo
DEFINE_UNARY_TASK(l2_norm, false, false, l2_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK(tri, false, true, tri_f32(src0_vtcm, dst_vtcm, block_size, ir, uctx))
// F16 unary tasks: same DMA/VTCM plumbing as DEFINE_UNARY_TASK, but VTCM buffers are
// _Float16-typed. None of the current F16 ops need RMS_NORM_MUL or TRI support.
DEFINE_UNARY_TASK_IMPL(norm, _Float16, f16, false, false, norm_f16(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK_IMPL(rms_norm, _Float16, f16, false, false, rms_norm_f16(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK_IMPL(scale, _Float16, f16, false, false, scale_f16(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK_IMPL(clamp, _Float16, f16, false, false, clamp_f16(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK_IMPL(sqr, _Float16, f16, false, false, sqr_f16(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK_IMPL(sqrt, _Float16, f16, false, false, sqrt_f16(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK_IMPL(l2_norm, _Float16, f16, false, false, l2_norm_f16(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK_IMPL(unary_abs, _Float16, f16, false, false, abs_f16(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK_IMPL(unary_log, _Float16, f16, false, false, log_f16(src0_vtcm, dst_vtcm, block_size, uctx))
// Apply a pointwise unary op to one column tile that is already in VTCM.
#define DEFINE_UNARY_TILED_TASK(NAME, IS_TRI, CORE_TILE_EXPR) \
static void unary_task_f32_tiled_##NAME(unsigned int nth, unsigned int ith, void * data) { \
@@ -892,50 +1048,76 @@ DEFINE_UNARY_TILED_TASK(unary_abs, false, hvx_abs_f32_aa(dst_vtcm, src_vtcm
DEFINE_UNARY_TILED_TASK(unary_log, false, hvx_log_f32_aa(dst_vtcm, src_vtcm, tw))
DEFINE_UNARY_TILED_TASK(tri, true, tri_apply_tile_f32(src_vtcm, dst_vtcm, tw, col, i01, ne0, tri_ttype))
static int execute_op_unary_f32(struct htp_ops_context * octx) {
static int execute_op_unary(struct htp_ops_context * octx) {
int err = HTP_STATUS_OK;
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
const bool is_f16 = (src0->type == HTP_TYPE_F16);
const char * op_type = NULL;
switch (octx->op) {
case HTP_OP_NORM: op_type = "norm-f32"; break;
case HTP_OP_RMS_NORM: op_type = "rmsnorm-f32"; break;
case HTP_OP_RMS_NORM_MUL: op_type = "rmsnorm-mul-f32"; break;
case HTP_OP_SCALE: op_type = "scale-f32"; break;
case HTP_OP_CLAMP: op_type = "clamp-f32"; break;
case HTP_OP_SQR: op_type = "sqr-f32"; break;
case HTP_OP_SQRT: op_type = "sqrt-f32"; break;
case HTP_OP_UNARY_NEG: op_type = "neg-f32"; break;
case HTP_OP_UNARY_EXP: op_type = "exp-f32"; break;
case HTP_OP_UNARY_SIGMOID: op_type = "sigmoid-f32"; break;
case HTP_OP_UNARY_SILU: op_type = "silu-f32"; break;
case HTP_OP_UNARY_GELU: op_type = "gelu-f32"; break;
case HTP_OP_UNARY_SOFTPLUS: op_type = "softplus-f32"; break;
case HTP_OP_UNARY_TANH: op_type = "tanh-f32"; break;
case HTP_OP_UNARY_ABS: op_type = "abs-f32"; break;
case HTP_OP_UNARY_LOG: op_type = "log-f32"; break;
case HTP_OP_L2_NORM: op_type = "l2norm-f32"; break;
case HTP_OP_TRI: op_type = "tri-f32"; break;
case HTP_OP_NORM: op_type = is_f16 ? "norm-f16" : "norm-f32"; break;
case HTP_OP_RMS_NORM: op_type = is_f16 ? "rmsnorm-f16" : "rmsnorm-f32"; break;
case HTP_OP_RMS_NORM_MUL: op_type = "rmsnorm-mul-f32"; break;
case HTP_OP_SCALE: op_type = is_f16 ? "scale-f16" : "scale-f32"; break;
case HTP_OP_CLAMP: op_type = is_f16 ? "clamp-f16" : "clamp-f32"; break;
case HTP_OP_SQR: op_type = is_f16 ? "sqr-f16" : "sqr-f32"; break;
case HTP_OP_SQRT: op_type = is_f16 ? "sqrt-f16" : "sqrt-f32"; break;
case HTP_OP_UNARY_NEG: op_type = "neg-f32"; break;
case HTP_OP_UNARY_EXP: op_type = "exp-f32"; break;
case HTP_OP_UNARY_SIGMOID: op_type = "sigmoid-f32"; break;
case HTP_OP_UNARY_SILU: op_type = "silu-f32"; break;
case HTP_OP_UNARY_GELU: op_type = "gelu-f32"; break;
case HTP_OP_UNARY_SOFTPLUS: op_type = "softplus-f32"; break;
case HTP_OP_UNARY_TANH: op_type = "tanh-f32"; break;
case HTP_OP_UNARY_ABS: op_type = is_f16 ? "abs-f16" : "abs-f32"; break;
case HTP_OP_UNARY_LOG: op_type = is_f16 ? "log-f16" : "log-f32"; break;
case HTP_OP_L2_NORM: op_type = is_f16 ? "l2norm-f16" : "l2norm-f32"; break;
case HTP_OP_TRI: op_type = "tri-f32"; break;
default:
FARF(ERROR, "Unsupported unary Op %u\n", octx->op);
return HTP_STATUS_NO_SUPPORT;
}
// F16 only has row-block kernels for this subset of ops (see the dispatch switch
// below) - reject everything else up front, before touching kparams/VTCM.
if (is_f16) {
switch (octx->op) {
case HTP_OP_NORM:
case HTP_OP_RMS_NORM:
case HTP_OP_SCALE:
case HTP_OP_CLAMP:
case HTP_OP_SQR:
case HTP_OP_SQRT:
case HTP_OP_L2_NORM:
case HTP_OP_UNARY_ABS:
case HTP_OP_UNARY_LOG:
break;
default:
FARF(ERROR, "unary-%s: not supported for F16\n", op_type);
return HTP_STATUS_NO_SUPPORT;
}
}
const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params;
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
const uint32_t n_threads = kparams->n_threads;
const size_t src0_data_row_size = src0->ne[0] * sizeof(float);
const size_t dst_data_row_size = dst->ne[0] * sizeof(float);
const size_t elem_size = is_f16 ? sizeof(_Float16) : sizeof(float);
const size_t src0_data_row_size = src0->ne[0] * elem_size;
const size_t dst_data_row_size = dst->ne[0] * elem_size;
const size_t src0_row_size_aligned = kparams->src0_row_size_aligned;
const size_t dst_row_size_aligned = kparams->dst_row_size_aligned;
// Always 0 for F16 - htp_unary_vtcm_layout_build() keeps F16 on the row-block path,
// since only F32 has unary_task_f32_tiled_* kernels.
const uint32_t col_tile = kparams->col_tile;
size_t src1_data_row_size = 0;
@@ -943,6 +1125,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
bool broadcast_weight = kparams->broadcast_weight;
const struct htp_tensor * src1 = NULL;
// RMS_NORM_MUL fusion is F32-only (its weight tensor is always F32; see
// try_fuse_node()'s type guard), so this never triggers when is_f16 is true.
if (octx->op == HTP_OP_RMS_NORM_MUL) {
src1 = octx->src[1];
src1_data_row_size = src1->ne[0] * sizeof(float);
@@ -987,7 +1171,7 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
.block = kparams->block,
.nc = src0->ne[0],
.col_tile = (uint32_t) kparams->col_tile,
.col_tile = col_tile,
.broadcast_weight = broadcast_weight,
.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, 0),
@@ -1020,6 +1204,19 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
case HTP_OP_TRI: task_func = unary_task_f32_tiled_tri; break;
default: break;
}
} else if (is_f16) {
switch (octx->op) {
case HTP_OP_NORM: task_func = unary_task_f16_norm; break;
case HTP_OP_RMS_NORM: task_func = unary_task_f16_rms_norm; break;
case HTP_OP_SCALE: task_func = unary_task_f16_scale; break;
case HTP_OP_CLAMP: task_func = unary_task_f16_clamp; break;
case HTP_OP_SQR: task_func = unary_task_f16_sqr; break;
case HTP_OP_SQRT: task_func = unary_task_f16_sqrt; break;
case HTP_OP_L2_NORM: task_func = unary_task_f16_l2_norm; break;
case HTP_OP_UNARY_ABS: task_func = unary_task_f16_unary_abs; break;
case HTP_OP_UNARY_LOG: task_func = unary_task_f16_unary_log; break;
default: break;
}
} else {
switch (octx->op) {
case HTP_OP_NORM: task_func = unary_task_f32_norm; break;
@@ -1047,7 +1244,7 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
if (task_func) {
worker_pool_run_func(octx->ctx->worker_pool, task_func, &uctx, n_threads);
} else {
FARF(ERROR, "execute_op_unary_f32: task function is NULL for op %d\n", octx->op);
FARF(ERROR, "execute_op_unary: task function is NULL for op %d\n", octx->op);
err = HTP_STATUS_NO_SUPPORT;
}
}
@@ -1058,7 +1255,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
int op_unary(struct htp_ops_context * octx) {
switch (octx->src[0]->type) {
case HTP_TYPE_F32:
return execute_op_unary_f32(octx);
case HTP_TYPE_F16:
return execute_op_unary(octx);
default:
return HTP_STATUS_NO_SUPPORT;
ggml/src/ggml-hexagon/htp/unary-ops.h
+13 -4
View File
@@ -85,17 +85,19 @@ static inline void htp_unary_vtcm_layout_build(
bool broadcast_weight,
uint32_t n_threads,
size_t vtcm_size,
size_t elem_size,
uint32_t * out_col_tile,
uint32_t * out_vtcm_row_per_thread
) {
const size_t src0_data_row_size = ne00 * sizeof(float);
const size_t dst_data_row_size = ne10 * sizeof(float);
const size_t src0_data_row_size = ne00 * elem_size;
const size_t dst_data_row_size = ne10 * elem_size;
const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128);
const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, 128);
size_t src1_row_size_aligned = 0;
if (op == HTP_OP_RMS_NORM_MUL) {
// RMS_NORM_MUL fusion is F32-only; its weight tensor is always F32.
const size_t src1_data_row_size = ne11 * sizeof(float);
src1_row_size_aligned = hex_round_up(src1_data_row_size, 128);
}
@@ -125,12 +127,19 @@ static inline void htp_unary_vtcm_layout_build(
const bool is_reduction = (op == HTP_OP_NORM || op == HTP_OP_RMS_NORM ||
op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_L2_NORM);
// The tiled fallback path below only has F32 task functions (unary_task_f32_tiled_*);
// F16 has no tiled kernels, so it must stay on the row-block path like reduction ops.
// NOTE: if F16 ends up with vtcm_row_per_thread == 0 here (row too large for the VTCM
// budget), execute_op_unary() will see BLOCK == 0 and skip computation for that op
// (logged via FARF(ERROR, ...)) since there is no F16 tiled fallback. This is a known
// limitation; supporting it would require adding F16 tiled kernels.
const bool is_f16 = (elem_size == sizeof(_Float16));
uint32_t col_tile = 0;
if (vtcm_row_per_thread == 0 && !is_reduction) {
if (vtcm_row_per_thread == 0 && !is_reduction && !is_f16) {
const size_t per_thread_budget = vtcm_size / n_threads;
const size_t col_tile_bytes = hex_align_down(per_thread_budget / 4, 128);
col_tile = (uint32_t) (col_tile_bytes / sizeof(float));
col_tile = (uint32_t) (col_tile_bytes / elem_size);
L->src0_bytes = col_tile_bytes * 2;
L->dst_bytes = col_tile_bytes * 2;