From 4fea119de30f6a923992780f6fd5ccb0bee5d47d Mon Sep 17 00:00:00 2001 From: bri-prism <288398250+bri-prism@users.noreply.github.com> Date: Fri, 18 Sep 2026 07:17:38 -0700 Subject: [PATCH] ggml-cpu: add F16 input to the FWHT (#27779) * ggml-cpu: add F16 input to the FWHT The CPU FWHT accepts F32 input only. This change makes the source type a template parameter. The CPU path now accepts F16 input and F32 input. The CPU MUL_MAT reference now converts an F16 src1 to F32. It does this when the caller sets the Hadamard hint. No backend has an F16 FWHT kernel yet. The test cases come with the backend changes that add one. * ggml-cpu: assert the F16 FWHT input path, and use the bulk converter Address review feedback. The F16 branch writes plain floats into wdata, which is only correct when vec_dot_type is F32. That invariant held because supports_op only accepts an F16 src1 for the Hadamard hint with F32 src0 and dst, but nothing enforced it. Assert it next to the existing src1 type check so widening supports_op cannot silently break the write. Replace the hand-rolled conversion loop with ggml_cpu_fp16_to_fp32. --- ggml/src/ggml-cpu/ggml-cpu.c | 16 ++++++++++++---- ggml/src/ggml-cpu/ggml-cpu.cpp | 4 ++++ ggml/src/ggml-cpu/ops.cpp | 26 ++++++++++++++++++++------ 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 87a329f269..8bb0ff7bc3 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -1329,7 +1329,9 @@ UseGgmlGemm1:; const size_t nbw3 = nbw2*ne12; assert(params->wsize >= ne13*nbw3); - GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT(src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); + // the F16 path below writes plain floats into wdata, so it needs an F32 vec_dot_type + GGML_ASSERT(src1->type == GGML_TYPE_F32 || vec_dot_type == GGML_TYPE_F32); #if 0 for (int64_t i13 = 0; i13 < ne13; ++i13) { @@ -1348,9 +1350,15 @@ UseGgmlGemm1:; size_t bs = ggml_blck_size(vec_dot_type); int64_t ne10_block_start = (ith * ne10/bs) / nth; int64_t ne10_block_end = ((ith + 1) * ne10/bs) / nth; - from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10), - (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0), - (ne10_block_end - ne10_block_start) * bs); + const char * src1_block = (const char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10; + char * dst_block = wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0; + const int64_t n_block = (ne10_block_end - ne10_block_start) * bs; + + if (src1->type == GGML_TYPE_F32) { + from_float((const float *) src1_block, dst_block, n_block); + } else { + ggml_cpu_fp16_to_fp32((const ggml_fp16_t *) src1_block, (float *) dst_block, n_block); + } } } } diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp index 8cece71f18..1df0f2bb92 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.cpp +++ b/ggml/src/ggml-cpu/ggml-cpu.cpp @@ -451,6 +451,10 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st op->type != GGML_TYPE_IQ1_S && op->type != GGML_TYPE_IQ1_M; // missing type_traits.from_float case GGML_OP_MUL_MAT: + if (ggml_get_op_params_i32(op, 1) == GGML_HINT_SRC0_IS_HADAMARD && + src0->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32) { + return src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16; + } return src1->type == GGML_TYPE_F32 || src1->type == ggml_get_type_traits_cpu(src0->type)->vec_dot_type; case GGML_OP_SOFT_MAX_BACK: { if (op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type != GGML_TYPE_F32) { diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 23001254c1..ba00a0a73e 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -12015,11 +12015,20 @@ void ggml_compute_forward_opt_step_sgd(const ggml_compute_params * params, ggml_ } } -static void ggml_compute_forward_fwht_f32(const ggml_compute_params * params, ggml_tensor * dst) { +static inline float ggml_fwht_load(const float value) { + return value; +} + +static inline float ggml_fwht_load(const ggml_fp16_t value) { + return ggml_fp16_to_fp32(value); +} + +template +static void ggml_compute_forward_fwht_impl(const ggml_compute_params * params, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; - GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT(src1->type == (std::is_same_v ? GGML_TYPE_F32 : GGML_TYPE_F16)); GGML_ASSERT(dst->type == GGML_TYPE_F32); GGML_TENSOR_BINARY_OP_LOCALS @@ -12046,11 +12055,11 @@ static void ggml_compute_forward_fwht_f32(const ggml_compute_params * params, gg const int64_t i12 = (r - i13 * ne11 * ne12) / ne11; const int64_t i11 = r - i13 * ne11 * ne12 - i12 * ne11; - const float * src_row = (const float *) ((const char *) src1->data + i11 * nb11 + i12 * nb12 + i13 * nb13); + const src_t * src_row = (const src_t *) ((const char *) src1->data + i11 * nb11 + i12 * nb12 + i13 * nb13); float * dst_row = (float *) ((char *) dst->data + i11 * nb1 + i12 * nb2 + i13 * nb3); for (int64_t j = 0; j < n; j++) { - dst_row[j] = src_row[j] * scale; + dst_row[j] = ggml_fwht_load(src_row[j]) * scale; } // Scalar passes @@ -12097,12 +12106,17 @@ void ggml_compute_forward_fwht(const ggml_compute_params * params, ggml_tensor * switch (src1->type) { case GGML_TYPE_F32: { - ggml_compute_forward_fwht_f32(params, dst); + ggml_compute_forward_fwht_impl(params, dst); + } + break; + case GGML_TYPE_F16: + { + ggml_compute_forward_fwht_impl(params, dst); } break; default: { - GGML_ABORT("fatal error - fwht is F32 only"); + GGML_ABORT("fatal error - fwht supports F32 and F16 input"); } } }