Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-09 14:29:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add IQ type handling for MoE (#28476)

Browse Source 
Co-authored-by: cwriter <cwriter@localhost>
This commit is contained in:
cwriter
2026-09-09 03:25:41 +02:00
committed by GitHub
parent f3f1a8f276
commit 304665fe7a
2 changed files with 94 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-sycl/mmvq.cpp
+73
View File
@@ -2671,6 +2671,34 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens
GGML_UNUSED(ctx);
}
// vec_dot_q_sycl_t adapters for the IQ vec_dots that take their codebook tables as extra
// arguments: bind the constant tables here (as vec_dot_iq2_s_q8_1 / vec_dot_iq1_m_q8_1 already do
// internally) so they can be used as template arguments of mul_mat_vec_q_moe.
static __dpct_inline__ float vec_dot_iq2_xxs_q8_1_moe(const void * __restrict__ vbq,
const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
return vec_dot_iq2_xxs_q8_1(vbq, bq8_1, iqs, iq2xxs_grid, ksigns_iq2xs, kmask_iq2xs);
}
static __dpct_inline__ float vec_dot_iq2_xs_q8_1_moe(const void * __restrict__ vbq,
const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
return vec_dot_iq2_xs_q8_1(vbq, bq8_1, iqs, iq2xs_grid, ksigns64);
}
static __dpct_inline__ float vec_dot_iq3_xxs_q8_1_moe(const void * __restrict__ vbq,
const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
return vec_dot_iq3_xxs_q8_1(vbq, bq8_1, iqs, iq3xxs_grid, ksigns64);
}
static __dpct_inline__ float vec_dot_iq3_s_q8_1_moe(const void * __restrict__ vbq,
const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
return vec_dot_iq3_s_q8_1(vbq, bq8_1, iqs, iq3s_grid);
}
static __dpct_inline__ float vec_dot_iq1_s_q8_1_moe(const void * __restrict__ vbq,
const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
return vec_dot_iq1_s_q8_1(vbq, bq8_1, iqs, iq1s_grid_gpu);
}
// src1_row_stride: 0 for shared src1 (gate/up proj), else per-expert stride (down proj).
template <int qk, int qi, typename block_q_t, int vdr, vec_dot_q_sycl_t vec_dot_q_sycl>
static void mul_mat_vec_q_moe(
@@ -2822,6 +2850,51 @@ bool ggml_sycl_mul_mat_vec_q_id(
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
return true;
case GGML_TYPE_IQ2_XXS:
launch_mul_mat_vec_q_moe<QK_K, QI2_XXS/2, block_iq2_xxs, VDR_IQ2_XXS_Q8_1_MMVQ, vec_dot_iq2_xxs_q8_1_moe>(
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
return true;
case GGML_TYPE_IQ2_XS:
launch_mul_mat_vec_q_moe<QK_K, QI2_XS/2, block_iq2_xs, VDR_IQ2_XS_Q8_1_MMVQ, vec_dot_iq2_xs_q8_1_moe>(
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
return true;
case GGML_TYPE_IQ2_S:
launch_mul_mat_vec_q_moe<QK_K, QI2_S/2, block_iq2_s, VDR_IQ2_S_Q8_1_MMVQ, vec_dot_iq2_s_q8_1>(
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
return true;
case GGML_TYPE_IQ3_XXS:
launch_mul_mat_vec_q_moe<QK_K, QI3_XXS/2, block_iq3_xxs, VDR_IQ3_XXS_Q8_1_MMVQ, vec_dot_iq3_xxs_q8_1_moe>(
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
return true;
case GGML_TYPE_IQ3_S:
launch_mul_mat_vec_q_moe<QK_K, QI3_S/2, block_iq3_s, VDR_IQ3_S_Q8_1_MMVQ, vec_dot_iq3_s_q8_1_moe>(
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
return true;
case GGML_TYPE_IQ1_S:
launch_mul_mat_vec_q_moe<QK_K, QI1_S, block_iq1_s, VDR_IQ1_S_Q8_1_MMVQ, vec_dot_iq1_s_q8_1_moe>(
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
return true;
case GGML_TYPE_IQ1_M:
launch_mul_mat_vec_q_moe<QK_K, QI1_S, block_iq1_m, VDR_IQ1_M_Q8_1_MMVQ, vec_dot_iq1_m_q8_1>(
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
return true;
case GGML_TYPE_IQ4_NL:
launch_mul_mat_vec_q_moe<QK4_NL, QI4_NL, block_iq4_nl, VDR_IQ4_NL_Q8_1_MMVQ, vec_dot_iq4_nl_q8_1>(
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
return true;
case GGML_TYPE_IQ4_XS:
launch_mul_mat_vec_q_moe<QK_K, QI4_XS/4, block_iq4_xs, VDR_IQ4_XS_Q8_1_MMVQ, vec_dot_iq4_xs_q8_1>(
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
return true;
default:
return false;
}
ggml/src/ggml-sycl/vecdotq.hpp
+21
View File
@@ -1398,6 +1398,11 @@ vec_dot_q6_K_q8_1(const void *__restrict__ vbq,
}
// NOTE: the VDR_IQ*_Q8_1_MMVQ values deliberately differ from the identically named CUDA constants
// (vecdotq.cuh): the SYCL kernels pair them with a halved qi (e.g. QI3_S/2), so the values are not
// interchangeable and must not be copied across backends.
#define VDR_IQ2_XXS_Q8_1_MMVQ 1
static __dpct_inline__ float
vec_dot_iq2_xxs_q8_1(const void *__restrict__ vbq,
const block_q8_1 *__restrict__ bq8_1, const int &iqs,
@@ -1429,6 +1434,8 @@ vec_dot_iq2_xxs_q8_1(const void *__restrict__ vbq,
#endif
}
#define VDR_IQ2_XS_Q8_1_MMVQ 1
static __dpct_inline__ float
vec_dot_iq2_xs_q8_1(const void *__restrict__ vbq,
const block_q8_1 *__restrict__ bq8_1, const int &iqs,
@@ -1479,6 +1486,8 @@ vec_dot_iq2_xs_q8_1(const void *__restrict__ vbq,
#endif
}
#define VDR_IQ2_S_Q8_1_MMVQ 1
static __dpct_inline__ float
vec_dot_iq2_s_q8_1(const void *__restrict__ vbq,
const block_q8_1 *__restrict__ bq8_1, const int &iqs) {
@@ -1531,6 +1540,8 @@ vec_dot_iq2_s_q8_1(const void *__restrict__ vbq,
#endif
}
#define VDR_IQ3_XXS_Q8_1_MMVQ 1
static __dpct_inline__ float
vec_dot_iq3_xxs_q8_1(const void *__restrict__ vbq,
const block_q8_1 *__restrict__ bq8_1, const int &iqs,
@@ -1571,6 +1582,8 @@ vec_dot_iq3_xxs_q8_1(const void *__restrict__ vbq,
#endif
}
#define VDR_IQ3_S_Q8_1_MMVQ 1
static __dpct_inline__ float
vec_dot_iq3_s_q8_1(const void *__restrict__ vbq,
const block_q8_1 *__restrict__ bq8_1, const int &iqs,
@@ -1609,6 +1622,8 @@ vec_dot_iq3_s_q8_1(const void *__restrict__ vbq,
#endif
}
#define VDR_IQ1_S_Q8_1_MMVQ 1
static __dpct_inline__ float
vec_dot_iq1_s_q8_1(const void *__restrict__ vbq,
const block_q8_1 *__restrict__ bq8_1, const int &iqs,
@@ -1637,6 +1652,8 @@ vec_dot_iq1_s_q8_1(const void *__restrict__ vbq,
#endif
}
#define VDR_IQ1_M_Q8_1_MMVQ 1
static __dpct_inline__ float
vec_dot_iq1_m_q8_1(const void *__restrict__ vbq,
const block_q8_1 *__restrict__ bq8_1, const int &iqs) {
@@ -1671,6 +1688,8 @@ vec_dot_iq1_m_q8_1(const void *__restrict__ vbq,
}
#define VDR_IQ4_NL_Q8_1_MMVQ 2
static __dpct_inline__ float
vec_dot_iq4_nl_q8_1(const void *__restrict__ vbq,
const block_q8_1 *__restrict__ bq8_1, const int &iqs) {
@@ -1696,6 +1715,8 @@ vec_dot_iq4_nl_q8_1(const void *__restrict__ vbq,
}
#define VDR_IQ4_XS_Q8_1_MMVQ 1
static __dpct_inline__ float
vec_dot_iq4_xs_q8_1(const void *__restrict__ vbq,
const block_q8_1 *__restrict__ bq8_1, const int &iqs) {