From 884729830d8864dce063d663ad789d08644e6395 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Wed, 19 Aug 2026 10:38:19 +0200 Subject: [PATCH] ggml: support ggml_rope_set_offset on vulkan (#27344) * ggml: support ggml_rope_set_offset on vulkan * remove inplace optimization --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 15 ++--- .../vulkan-shaders/rope_funcs.glsl | 64 +++++++++++-------- .../vulkan-shaders/rope_params.glsl | 1 + 3 files changed, 44 insertions(+), 36 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 5ce83d7e97..95accd3113 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -1644,6 +1644,7 @@ struct vk_op_rope_push_constants { uint32_t rope_mode; uint32_t nrows; uint32_t n_dims; + uint32_t n_offs; float freq_scale; float freq_base; float ext_factor; @@ -13120,6 +13121,7 @@ static uint32_t ggml_vk_rms_partials_size(ggml_backend_vk_context * ctx, const g static vk_op_rope_push_constants ggml_vk_make_rope_constants(const ggml_tensor *dst, const ggml_tensor *src0, const bool has_ff, bool backprop, const uint32_t set_rows_stride) { const int n_dims = ((const int32_t *) dst->op_params)[1]; const int mode = ((const int32_t *) dst->op_params)[2]; + const int n_offs = ((const int32_t *) dst->op_params)[15]; // const int n_ctx = ((const int32_t *) dst->op_params)[3]; const int n_ctx_orig = ((const int32_t *) dst->op_params)[4]; const float freq_base = ((const float *) dst->op_params)[5]; @@ -13149,7 +13151,7 @@ static vk_op_rope_push_constants ggml_vk_make_rope_constants(const ggml_tensor * uint32_t nb13 = dst->nb[3] / ggml_type_size(dst->type); vk_op_rope_push_constants rope { - (uint32_t)mode, (uint32_t)ggml_nrows(src0), (uint32_t)n_dims, freq_scale, + (uint32_t)mode, (uint32_t)ggml_nrows(src0), (uint32_t)n_dims, (uint32_t)n_offs, freq_scale, freq_base, ext_factor, attn_factor, {corr_dims[0], corr_dims[1]}, theta_scale, has_ff, { sections[0], sections[1], sections[2], sections[3] }, is_imrope, backprop, set_rows_stride, @@ -18253,15 +18255,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_OP_REPEAT_BACK: return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32; case GGML_OP_ROPE: - if (((const int32_t *) op->op_params)[15] != 0) { - return false; // FIXME: support ggml_rope_set_offset - } return ggml_is_contiguous_rows(op) && ggml_is_contiguous_rows(op->src[0]); case GGML_OP_ROPE_BACK: - if (((const int32_t *) op->op_params)[15] != 0) { - return false; // FIXME: support ggml_rope_set_offset - } - return true; case GGML_OP_NONE: case GGML_OP_RESHAPE: case GGML_OP_VIEW: @@ -19202,6 +19197,10 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * tensor_clone = ggml_rope_ext_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); } } + const int n_offs = ((int32_t *) tensor->op_params)[15]; + if (n_offs != 0) { + tensor_clone = ggml_rope_set_offset(tensor_clone, n_offs); + } } else if (tensor->op == GGML_OP_UNARY) { switch (ggml_get_unary_op(tensor)) { case GGML_UNARY_OP_EXP: diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl index 0335879314..feb55b2039 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl @@ -50,19 +50,21 @@ void rope_norm(const uint i0, const uint i1, const uint i2, const uint i3, rope_ } idst += p.d_offset; - if (i0 >= p.n_dims) { + if (i0 < p.n_offs || i0 >= p.n_offs + p.n_dims) { rope_data_d[idst + 0] = ROPE_D_TYPE(rope_data_a[ix + 0]); rope_data_d[idst + 1] = ROPE_D_TYPE(rope_data_a[ix + 1]); return; } - const float theta_base = rope_data_pos[i2] * pow(p.theta_scale, i0/2.0f); + const uint iw = i0 - p.n_offs; // relative idx - const float freq_factor = p.has_ff != 0 ? rope_data_ff[i0/2] : 1.0f; + const float theta_base = rope_data_pos[i2] * pow(p.theta_scale, iw/2.0f); + + const float freq_factor = p.has_ff != 0 ? rope_data_ff[iw/2] : 1.0f; float cos_theta, sin_theta; - rope_yarn(theta_base / freq_factor, i0, cos_theta, sin_theta, p); + rope_yarn(theta_base / freq_factor, iw, cos_theta, sin_theta, p); const float x0 = float(rope_data_a[ix + 0]); const float x1 = float(rope_data_a[ix + 1]); @@ -87,25 +89,28 @@ void rope_neox(const uint i0, const uint i1, const uint i2, const uint i3, rope_ } idst += p.d_offset; - if (i0 >= p.n_dims) { + if (i0 < p.n_offs || i0 >= p.n_offs + p.n_dims) { rope_data_d[idst + i0/2 + 0] = ROPE_D_TYPE(rope_data_a[ix + i0/2 + 0]); rope_data_d[idst + i0/2 + 1] = ROPE_D_TYPE(rope_data_a[ix + i0/2 + 1]); return; } - const float theta_base = rope_data_pos[i2] * pow(p.theta_scale, i0/2.0f); + const uint iw = i0 - p.n_offs; // relative idx - const float freq_factor = p.has_ff != 0 ? rope_data_ff[i0/2] : 1.0f; + const float theta_base = rope_data_pos[i2] * pow(p.theta_scale, iw/2.0f); + + const float freq_factor = p.has_ff != 0 ? rope_data_ff[iw/2] : 1.0f; float cos_theta, sin_theta; - rope_yarn(theta_base / freq_factor, i0, cos_theta, sin_theta, p); + rope_yarn(theta_base / freq_factor, iw, cos_theta, sin_theta, p); - const float x0 = float(rope_data_a[ix + 0]); - const float x1 = float(rope_data_a[ix + p.n_dims/2]); + // idst/ix point at channel i0/2; the first channel of the rotated pair is p.n_offs + iw/2 = i0/2 + p.n_offs/2 + const float x0 = float(rope_data_a[ix + p.n_offs/2 + 0]); + const float x1 = float(rope_data_a[ix + p.n_offs/2 + p.n_dims/2]); - rope_data_d[idst + 0] = ROPE_D_TYPE(x0*cos_theta - x1*sin_theta); - rope_data_d[idst + p.n_dims/2] = ROPE_D_TYPE(x0*sin_theta + x1*cos_theta); + rope_data_d[idst + p.n_offs/2 + 0] = ROPE_D_TYPE(x0*cos_theta - x1*sin_theta); + rope_data_d[idst + p.n_offs/2 + p.n_dims/2] = ROPE_D_TYPE(x0*sin_theta + x1*cos_theta); } @@ -125,53 +130,56 @@ void rope_multi(const uint i0, const uint i1, const uint i2, const uint i3, rope } idst += p.d_offset; - if (i0 >= p.n_dims) { + if (i0 < p.n_offs || i0 >= p.n_offs + p.n_dims) { rope_data_d[idst + i0/2 + 0] = ROPE_D_TYPE(rope_data_a[ix + i0/2 + 0]); rope_data_d[idst + i0/2 + 1] = ROPE_D_TYPE(rope_data_a[ix + i0/2 + 1]); return; } + const uint iw = i0 - p.n_offs; // relative idx + const int sect_dims = p.sections[0] + p.sections[1] + p.sections[2] + p.sections[3]; const int sec_w = p.sections[1] + p.sections[0]; - const uint sector = (i0 / 2) % sect_dims; + const uint sector = (iw / 2) % sect_dims; float theta_base = 0.0; if (p.is_imrope != 0) { if (sector % 3 == 1 && sector < 3 * p.sections[1]) { - theta_base = rope_data_pos[i2 + p.ne02 * 1]*pow(p.theta_scale, i0/2.0f); + theta_base = rope_data_pos[i2 + p.ne02 * 1]*pow(p.theta_scale, iw/2.0f); } else if (sector % 3 == 2 && sector < 3 * p.sections[2]) { - theta_base = rope_data_pos[i2 + p.ne02 * 2]*pow(p.theta_scale, i0/2.0f); + theta_base = rope_data_pos[i2 + p.ne02 * 2]*pow(p.theta_scale, iw/2.0f); } else if (sector % 3 == 0 && sector < 3 * p.sections[0]) { - theta_base = rope_data_pos[i2]*pow(p.theta_scale, i0/2.0f); + theta_base = rope_data_pos[i2]*pow(p.theta_scale, iw/2.0f); } else { - theta_base = rope_data_pos[i2 + p.ne02 * 3]*pow(p.theta_scale, i0/2.0f); + theta_base = rope_data_pos[i2 + p.ne02 * 3]*pow(p.theta_scale, iw/2.0f); } } else { if (sector < p.sections[0]) { - theta_base = rope_data_pos[i2]*pow(p.theta_scale, i0/2.0f); + theta_base = rope_data_pos[i2]*pow(p.theta_scale, iw/2.0f); } else if (sector >= p.sections[0] && sector < sec_w) { - theta_base = rope_data_pos[i2 + p.ne02 * 1]*pow(p.theta_scale, i0/2.0f); + theta_base = rope_data_pos[i2 + p.ne02 * 1]*pow(p.theta_scale, iw/2.0f); } else if (sector >= sec_w && sector < sec_w + p.sections[2]) { - theta_base = rope_data_pos[i2 + p.ne02 * 2]*pow(p.theta_scale, i0/2.0f); + theta_base = rope_data_pos[i2 + p.ne02 * 2]*pow(p.theta_scale, iw/2.0f); } else if (sector >= sec_w + p.sections[2]) { - theta_base = rope_data_pos[i2 + p.ne02 * 3]*pow(p.theta_scale, i0/2.0f); + theta_base = rope_data_pos[i2 + p.ne02 * 3]*pow(p.theta_scale, iw/2.0f); } } - const float freq_factor = p.has_ff != 0 ? rope_data_ff[i0/2] : 1.0f; + const float freq_factor = p.has_ff != 0 ? rope_data_ff[iw/2] : 1.0f; float cos_theta, sin_theta; - rope_yarn(theta_base / freq_factor, i0, cos_theta, sin_theta, p); + rope_yarn(theta_base / freq_factor, iw, cos_theta, sin_theta, p); - const float x0 = float(rope_data_a[ix + 0]); - const float x1 = float(rope_data_a[ix + p.n_dims/2]); + // idst/ix point at channel i0/2; the first channel of the rotated pair is p.n_offs + iw/2 = i0/2 + p.n_offs/2 + const float x0 = float(rope_data_a[ix + p.n_offs/2 + 0]); + const float x1 = float(rope_data_a[ix + p.n_offs/2 + p.n_dims/2]); - rope_data_d[idst + 0] = ROPE_D_TYPE(x0*cos_theta - x1*sin_theta); - rope_data_d[idst + p.n_dims/2] = ROPE_D_TYPE(x0*sin_theta + x1*cos_theta); + rope_data_d[idst + p.n_offs/2 + 0] = ROPE_D_TYPE(x0*cos_theta - x1*sin_theta); + rope_data_d[idst + p.n_offs/2 + p.n_dims/2] = ROPE_D_TYPE(x0*sin_theta + x1*cos_theta); } void rope_vision(const uint i0, const uint i1, const uint i2, const uint i3, rope_params p) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl index 3602485b94..b88a73fccf 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl @@ -5,6 +5,7 @@ struct rope_params { uint rope_mode; uint nrows; uint n_dims; + uint n_offs; float freq_scale; float freq_base; float ext_factor;