diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 738e7cd92..8f37f65b8 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -1071,6 +1071,9 @@ struct vk_device_struct { vk_pipeline pipeline_trunc[2]; vk_pipeline pipeline_sgn[2]; + // fused UNARY+MUL pipelines: [op][f16][norepeat][op_on_b] + vk_pipeline pipeline_unary_mul[4][2][2][2]; + vk_pipeline pipeline_add1_f16_f16; vk_pipeline pipeline_add1_f16_f32; vk_pipeline pipeline_add1_f32_f32; @@ -5930,6 +5933,26 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { CREATE_UNARY(expm1) #undef CREATE_UNARY +// spec constants: {norepeat, op_on_b} +#define CREATE_UNARY_MUL(name, idx) \ + for (int dt = 0; dt < 2; ++dt) { \ + const size_t len_ = dt ? name ## _mul_f16_len : name ## _mul_f32_len; \ + const unsigned char * data_ = dt ? name ## _mul_f16_data : name ## _mul_f32_data; \ + const std::string dts_ = dt ? "f16" : "f32"; \ + for (int ob = 0; ob < 2; ++ob) \ + for (int nr = 0; nr < 2; ++nr) \ + ggml_vk_create_pipeline(device, device->pipeline_unary_mul[(idx)][dt][nr][ob], \ + (#name "_mul" + std::string(ob ? "_b" : "") + "_" + dts_ + (nr ? "_norepeat" : "")).c_str(), \ + len_, data_, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, \ + { (uint32_t) nr, (uint32_t) ob }, 1); \ + } + + CREATE_UNARY_MUL(gelu, 0) + CREATE_UNARY_MUL(sigmoid, 1) + CREATE_UNARY_MUL(silu, 2) + CREATE_UNARY_MUL(softplus, 3) +#undef CREATE_UNARY_MUL + ggml_vk_create_pipeline(device, device->pipeline_add1_f16_f16, "add1_f16_f16", add1_f16_f16_len, add1_f16_f16_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_add1_f16_f32, "add1_f16_f32", add1_f16_f32_len, add1_f16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_add1_f32_f32, "add1_f32_f32", add1_f32_f32_len, add1_f32_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); @@ -12416,7 +12439,7 @@ template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk } template -static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst, ggml_op op, PC&& pc) { +static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst, ggml_op op, PC&& pc, vk_pipeline pipeline_override = nullptr) { VK_LOG_DEBUG("ggml_vk_op_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; if (src1 != nullptr) { std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; @@ -12447,7 +12470,12 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co init_pushconst_fastdiv(pc); - vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, src2, dst, op); + vk_pipeline pipeline; + if (pipeline_override) { + pipeline = pipeline_override; + } else { + pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, src2, dst, op); + } if (pipeline == nullptr) { std::cerr << "ggml_vulkan: Error: Missing op: " << ggml_op_name(op) << " for " << ggml_type_name(src0->type); @@ -13032,6 +13060,52 @@ static void ggml_vk_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const }); } +// index into device->pipeline_unary_mul for the supported unary ops, or -1 +static int ggml_vk_unary_mul_op_index(ggml_unary_op op) { + switch (op) { + case GGML_UNARY_OP_GELU: return 0; + case GGML_UNARY_OP_SIGMOID: return 1; + case GGML_UNARY_OP_SILU: return 2; + case GGML_UNARY_OP_SOFTPLUS: return 3; + default: return -1; + } +} + +static void ggml_vk_unary_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { + const ggml_tensor * unary = cgraph->nodes[node_idx]; + ggml_tensor * mul = cgraph->nodes[node_idx + 1]; + + // unary on src1 that tiles into src0 + const bool op_on_b = mul->src[1] == unary && + !ggml_are_same_shape(unary->src[0], mul->src[0]) && + ggml_can_repeat(unary, mul->src[0]); + + const ggml_tensor * src0 = op_on_b ? mul->src[0] : unary->src[0]; + const ggml_tensor * src1 = op_on_b ? unary->src[0] : + ((mul->src[0] == unary) ? mul->src[1] : mul->src[0]); + + const bool f16 = src0->type == GGML_TYPE_F16; + const bool norepeat = ggml_are_same_shape(src0, src1); + const int oi = ggml_vk_unary_mul_op_index(ggml_get_unary_op(unary)); + if (oi < 0) { + GGML_ABORT("fatal error"); + } + vk_pipeline pipeline = ctx->device->pipeline_unary_mul[oi][f16][norepeat][op_on_b]; + + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t src1_type_size = ggml_type_size(src1->type); + const uint32_t dst_type_size = ggml_type_size(mul->type); + + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, mul, GGML_OP_UNARY, { + (uint32_t)ggml_nelements(op_on_b ? mul : src0), + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, + (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, + (uint32_t) mul->ne[0], (uint32_t) mul->ne[1], (uint32_t) mul->ne[2],(uint32_t) mul->ne[3], (uint32_t) mul->nb[0] / dst_type_size, (uint32_t) mul->nb[1] / dst_type_size, (uint32_t) mul->nb[2] / dst_type_size, (uint32_t) mul->nb[3] / dst_type_size, + 0, + 0.0f, 0.0f, 0, + }, pipeline); +} + static void ggml_vk_div(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { const uint32_t src0_type_size = ggml_type_size(src0->type); const uint32_t src1_type_size = ggml_type_size(src1->type); @@ -16314,6 +16388,10 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr ggml_vk_topk_moe(ctx, compute_ctx, cgraph, node_idx); break; } + if (ctx->num_additional_fused_ops) { + ggml_vk_unary_mul(ctx, compute_ctx, cgraph, node_idx); + break; + } switch (ggml_get_unary_op(node)) { case GGML_UNARY_OP_ELU: @@ -17275,7 +17353,48 @@ static bool ggml_vk_is_empty(ggml_tensor * node) { return ggml_is_empty(node) || node->op == GGML_OP_NONE || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE; } +static bool ggml_vk_can_fuse_unary_mul(const struct ggml_cgraph * cgraph, int unary_idx, int mul_idx) { + const ggml_tensor * unary = cgraph->nodes[unary_idx]; + const ggml_tensor * mul = cgraph->nodes[mul_idx]; + + if (ggml_vk_unary_mul_op_index(ggml_get_unary_op(unary)) < 0) { + return false; + } + if (unary->type != GGML_TYPE_F32 && unary->type != GGML_TYPE_F16) { + return false; + } + if (unary->type != mul->type) { + return false; + } + if (mul->src[0] != unary && mul->src[1] != unary) { + return false; + } + const ggml_tensor * other = (mul->src[0] == unary) ? mul->src[1] : mul->src[0]; + if (other == nullptr || other->type != unary->type) { + return false; + } + if (!ggml_is_contiguous_1(other) || !ggml_is_contiguous_1(unary->src[0])) { + return false; + } + // fastmod needs src to tile into dst + if (mul->src[0] == unary) { + return ggml_can_repeat(other, unary); + } + return ggml_can_repeat(unary, mul->src[0]); +} + +static bool ggml_vk_can_fuse_unary_mul_pair(const struct ggml_cgraph * cgraph, int node_idx) { + const enum ggml_op ops[] = { GGML_OP_UNARY, GGML_OP_MUL }; + const int outputs[] = { node_idx + 1 }; + return ggml_can_fuse_subgraph(cgraph, node_idx, 2, ops, outputs, 1) && + ggml_vk_can_fuse_unary_mul(cgraph, node_idx, node_idx + 1); +} + static bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list ops) { + if (ops.size() == 2 && ops.begin()[0] == GGML_OP_UNARY && ops.begin()[1] == GGML_OP_MUL) { + return ggml_vk_can_fuse_unary_mul_pair(cgraph, node_idx); + } + if (!ggml_can_fuse(cgraph, node_idx, ops)) { return false; } @@ -17341,6 +17460,7 @@ static bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct g } } } + auto const &mm_add_ok = [&](const ggml_tensor *mul, const ggml_tensor *add) { const ggml_tensor *bias = add->src[0] == mul ? add->src[1] : add->src[0]; @@ -18209,6 +18329,16 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg // they are overwritten, and one workgroup per row. So close enough. op_srcs_fused_elementwise[0] = true; op_srcs_fused_elementwise[1] = true; + } else if (ggml_vk_can_fuse(ctx, cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL })) { + ctx->num_additional_fused_ops = 1; + switch (ggml_get_unary_op(cgraph->nodes[i])) { + case GGML_UNARY_OP_GELU: fusion_string = "GELU_MUL"; break; + case GGML_UNARY_OP_SIGMOID: fusion_string = "SIGMOID_MUL"; break; + case GGML_UNARY_OP_SILU: fusion_string = "SILU_MUL"; break; + default: fusion_string = "SOFTPLUS_MUL"; break; + } + op_srcs_fused_elementwise[0] = true; + op_srcs_fused_elementwise[1] = true; } else if (ggml_vk_can_fuse_ssm_conv(ctx, cgraph, i, 2)) { ctx->num_additional_fused_ops = 2; fusion_string = "SSM_CONV_BIAS_SILU"; @@ -18507,6 +18637,16 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * std::set used_node_set; int first_unused = 0; + + // scheduled or zero-compute nodes in [lo, hi) + auto const &empty_or_scheduled_between = [&](int lo, int hi) -> bool { + for (int v = lo; v < hi; ++v) { + if (!used[v] && !is_empty(graph->nodes[v])) { + return false; + } + } + return true; + }; while (first_unused < graph->n_nodes) { std::vector current_set; @@ -18622,7 +18762,8 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * for (int c = first_unused; c < j; ++c) { if (!used[c] && is_src_of(graph->nodes[j], graph->nodes[c]) && - !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_RMS_NORM && graph->nodes[j]->op == GGML_OP_MUL) && + !(c == current_set.back() && graph->nodes[c]->op == GGML_OP_RMS_NORM && graph->nodes[j]->op == GGML_OP_MUL && empty_or_scheduled_between(c+1, j)) && + !(c == current_set.back() && graph->nodes[c]->op == GGML_OP_UNARY && graph->nodes[j]->op == GGML_OP_MUL && empty_or_scheduled_between(c+1, j)) && !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL_MAT && graph->nodes[j]->op == GGML_OP_ADD) && !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL_MAT_ID && graph->nodes[j]->op == GGML_OP_ADD_ID) && !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL_MAT_ID && graph->nodes[j]->op == GGML_OP_MUL) && @@ -18735,6 +18876,27 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * } } } + // UNARY + MUL: pull the consuming MUL forward + if (j > 0 && + graph->nodes[j]->op == GGML_OP_UNARY) { + for (int k = j + 1; k < std::min(j + 15, graph->n_nodes); ++k) { + ggml_tensor * mul = graph->nodes[k]; + if (mul->op != GGML_OP_MUL || (mul->src[0] != graph->nodes[j] && mul->src[1] != graph->nodes[j])) { + continue; + } + ggml_tensor * other = (mul->src[0] == graph->nodes[j]) ? mul->src[1] : mul->src[0]; + // the other src must either be weights or already processed + if (!(other->op == GGML_OP_NONE || used_node_set.find(other) != used_node_set.end())) { + continue; + } + if (!ggml_vk_can_fuse_unary_mul(graph, j, k)) { + continue; + } + current_set.push_back(k); + used[k] = true; + break; + } + } } } // Second pass grabs view nodes. diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp b/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp index 5ee5275d2..9ee7769ba 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp @@ -1,9 +1,23 @@ #version 450 #include "types.glsl" +#if defined(UNARY_MUL_FUSION) +#include "generic_binary_head.glsl" +#else #include "generic_unary_head.glsl" +#endif +#if defined(UNARY_MUL_FUSION) +// OP on src1 +layout(constant_id = 1) const bool op_on_b = false; +#endif + +#if defined(UNARY_MUL_FUSION) +layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in; +const uint num_threads = 256; +#else layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in; +#endif float op_abs(float x) { return abs(x); @@ -123,6 +137,7 @@ float op_gelu_erf(float a) { return 0.5f * a * (1.0f + sign_x * y); } +#if !defined(UNARY_MUL_FUSION) float op_xielu(float x) { const float alpha_n = p.param1; const float alpha_p = p.param2; @@ -136,6 +151,7 @@ float op_xielu(float x) { const float min_x_eps = min(x, eps); return (op_expm1(min_x_eps) - x) * alpha_n + beta * x; } +#endif float op_floor(float x) { return floor(x); @@ -155,8 +171,28 @@ float op_trunc(float x) { } void main() { - const uint idx = get_idx(); + uint idx = get_idx(); +#if defined(UNARY_MUL_FUSION) + // keep total threads at 512 + [[unroll]] for (uint iter = 0; iter < 2; ++iter) { + if (idx >= p.ne) { + continue; + } + uint i00, i01, i02, i03; + get_indices(idx, i00, i01, i02, i03); + + if (op_on_b) { + data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = + D_TYPE(FLOAT_TYPE(OP(float(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]))) * FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)])); + } else { + data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = + D_TYPE(FLOAT_TYPE(OP(float(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]))) * FLOAT_TYPE(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)])); + } + + idx += num_threads; + } +#else if (idx >= p.ne) { return; } @@ -165,4 +201,5 @@ void main() { const uint d_idx = get_doffset() + dst_idx(idx); data_d[d_idx] = D_TYPE(OP(float(data_a[a_idx]))); +#endif } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 2daafdf43..cb1128dcc 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -966,6 +966,15 @@ void process_shaders() { string_to_spv("softplus_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_softplus"}}); string_to_spv("softplus_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_softplus"}}); + string_to_spv("gelu_mul_f32", "unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_gelu"}, {"UNARY_MUL_FUSION", "1"}}); + string_to_spv("gelu_mul_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_gelu"}, {"UNARY_MUL_FUSION", "1"}}); + string_to_spv("sigmoid_mul_f32", "unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_sigmoid"}, {"UNARY_MUL_FUSION", "1"}}); + string_to_spv("sigmoid_mul_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_sigmoid"}, {"UNARY_MUL_FUSION", "1"}}); + string_to_spv("silu_mul_f32", "unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_silu"}, {"UNARY_MUL_FUSION", "1"}}); + string_to_spv("silu_mul_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_silu"}, {"UNARY_MUL_FUSION", "1"}}); + string_to_spv("softplus_mul_f32","unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_softplus"}, {"UNARY_MUL_FUSION", "1"}}); + string_to_spv("softplus_mul_f16","unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_softplus"}, {"UNARY_MUL_FUSION", "1"}}); + string_to_spv("add1_f16_f16", "add1.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}}); string_to_spv("add1_f16_f32", "add1.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}}); string_to_spv("add1_f32_f32", "add1.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}}); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 5121578ff..01e72041b 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -3908,8 +3908,7 @@ struct test_relu_sqr : public test_case { } }; -// GGML_OP_UNARY(SILU|SIGMOID|SOFTPLUS) + GGML_OP_MUL (fused operation). -// `layout` and `tail` are used for fallback cases where fusion must be skipped +// GGML_OP_UNARY(GELU|SILU|SIGMOID|SOFTPLUS) + GGML_OP_MUL (fused operation). struct test_unary_mul : public test_case { const ggml_unary_op op; const ggml_type type; @@ -3930,7 +3929,8 @@ struct test_unary_mul : public test_case { // performs; relax the tolerance to match that drift switch (type) { case GGML_TYPE_F16: return 5e-5; - default: return 1e-7; + // gelu shader uses exp form, CPU uses tanhf + default: return op == GGML_UNARY_OP_GELU ? 5e-7 : 1e-7; } } @@ -3989,17 +3989,45 @@ struct test_unary_mul : public test_case { } else if (layout == "bcast") { a = ggml_new_tensor(ctx, type, 4, ne.data()); b = ggml_new_tensor_4d(ctx, type, ne[0], 1, 1, 1); + } else if (layout == "rep_ne0") { + // repeat on dim 0 + a = ggml_new_tensor(ctx, type, 4, ne.data()); + std::array ne_b = ne; + ne_b[0] /= 4; + b = ggml_new_tensor(ctx, type, 4, ne_b.data()); + } else if (layout == "view_mid") { + // VIEW between UNARY and MUL + a = ggml_new_tensor(ctx, type, 4, ne.data()); + b = nullptr; + } else if (layout == "gate") { + // small gate on src1 + const std::array ne_gate = { 1, ne[1], ne[2], ne[3] }; + a = ggml_new_tensor(ctx, type, 4, ne_gate.data()); + b = ggml_new_tensor(ctx, type, 4, ne.data()); } else { GGML_ABORT("unknown layout %s", layout.c_str()); } - ggml_set_name(a, "a"); - ggml_set_name(b, "b"); + if (a != nullptr) { + ggml_set_name(a, "a"); + } + if (b != nullptr) { + ggml_set_name(b, "b"); + } ggml_tensor * u = ggml_unary(ctx, a, op); ggml_set_name(u, "unary"); // a broadcasting operand can only be the second one - const bool second = swap && layout != "bcast"; + const bool second = layout == "gate" || (swap && layout != "bcast" && layout != "view_mid"); + if (layout == "view_mid") { + std::array ne_base = ne; + ne_base[0] *= 2; + ggml_tensor * base = ggml_new_tensor(ctx, type, 4, ne_base.data()); + ggml_set_name(base, "base"); + b = ggml_view_4d(ctx, base, ne[0], ne[1], ne[2], ne[3], + base->nb[1], base->nb[2], base->nb[3], 0); + ggml_set_name(b, "b"); + } ggml_tensor * out = second ? ggml_mul(ctx, b, u) : ggml_mul(ctx, u, b); if (tail == "reuse") { @@ -8815,7 +8843,7 @@ static std::vector> make_test_cases_eval() { } // fused unary + mul (gated activations that are not expressed as GGML_OP_GLU) - for (ggml_unary_op op : { GGML_UNARY_OP_SILU, GGML_UNARY_OP_SIGMOID, GGML_UNARY_OP_SOFTPLUS }) { + for (ggml_unary_op op : { GGML_UNARY_OP_GELU, GGML_UNARY_OP_SILU, GGML_UNARY_OP_SIGMOID, GGML_UNARY_OP_SOFTPLUS }) { for (ggml_type type : { GGML_TYPE_F16, GGML_TYPE_F32 }) { for (bool swap : { false, true }) { test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, swap)); @@ -8826,9 +8854,12 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, true, "pad_other")); test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, true, "halves")); test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "packed", "consumer")); + test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "bcast")); + test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "rep_ne0")); + test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "view_mid")); + test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "gate")); // must not fuse test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "strided_dim1")); - test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "bcast")); test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "packed", "reuse")); } }