diff --git a/Makefile b/Makefile index 20ed9b221..1139cfe55 100644 --- a/Makefile +++ b/Makefile @@ -327,7 +327,7 @@ ifdef LLAMA_METAL CFLAGS += -DGGML_USE_METAL -DGGML_METAL_NDEBUG CXXFLAGS += -DGGML_USE_METAL LDFLAGS += -framework Foundation -framework Metal -framework MetalKit -framework MetalPerformanceShaders -OBJS += ggml-metal.o ggml-metal-device.o ggml-metal-device-m.o ggml-metal-context-m.o ggml-metal-common.o ggml-metal-ops.o ggml-metal-tuning.o +OBJS += ggml-metal.o ggml-metal-device.o ggml-metal-device-m.o ggml-metal-context-m.o ggml-metal-common.o ggml-metal-ops.o ggml-metal-tuning.o ggml-metal-fusion.o ggml-metal-common.o: ggml/src/ggml-metal/ggml-metal-common.cpp ggml/src/ggml-metal/ggml-metal-common.h $(CXX) $(CXXFLAGS) -c $< -o $@ @@ -338,6 +338,9 @@ ggml-metal-ops.o: ggml/src/ggml-metal/ggml-metal-ops.cpp ggml/src/ggml-metal/ggm ggml-metal-tuning.o: ggml/src/ggml-metal/ggml-metal-tuning.cpp ggml/src/ggml-metal/ggml-metal-tuning.h $(CXX) $(CXXFLAGS) -c $< -o $@ +ggml-metal-fusion.o: ggml/src/ggml-metal/ggml-metal-fusion.cpp ggml/src/ggml-metal/ggml-metal-fusion.h + $(CXX) $(CXXFLAGS) -c $< -o $@ + ggml-metal.o: ggml/src/ggml-metal/ggml-metal.cpp $(CXX) $(CXXFLAGS) -c $< -o $@ diff --git a/common/jinja/runtime.cpp b/common/jinja/runtime.cpp index bf3fd51b3..a111b733f 100644 --- a/common/jinja/runtime.cpp +++ b/common/jinja/runtime.cpp @@ -845,6 +845,12 @@ value member_expression::execute_impl(context & ctx) { } else { property = this->property->execute(ctx); } + } else if (is_stmt(this->property)) { + // syntax: obj.index + property = mk_val(cast_stmt(this->property)->val); + if (property->as_int() < 0) { + throw std::runtime_error("Static member property cannot be negative"); + } } else { // syntax: obj.prop if (!is_stmt(this->property)) { diff --git a/common/speculative.cpp b/common/speculative.cpp index 942fcaf1b..e60e24243 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -296,7 +296,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { drafting[seq_id] = true; common_sampler_reset(smpls[seq_id].get()); - common_batch_add(batch, dp.id_last, dp.n_past, { seq_id }, true); + common_batch_add(batch, dp.id_last, dp.pos0, { seq_id }, true); } int ret = llama_decode(ctx_dft, batch); @@ -355,7 +355,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { continue; } - common_batch_add(batch, id, dp.n_past + i + 1, { seq_id }, true); + common_batch_add(batch, id, dp.pos0 + i + 1, { seq_id }, true); } if (batch.n_tokens == 0) { @@ -1197,7 +1197,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { common_sampler_reset(smpls[seq_id].get()); - const int32_t n = (int32_t) dp.n_past; + const int32_t n = (int32_t) dp.pos0; const int32_t n_draft = params.n_max; @@ -1493,7 +1493,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { const int32_t n_tokens = batch_in.n_tokens; - // remember the frist and last batch index for each sequence + // remember the first and last batch index for each sequence std::fill(i_batch_beg.begin(), i_batch_beg.end(), -1); std::fill(i_batch_end.begin(), i_batch_end.end(), -1); @@ -1621,7 +1621,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { drafting[seq_id] = true; common_sampler_reset(smpls[seq_id].get()); - common_batch_add(batch, dp.id_last, dp.n_past, { seq_id }, true); + common_batch_add(batch, dp.id_last, dp.pos0, { seq_id }, true); std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, pending_h[seq_id].data(), row_bytes); i_last[seq_id] = batch.n_tokens - 1; @@ -1635,16 +1635,16 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { while (n_drafting > 0) { // each step decodes under a different head, i.e. a different decoder layer, and - // KV is per layer. process() filled this layer's KV only for positions < n_past + // KV is per layer. process() filled this layer's KV only for positions < pos0 // (prompt + accepted prefix) — nothing in the draft region yet. so reset the - // draft region (the seq_rm lower bound is n_past, leaving the prompt KV intact) + // draft region (the seq_rm lower bound is pos0, leaving the prompt KV intact) // and select head i so it rebuilds its own layer's KV there; decoding just the // latest token would leave its attention reading cells only another head wrote. if (chain_heads) { auto * mem_dft = llama_get_memory(ctx_dft); for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { if (drafting[seq_id]) { - llama_memory_seq_rm(mem_dft, seq_id, dparams[seq_id].n_past, -1); + llama_memory_seq_rm(mem_dft, seq_id, dparams[seq_id].pos0, -1); } } llama_set_nextn_layer_offset(ctx_dft, i); @@ -1710,17 +1710,17 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { const int n_rows = (int) result.size() + 1; // id_last + tokens drafted so far for (int t = 0; t < n_rows; ++t) { const llama_token tok = (t == 0) ? dp.id_last : result[t - 1]; - common_batch_add(batch, tok, dp.n_past + t, { seq_id }, t == n_rows - 1); + common_batch_add(batch, tok, dp.pos0 + t, { seq_id }, t == n_rows - 1); std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, chain_h[seq_id].data() + (size_t) t * n_embd, row_bytes); } } else if (is_mem_shared) { // note: with shared memory (e.g. Gemma4 assistants) we use the same position for all draft tokens // ref: https://github.com/huggingface/transformers/blob/effde20942e3f82a1b97449f60b3a48c5ff96145/docs/source/en/model_doc/gemma4_assistant.md?plain=1#L36-L37 - common_batch_add(batch, id, dp.n_past, { seq_id }, true); + common_batch_add(batch, id, dp.pos0, { seq_id }, true); std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, h_row, row_bytes); } else { - common_batch_add(batch, id, dp.n_past + i + 1, { seq_id }, true); + common_batch_add(batch, id, dp.pos0 + i + 1, { seq_id }, true); std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, h_row, row_bytes); } diff --git a/common/speculative.h b/common/speculative.h index 22505891f..c968750e2 100644 --- a/common/speculative.h +++ b/common/speculative.h @@ -61,7 +61,7 @@ struct common_speculative_draft_params { // can be used to constraint the max draft based on the remaining context size int32_t n_max = -1; - llama_pos n_past; + llama_pos pos0; llama_token id_last; // TODO: remove in the future by keeping track of the prompt from the _begin() call and the consecutive accept calls diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index f538f1b0c..a57d22906 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -1712,6 +1712,10 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s ggml_tensor * ids_tensor = node->src[2]; ggml_backend_t ids_backend = split_backend; + if (ggml_nelements(ids_tensor) == 0) { + continue; + } + // if the ids tensor is also an input of the split, it may not have been copied yet to the split backend // in that case, we use the original ids tensor for (int i = input_id + 1; i < split->n_inputs; i++) { diff --git a/ggml/src/ggml-cpu/ops.h b/ggml/src/ggml-cpu/ops.h index 4c1642a67..ce2b3e870 100644 --- a/ggml/src/ggml-cpu/ops.h +++ b/ggml/src/ggml-cpu/ops.h @@ -18,7 +18,15 @@ #endif #endif +// -Winterference-size was introduced in GCC 12 +#if defined(__cplusplus) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winterference-size" +#endif static const size_t CACHE_LINE_SIZE_F32 = CACHE_LINE_SIZE/sizeof(float); +#if defined(__cplusplus) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12 +#pragma GCC diagnostic pop +#endif // Work buffer size for im2col operations in CONV2D #define GGML_IM2COL_WORK_SIZE (16 * 1024 * 1024) diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh index 6f588374b..50f2babad 100644 --- a/ggml/src/ggml-cuda/fattn-common.cuh +++ b/ggml/src/ggml-cuda/fattn-common.cuh @@ -1129,12 +1129,21 @@ void launch_fattn( dim3 blocks_num; if (stream_k) { - // For short contexts it can be faster to have the SMs work on whole tiles because this lets us skip the fixup. - const int max_blocks = max_blocks_per_sm*nsm; - const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks; - const int tiles_efficiency_percent = 100 * ntiles_dst / (max_blocks*tiles_nwaves); + auto should_use_stream_k = [](const int cc, const int ntiles_dst, const int max_blocks, const int DKQ) { + const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks; + const int tiles_efficiency_percent = 100 * ntiles_dst / (max_blocks*tiles_nwaves); - const bool use_stream_k = cc >= GGML_CUDA_CC_ADA_LOVELACE || amd_wmma_available(cc) || tiles_efficiency_percent < 75; + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_ADA_LOVELACE) { + return true; + } + if (amd_wmma_available(cc) && DKQ == 64) { + return true; // TODO better configuration + } + return tiles_efficiency_percent < 75; + }; + + const int max_blocks = max_blocks_per_sm*nsm; + const bool use_stream_k = should_use_stream_k(cc, ntiles_dst, max_blocks, Q->ne[0]); blocks_num.x = ntiles_dst; blocks_num.y = 1; diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuh index bc5060e81..578f6cf79 100644 --- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh @@ -158,8 +158,8 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 8, 64, 2, 32, 128, 128, 128, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 64, 2, 32, 128, 128, 128, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2, 64, 128, 128, 64, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2, 64, 128, 128, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 256, 2, 64, 128, 128, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 256, 2, 64, 128, 128, 64, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 32, 128, 2, 32, 160, 128, 128, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 64, 128, 2, 32, 160, 128, 128, 1, true); @@ -1826,7 +1826,7 @@ static __global__ void flash_attn_ext_f16( #endif // __CUDA_ARCH__ == GGML_CUDA_CC_TURING #if defined(AMD_WMMA_AVAILABLE) - if (ncols1*ncols2 < 16 || ncols2 == 1 || DKQ > 128) { + if (ncols1*ncols2 < 16 || ncols2 == 1 || DKQ > 256) { NO_DEVICE_CODE; return; } diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index b7147ad50..fd0ecdc3a 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -279,6 +279,24 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2(ggml_backend_cuda_con } } + // On RDNA it is preferable to minimize wasted compute vs. duplicate I/O for the mask. + if (amd_wmma_available(cc)) { + if (use_gqa_opt && gqa_ratio % 8 == 0) { + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); + return; + } + + if (use_gqa_opt && gqa_ratio % 4 == 0) { + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); + return; + } + + if (use_gqa_opt && gqa_ratio % 2 == 0) { + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); + return; + } + } + if (use_gqa_opt && gqa_ratio > 4) { ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); return; @@ -704,8 +722,9 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const } } - // AMD WMMA is always faster than the tile kernel if the full tile width of 16 can be utilized. - if ((amd_wmma_available(cc) && gqa_opt_applies && Q->ne[0] <= 128) && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[1] * gqa_ratio_eff > 8) { + // AMD WMMA is faster than the tile kernel if the wide tiles with high arithmetic intensity can be utilized. + if ((amd_wmma_available(cc) && gqa_opt_applies && Q->ne[0] <= 256) && Q->ne[0] != 40 && Q->ne[0] != 72 && + Q->ne[1] * gqa_ratio_eff > (Q->ne[0] <= 128 ? 8 : 16)) { return BEST_FATTN_KERNEL_MMA_F16; } diff --git a/ggml/src/ggml-cuda/mmq-config-gcn.cuh b/ggml/src/ggml-cuda/mmq-config-gcn.cuh new file mode 100644 index 000000000..24af2ef2b --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-config-gcn.cuh @@ -0,0 +1,281 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_gcn(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 3, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index 221fcd8ee..a22cd36ee 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -219,6 +219,7 @@ struct ggml_cuda_mmq_config { #include "mmq-config-ampere.cuh" #include "mmq-config-blackwell.cuh" +#include "mmq-config-gcn.cuh" #include "mmq-config-cdna.cuh" #include "mmq-config-rdna2.cuh" #include "mmq-config-rdna3.cuh" @@ -229,6 +230,9 @@ struct ggml_cuda_mmq_config { static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type type, const int J, const bool fallback, const int cc) { if (GGML_CUDA_CC_IS_AMD(cc)) { + if (GGML_CUDA_CC_IS_GCN(cc)) { + return ggml_cuda_mmq_get_config_gcn(type, J, fallback); + } if (GGML_CUDA_CC_IS_CDNA(cc)) { return ggml_cuda_mmq_get_config_cdna(type, J, fallback); } @@ -257,7 +261,9 @@ static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type ty static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback) { #ifdef GGML_USE_HIP -#ifdef CDNA +#ifdef GCN + return ggml_cuda_mmq_get_config_gcn(type, J, fallback); +#elif defined(CDNA) return ggml_cuda_mmq_get_config_cdna(type, J, fallback); #elif defined(RDNA4) return ggml_cuda_mmq_get_config_rdna4(type, J, fallback); diff --git a/ggml/src/ggml-metal/ggml-metal-common.cpp b/ggml/src/ggml-metal/ggml-metal-common.cpp index 6f1638a11..05755eb3b 100644 --- a/ggml/src/ggml-metal/ggml-metal-common.cpp +++ b/ggml/src/ggml-metal/ggml-metal-common.cpp @@ -1,4 +1,5 @@ #include "ggml-metal-common.h" +#include "ggml-metal-fusion.h" #include "ggml.h" #include "ggml-impl.h" @@ -390,59 +391,31 @@ static std::vector ggml_metal_graph_optimize_reorder(const std::vectorn_nodes; - enum ggml_op ops[MAX_FUSE]; - std::vector nodes; nodes.reserve(gf->n_nodes); // fuse nodes: // we don't want to make reorders that break fusing, so we first pack all fusable tensors // and perform the reorder over the fused nodes. after the reorder is done, we unfuse + // + // the fusable sequences are declared in the fusion table (ggml-metal-fuse.cpp), so the + // packing here is driven by the same patterns that the op encoders will later use for (int i = 0; i < n; i++) { node_info node = { /*.node =*/ gf->nodes[i], /*.fused =*/ {}, }; - // fuse only ops that start with these operations - // can be expanded when needed - if (node.op() == GGML_OP_ADD || - node.op() == GGML_OP_NORM || - node.op() == GGML_OP_RMS_NORM) { - ops[0] = node.op(); + const int f = ggml_metal_fusion_max(gf, i); - int f = i + 1; - while (f < n && f < i + MAX_FUSE) { - // conservatively allow fusing only these ops - // can be expanded when needed - if (gf->nodes[f]->op != GGML_OP_ADD && - gf->nodes[f]->op != GGML_OP_MUL && - gf->nodes[f]->op != GGML_OP_NORM && - gf->nodes[f]->op != GGML_OP_RMS_NORM) { - break; - } - ops[f - i] = gf->nodes[f]->op; - f++; - } + // add the fused tensors into the node info so we can unfuse them later + for (int k = 1; k < f; k++) { + ++i; - f -= i; - for (; f > 1; f--) { - if (ggml_can_fuse(gf, i, ops, f)) { - break; - } - } - - // add the fused tensors into the node info so we can unfuse them later - for (int k = 1; k < f; k++) { - ++i; - - // the .dst() becomes the last fused tensor - node.add_fused(gf->nodes[i]); - } + // the .dst() becomes the last fused tensor + node.add_fused(gf->nodes[i]); } nodes.push_back(std::move(node)); diff --git a/ggml/src/ggml-metal/ggml-metal-context.h b/ggml/src/ggml-metal/ggml-metal-context.h index abf4b06ed..b538b1ad2 100644 --- a/ggml/src/ggml-metal/ggml-metal-context.h +++ b/ggml/src/ggml-metal/ggml-metal-context.h @@ -33,6 +33,7 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx); void ggml_metal_set_n_cb (ggml_metal_t ctx, int n_cb); void ggml_metal_set_abort_callback (ggml_metal_t ctx, ggml_abort_callback abort_callback, void * user_data); + bool ggml_metal_supports_family (ggml_metal_t ctx, int family); void ggml_metal_capture_next_compute(ggml_metal_t ctx); diff --git a/ggml/src/ggml-metal/ggml-metal-context.m b/ggml/src/ggml-metal/ggml-metal-context.m index 6cdc4006b..bf4fe2dcd 100644 --- a/ggml/src/ggml-metal/ggml-metal-context.m +++ b/ggml/src/ggml-metal/ggml-metal-context.m @@ -6,6 +6,7 @@ #import "ggml-metal-impl.h" #import "ggml-metal-common.h" #import "ggml-metal-ops.h" +#import "ggml-metal-fusion.h" #import @@ -36,15 +37,12 @@ struct ggml_metal { // additional, inference-time compiled pipelines ggml_metal_pipelines_t pipelines_ext; - bool use_fusion; bool use_concurrency; bool use_graph_optimize; int debug_graph; - int debug_fusion; - // how many times a given op was fused - uint64_t fuse_cnt[GGML_OP_COUNT]; + struct ggml_metal_fusion_info * finfo; // capture state int capture_compute; @@ -139,7 +137,6 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) { res->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT); - res->use_fusion = getenv("GGML_METAL_FUSION_DISABLE") == nil; res->use_concurrency = getenv("GGML_METAL_CONCURRENCY_DISABLE") == nil; { @@ -147,20 +144,19 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) { res->debug_graph = val ? atoi(val) : 0; } - { - const char * val = getenv("GGML_METAL_FUSION_DEBUG"); - res->debug_fusion = val ? atoi(val) : 0; - } - res->use_graph_optimize = true; if (getenv("GGML_METAL_GRAPH_OPTIMIZE_DISABLE") != NULL) { res->use_graph_optimize = false; } - memset(res->fuse_cnt, 0, sizeof(res->fuse_cnt)); + res->finfo = ggml_metal_device_get_fusion_info(dev); + if (ggml_metal_fusion_info_stats(res->finfo)) { + ggml_metal_fusion_info_labels_init(res->finfo); + res->n_cb = 0; + } - GGML_LOG_INFO("%s: use fusion = %s\n", __func__, res->use_fusion ? "true" : "false"); + GGML_LOG_INFO("%s: use fusion = %s\n", __func__, ggml_metal_fusion_info_enabled(res->finfo) ? "true" : "false"); GGML_LOG_INFO("%s: use concurrency = %s\n", __func__, res->use_concurrency ? "true" : "false"); GGML_LOG_INFO("%s: use graph optimize = %s\n", __func__, res->use_graph_optimize ? "true" : "false"); @@ -222,15 +218,18 @@ void ggml_metal_free(ggml_metal_t ctx) { ctx->pipelines_ext = nil; } - if (ctx->debug_fusion > 0) { + if (ggml_metal_fusion_info_debug(ctx->finfo) > 0) { GGML_LOG_DEBUG("%s: fusion stats:\n", __func__); - for (int i = 0; i < GGML_OP_COUNT; i++) { - if (ctx->fuse_cnt[i] == 0) { + + const int n_fusions = ggml_metal_fusion_info_n_fusions(ctx->finfo); + for (int i = 0; i < n_fusions; i++) { + const uint64_t count = ggml_metal_fusion_info_count(ctx->finfo, i); + if (count == 0) { continue; } // note: cannot use ggml_log here - GGML_LOG_DEBUG("%s: - %s: %" PRIu64 "\n", __func__, ggml_op_name((enum ggml_op) i), ctx->fuse_cnt[i]); + GGML_LOG_DEBUG("%s: - %s: %" PRIu64 "\n", __func__, ggml_metal_fusion_info_label(ctx->finfo, i), count); } } @@ -481,10 +480,17 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * @autoreleasepool { ctx->gf = gf; - ctx->n_nodes_0 = MIN(n_main, gf->n_nodes); - ctx->n_nodes_1 = gf->n_nodes - ctx->n_nodes_0; + if (ctx->n_cb == 0) { + // single-threaded encoding: the whole graph is encoded by one command buffer + ctx->n_nodes_0 = gf->n_nodes; + ctx->n_nodes_1 = 0; + ctx->n_nodes_per_cb = 0; + } else { + ctx->n_nodes_0 = MIN(n_main, gf->n_nodes); + ctx->n_nodes_1 = gf->n_nodes - ctx->n_nodes_0; - ctx->n_nodes_per_cb = (ctx->n_nodes_1 + ctx->n_cb - 1) / ctx->n_cb; + ctx->n_nodes_per_cb = (ctx->n_nodes_1 + ctx->n_cb - 1) / ctx->n_cb; + } if (ctx->capture_compute >= 0) { ctx->capture_compute--; @@ -682,6 +688,12 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx) { } void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { + // when fusion stats are collected the graph must be encoded by a single thread so the + // counters are race-free; override whatever the caller requested + if (ggml_metal_fusion_info_stats(ctx->finfo)) { + n_cb = 0; + } + if (ctx->n_cb != n_cb) { ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_COMMAND_BUFFERS); @@ -717,13 +729,12 @@ void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { ctx->dev, cmd_buf, ctx->gf, + ctx->finfo, idx_start, idx_end, - ctx->use_fusion, ctx->use_concurrency, ctx->capture_compute, - ctx->debug_graph, - ctx->debug_fusion); + ctx->debug_graph); for (int idx = 0; idx < ggml_metal_op_n_nodes(ctx_op); ++idx) { const int res = ggml_metal_op_encode(ctx_op, idx); diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index 1137c5f6d..bf3d07e78 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -932,12 +932,24 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta nsg = N_SG_IQ2_XXS; nr0 = N_R0_IQ2_XXS; smem = 256*8+128; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_XXS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ2_XS: { nsg = N_SG_IQ2_XS; nr0 = N_R0_IQ2_XS; smem = 512*8+128; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_XS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ3_XXS: { @@ -957,21 +969,45 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta nsg = N_SG_IQ3_S; nr0 = N_R0_IQ3_S; smem = 512*4; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ3_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ2_S: { nsg = N_SG_IQ2_S; nr0 = N_R0_IQ2_S; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ1_S: { nsg = N_SG_IQ1_S; nr0 = N_R0_IQ1_S; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ1_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ1_M: { nsg = N_SG_IQ1_M; nr0 = N_R0_IQ1_M; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ1_M_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ4_NL: { @@ -1177,12 +1213,24 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m nsg = N_SG_IQ2_XXS; nr0 = N_R0_IQ2_XXS; smem = 256*8+128; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_XXS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ2_XS: { nsg = N_SG_IQ2_XS; nr0 = N_R0_IQ2_XS; smem = 512*8+128; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_XS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ3_XXS: { @@ -1202,21 +1250,45 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m nsg = N_SG_IQ3_S; nr0 = N_R0_IQ3_S; smem = 512*4; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ3_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ2_S: { nsg = N_SG_IQ2_S; nr0 = N_R0_IQ2_S; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ1_S: { nsg = N_SG_IQ1_S; nr0 = N_R0_IQ1_S; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ1_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ1_M: { nsg = N_SG_IQ1_M; nr0 = N_R0_IQ1_M; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ1_M_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ4_NL: { diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index 31fc07d44..ced33aadf 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -325,6 +325,11 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te const struct ggml_metal_device_props * ggml_metal_device_get_props(ggml_metal_device_t dev); +struct ggml_metal_fusion_info; + +// the device-owned fusion debugging context (NULL unless fusion debugging is enabled) +struct ggml_metal_fusion_info * ggml_metal_device_get_fusion_info(ggml_metal_device_t dev); + // // device buffers // diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index ddaec9fda..406bb2add 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1,4 +1,5 @@ #import "ggml-metal-device.h" +#import "ggml-metal-fusion.h" #import "ggml-impl.h" #import "ggml-backend-impl.h" @@ -896,6 +897,9 @@ struct ggml_metal_device { struct ggml_metal_device_props props; + // shared fusion debugging context + struct ggml_metal_fusion_info * finfo; + // virtual address for GPU memory allocations atomic_uintptr_t addr_virt; }; @@ -1280,6 +1284,13 @@ ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) { dev->props.max_working_set_size = dev->mtl_device.maxBufferLength; } + { + const char * val = getenv("GGML_METAL_FUSION_DEBUG"); + dev->finfo = ggml_metal_fusion_info_init( + getenv("GGML_METAL_FUSION_DISABLE") == nil, + val ? atoi(val) : 0); + } + snprintf(dev->props.name, sizeof(dev->props.name), "%s%d", "MTL", device); const char * gpu_name = [[dev->mtl_device name] UTF8String]; if (n_devices > 1) { @@ -1354,6 +1365,8 @@ void ggml_metal_device_free(ggml_metal_device_t dev) { assert(dev != NULL); @autoreleasepool { + ggml_metal_fusion_info_free(dev->finfo); + ggml_metal_rsets_free(dev->rsets); ggml_metal_library_free(dev->library); @@ -1941,6 +1954,10 @@ static void ggml_metal_device_disable_tensor(ggml_metal_device_t dev) { dev->props.has_tensor = false; } +struct ggml_metal_fusion_info * ggml_metal_device_get_fusion_info(ggml_metal_device_t dev) { + return dev->finfo; +} + // // device buffers // diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.cpp b/ggml/src/ggml-metal/ggml-metal-fusion.cpp new file mode 100644 index 000000000..ac3ac0414 --- /dev/null +++ b/ggml/src/ggml-metal/ggml-metal-fusion.cpp @@ -0,0 +1,502 @@ +#include "ggml-metal-fusion.h" + +#include "ggml-backend-impl.h" +#include "ggml-metal-device.h" + +#include +#include +#include + +// ---- helpers ------------------------------------------------------------- + +// true if two tensors live in the same Metal buffer +static bool ggml_metal_fusion_same_buffer(const ggml_tensor * a, const ggml_tensor * b) { + if (!a || !b) { + return false; + } + + ggml_backend_buffer_t ba = a->view_src ? a->view_src->buffer : a->buffer; + ggml_backend_buffer_t bb = b->view_src ? b->view_src->buffer : b->buffer; + + ggml_metal_buffer_t ca = (ggml_metal_buffer_t) ba->context; + ggml_metal_buffer_t cb = (ggml_metal_buffer_t) bb->context; + + return ggml_metal_buffer_get_id(ca, a).metal == ggml_metal_buffer_get_id(cb, b).metal; +} + +// ---- pattern checks ------------------------------------------------------ + +// NORM/RMS_NORM + MUL + ADD: the weight/bias of each fused step must match the norm input +// width, be contiguous rows, and the fused outputs must stay F32 +static bool ggml_metal_fusion_check_norm( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(mode); + + GGML_ASSERT(fusion->n_ops >= 2); + + for (int j = 1; j < fusion->n_ops; j++) { + // the fused MUL/ADD must read the previous node as src0 + if (nodes[j]->src[0] != nodes[j - 1]) { + return false; + } + + // the weight/bias must have the same row width as the norm input + if (nodes[j]->src[1]->ne[0] != nodes[0]->ne[0]) { + return false; + } + + if (!ggml_is_contiguous_rows(nodes[j]->src[1])) { + return false; + } + + if (nodes[j]->type != GGML_TYPE_F32) { + return false; + } + } + + return true; +} + +// ADD x N: each ADD reads the previous ADD as src0, and all addends must share layout +// (and, in FULL mode, live in the same Metal buffer) +static bool ggml_metal_fusion_check_add_chain( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + ggml_metal_fusion_mode mode) { + GGML_ASSERT(fusion->n_ops >= 2); + + for (int j = 1; j < fusion->n_ops; j++) { + if (nodes[j]->src[0] != nodes[j - 1]) { + return false; + } + + if (!ggml_are_same_layout(nodes[j]->src[1], nodes[j - 1]->src[1])) { + return false; + } + + if (mode == GGML_METAL_FUSION_FULL) { + if (!ggml_metal_fusion_same_buffer(nodes[j]->src[1], nodes[0]->src[1])) { + return false; + } + } + } + + return true; +} + +// GATED_DELTA_NET + CPY: the trailing cpy scatters the gdn state snapshots into the recurrent +// cache, so the gdn kernel writes them straight to the cache and the cpy is elided. +// mirrors ggml_metal_op_can_fuse_gdn_cache (PR #25788). the gdn output has other consumers (the +// attn scores view), so unlike the other patterns this is not an elision chain: the structural +// checks live entirely in this callback (unsafe = true). +static bool ggml_metal_fusion_check_gdn_cache( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(fusion); + + const ggml_tensor * gdn = nodes[0]; + const ggml_tensor * cpy = nodes[1]; + + // the kernel skips the snapshot tail, so the gdn output must not be a graph output + if (gdn->type != GGML_TYPE_F32 || (gdn->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return false; + } + + if (cpy->op != GGML_OP_CPY || (cpy->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return false; + } + + const int64_t S_v = gdn->src[2]->ne[0]; + const int64_t H = gdn->src[2]->ne[1]; + const int64_t n_tokens = gdn->src[2]->ne[2]; + const int64_t n_seqs = gdn->src[2]->ne[3]; + const int64_t K = ggml_get_op_params_i32(gdn, 0); + const size_t tail_off = ggml_row_size(GGML_TYPE_F32, S_v * H * n_tokens * n_seqs); + + const int64_t D = S_v * S_v * H; + const int64_t n_written = std::min(n_tokens, K); + + const ggml_tensor * src = cpy->src[0]; // gdn snapshot tail view + const ggml_tensor * dst = cpy->src[1]; // cache view + + // src must be this gdn's snapshot tail (contiguous, at the tail offset) + if (src->op != GGML_OP_VIEW || src->view_src != gdn || + src->view_offs != tail_off || !ggml_is_contiguous(src)) { + return false; + } + + const int64_t expected_ne[GGML_MAX_DIMS] = { D, n_seqs, n_written, 1 }; + if (dst->type != GGML_TYPE_F32 || + !std::equal(expected_ne, expected_ne + GGML_MAX_DIMS, dst->ne) || + dst->nb[0] != ggml_type_size(GGML_TYPE_F32) || + dst->nb[1] != ggml_row_size(GGML_TYPE_F32, D)) { + return false; + } + + if (mode == GGML_METAL_FUSION_FULL) { + // the cache must be allocated so the kernel can write straight to its buffer + if (dst->data == nullptr) { + return false; + } + } + + return true; +} + +// MUL + SIN + SQR + MUL + ADD (snake activation) +static bool ggml_metal_fusion_check_snake( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(fusion); + GGML_UNUSED(mode); + + const ggml_tensor * mul0 = nodes[0]; + const ggml_tensor * sin_node = nodes[1]; + const ggml_tensor * sqr = nodes[2]; + const ggml_tensor * mul1 = nodes[3]; + const ggml_tensor * add = nodes[4]; + + // x carries the full activation shape, a is the broadcast operand + const ggml_tensor * x = ggml_are_same_shape(mul0, mul0->src[0]) ? mul0->src[0] : mul0->src[1]; + const ggml_tensor * a = (x == mul0->src[0]) ? mul0->src[1] : mul0->src[0]; + + // mul1 reads sqr and inv_b in either operand order + const ggml_tensor * inv_b = (mul1->src[0] == sqr) ? mul1->src[1] : mul1->src[0]; + + // closure check: the trailing add reads the same x as the leading mul + const ggml_tensor * x_in_add = (add->src[0] == mul1) ? add->src[1] : add->src[0]; + + // x is in the supported whitelist and every chain intermediate shares x's type. + // a and inv_b bind as device const float * in the kernel, so they stay F32. + const bool types_ok = + (x->type == GGML_TYPE_F32 || x->type == GGML_TYPE_F16 || x->type == GGML_TYPE_BF16) && + (a->type == GGML_TYPE_F32) && (inv_b->type == GGML_TYPE_F32) && + (mul0->type == x->type) && (sin_node->type == x->type) && + (sqr->type == x->type) && (mul1->type == x->type) && + (add->type == x->type); + + // a / inv_b collapse to [1, C, 1, 1], x and add stay 2D + const bool shape_ok = ggml_are_same_shape(a, inv_b) && a->ne[0] == 1 && a->ne[1] == x->ne[1]; + const bool dim_ok = + (x->ne[2] == 1) && (x->ne[3] == 1) && + (add->ne[2] == 1) && (add->ne[3] == 1) && + (a->ne[2] == 1) && (a->ne[3] == 1) && + (inv_b->ne[2] == 1) && (inv_b->ne[3] == 1); + + // kernel reads x[idx] and a[c] / inv_b[c] linearly, so every operand is contiguous + const bool contig_ok = + ggml_is_contiguous(x) && ggml_is_contiguous(add) && + ggml_is_contiguous(a) && ggml_is_contiguous(inv_b); + + return types_ok && shape_ok && dim_ok && contig_ok && x_in_add == x; +} + +// ---- patterns ------------------------------------------------------------ + +static const ggml_op ops_norm_mul[] = { GGML_OP_NORM, GGML_OP_MUL }; +static const ggml_op ops_norm_mul_add[] = { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD }; +static const ggml_op ops_rms_norm_mul[] = { GGML_OP_RMS_NORM, GGML_OP_MUL }; +static const ggml_op ops_rms_norm_mul_add[] = { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD }; + +static const ggml_op ops_add_2[] = { GGML_OP_ADD, GGML_OP_ADD }; +static const ggml_op ops_add_3[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const ggml_op ops_add_4[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const ggml_op ops_add_5[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const ggml_op ops_add_6[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const ggml_op ops_add_7[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const ggml_op ops_snake[] = { GGML_OP_MUL, GGML_OP_SIN, GGML_OP_SQR, GGML_OP_MUL, GGML_OP_ADD }; + +static const ggml_op ops_gdn_cache[] = { GGML_OP_GATED_DELTA_NET, GGML_OP_CPY }; + +static const ggml_metal_fusion ggml_metal_fusions[] = { + { GGML_METAL_FUSION_NORM_MUL, ops_norm_mul, 2, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_MUL_ADD, ops_norm_mul_add, 3, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_MUL, ops_rms_norm_mul, 2, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_MUL_ADD, ops_rms_norm_mul_add, 3, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_2, 2, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_3, 3, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_4, 4, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_5, 5, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_6, 6, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_7, 7, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_SNAKE, ops_snake, 5, false, ggml_metal_fusion_check_snake }, + { GGML_METAL_FUSION_GDN_CACHE, ops_gdn_cache, 2, true, ggml_metal_fusion_check_gdn_cache }, +}; + +const ggml_metal_fusion * ggml_metal_fusion_all(int * n) { + *n = (int) sizeof(ggml_metal_fusions) / sizeof(ggml_metal_fusions[0]); + + return ggml_metal_fusions; +} + +// ---- shared fusion info --------------------------------------------------- + +static std::string ggml_metal_fusion_label(const ggml_metal_fusion * fusion) { + GGML_ASSERT(fusion != nullptr); + + std::string label; + for (int j = 0; j < fusion->n_ops; j++) { + if (j > 0) { + label += '+'; + } + label += ggml_op_name(fusion->ops[j]); + } + return label; +} + +struct ggml_metal_fusion_info { + std::vector labels; + std::vector counts; + bool enabled; + bool stats; + bool labels_set; + int debug; +}; + +struct ggml_metal_fusion_info * ggml_metal_fusion_info_init(bool enabled, int debug) { + ggml_metal_fusion_info * finfo = new ggml_metal_fusion_info; + finfo->enabled = enabled; + finfo->stats = debug > 0; + finfo->labels_set = false; + finfo->debug = debug; + + if (finfo->stats) { + ggml_metal_fusion_info_labels_init(finfo); + } + + return finfo; +} + +void ggml_metal_fusion_info_free(struct ggml_metal_fusion_info * finfo) { + delete finfo; +} + +bool ggml_metal_fusion_info_enabled(const struct ggml_metal_fusion_info * finfo) { + return finfo->enabled; +} + +bool ggml_metal_fusion_info_stats(const struct ggml_metal_fusion_info * finfo) { + return finfo->stats; +} + +int ggml_metal_fusion_info_debug(const struct ggml_metal_fusion_info * finfo) { + return finfo->debug; +} + +int ggml_metal_fusion_info_n_fusions(const struct ggml_metal_fusion_info * finfo) { + return (int) finfo->labels.size(); +} + +const char * ggml_metal_fusion_info_label(const struct ggml_metal_fusion_info * finfo, int idx) { + GGML_ASSERT(idx >= 0 && idx < (int) finfo->labels.size()); + return finfo->labels[idx].c_str(); +} + +uint64_t ggml_metal_fusion_info_count(const struct ggml_metal_fusion_info * finfo, int idx) { + GGML_ASSERT(idx >= 0 && idx < (int) finfo->counts.size()); + return finfo->counts[idx]; +} + +void ggml_metal_fusion_info_count_fusion(struct ggml_metal_fusion_info * finfo, const struct ggml_metal_fusion * fusion) { + if (!finfo->stats || fusion == nullptr) { + return; + } + + int n = 0; + const ggml_metal_fusion * all = ggml_metal_fusion_all(&n); + + int idx = -1; + for (int i = 0; i < n; i++) { + if (&all[i] == fusion) { + idx = i; + break; + } + } + + if (idx >= 0 && idx < (int) finfo->counts.size()) { + finfo->counts[idx]++; + } +} + +void ggml_metal_fusion_info_set_enabled(struct ggml_metal_fusion_info * finfo, bool enabled) { + finfo->enabled = enabled; +} + +void ggml_metal_fusion_info_labels_init(struct ggml_metal_fusion_info * finfo) { + if (finfo->labels_set) { + return; + } + + int n = 0; + const ggml_metal_fusion * all = ggml_metal_fusion_all(&n); + + finfo->labels.clear(); + finfo->counts.assign(n, 0); + finfo->labels.reserve(n); + + for (int i = 0; i < n; i++) { + finfo->labels.emplace_back(ggml_metal_fusion_label(&all[i])); + } + + finfo->labels_set = true; +} + +void ggml_metal_fusion_info_stats_init(struct ggml_metal_fusion_info * finfo) { + finfo->stats = true; + ggml_metal_fusion_info_labels_init(finfo); +} + +void ggml_metal_fusion_info_stats_reset(struct ggml_metal_fusion_info * finfo) { + std::fill(finfo->counts.begin(), finfo->counts.end(), 0); +} + +int ggml_metal_fusion_info_stats_get(const struct ggml_metal_fusion_info * finfo, const char ** labels, uint64_t * counts, int n) { + const int n_fusions = (int) finfo->labels.size(); + + if (labels == nullptr) { + return n_fusions; + } + + const int n_fill = std::min(n, n_fusions); + for (int i = 0; i < n_fill; i++) { + labels[i] = finfo->labels[i].c_str(); + if (counts != nullptr) { + counts[i] = finfo->counts[i]; + } + } + + return n_fill; +} + +// ---- queries ------------------------------------------------------------- + +// find the longest pattern matching the node sequence starting at idx +// (idx is a position in node_idxs, which maps to graph node indices) +const ggml_metal_fusion * ggml_metal_fusion_next( + const ggml_cgraph * gf, + const int * node_idxs, + int n_idxs, + int idx, + ggml_metal_fusion_mode mode, + int * n_out) { + int n = 0; + const ggml_metal_fusion * all = ggml_metal_fusion_all(&n); + + const ggml_metal_fusion * res = nullptr; + int best = 1; + + for (int i = 0; i < n; i++) { + const ggml_metal_fusion * fusion = &all[i]; + + // only look for a longer match than the current best + if (fusion->n_ops <= best) { + continue; + } + if (idx + fusion->n_ops > n_idxs) { + continue; + } + + const ggml_tensor * nodes[GGML_METAL_FUSION_MAX]; + + // the op sequence must match exactly + bool ok = true; + for (int j = 0; j < fusion->n_ops; j++) { + nodes[j] = gf->nodes[node_idxs[idx + j]]; + if (nodes[j]->op != fusion->ops[j]) { + ok = false; + break; + } + } + if (!ok) { + continue; + } + + if (!fusion->unsafe) { + // common element-wise chain constraints: each node reads the previous one, + // and all nodes have the same shape + for (int j = 1; j < fusion->n_ops && ok; j++) { + if (nodes[j]->src[0] != nodes[j - 1] && nodes[j]->src[1] != nodes[j - 1]) { + ok = false; + break; + } + if (!ggml_are_same_shape(nodes[j], nodes[j - 1])) { + ok = false; + break; + } + } + if (!ok) { + continue; + } + + // all current fusions are single-output elision chains, so the last node is the only output + // TODO: multi-output fusions: store pattern-relative offsets in the table and translate them here + int outputs_buf[1]; + outputs_buf[0] = node_idxs[idx + fusion->n_ops - 1]; + + // structural subgraph checks (op sequence, elidable uses, view containment) + if (!ggml_can_fuse_subgraph_ext(gf, node_idxs + idx, fusion->n_ops, fusion->ops, outputs_buf, 1)) { + continue; + } + } + + // pattern-specific checks (the sole validator for unsafe patterns) + if (fusion->check && !fusion->check(fusion, nodes, mode)) { + continue; + } + + best = fusion->n_ops; + res = fusion; + } + + *n_out = best; + + return res; +} + +// optimize phase: maximum number of nodes starting at idx (a raw sequential graph index) that +// could be fused, chaining patterns back-to-back. matching runs on the same filtered (view +// transparent) node sequence that the compute phase uses, so the returned count is the raw index +// span from idx to the last matched node (intermediate views are packed along). +int ggml_metal_fusion_max(const ggml_cgraph * gf, int idx) { + // an empty/view node cannot start a pattern - pack it alone + if (ggml_op_is_empty(gf->nodes[idx]->op) || ggml_is_empty(gf->nodes[idx])) { + return 1; + } + + // collect the non-empty node indices starting at idx + int idxs[GGML_METAL_FUSION_MAX]; + int n_idxs = 0; + for (int i = idx; i < gf->n_nodes && n_idxs < GGML_METAL_FUSION_MAX; i++) { + if (!ggml_op_is_empty(gf->nodes[i]->op) && !ggml_is_empty(gf->nodes[i])) { + idxs[n_idxs++] = i; + } + } + if (n_idxs == 0) { + return 1; + } + + int total = 0; + int i_f = 0; + + while (i_f < n_idxs && total < GGML_METAL_FUSION_MAX) { + int len = 1; + const ggml_metal_fusion * fusion = ggml_metal_fusion_next(gf, idxs, n_idxs, i_f, GGML_METAL_FUSION_STRUCTURAL, &len); + if (!fusion || total + len > GGML_METAL_FUSION_MAX) { + break; + } + + total += len; + i_f += len; + } + + if (i_f == 0) { + return 1; + } + + // map the matched non-empty nodes back to the raw index span (views are included) + return std::min(GGML_METAL_FUSION_MAX, idxs[i_f - 1] - idx + 1); +} diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.h b/ggml/src/ggml-metal/ggml-metal-fusion.h new file mode 100644 index 000000000..e8515bdec --- /dev/null +++ b/ggml/src/ggml-metal/ggml-metal-fusion.h @@ -0,0 +1,104 @@ +// single source of truth for the fusions supported by the Metal backend +// +// every fusable subgraph is declared exactly once as a ggml_metal_fusion entry in +// the table in ggml-metal-fusion.cpp. both the graph optimizer (ggml_metal_fusion_max) +// and the op encoders (ggml_metal_fusion_next) consult this same table, so the two +// phases can never disagree about what can be fused. + +#pragma once + +#include "ggml-impl.h" + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// the maximum number of nodes that can be fused in a single kernel +// (also the maximum length of a packed fusion group during graph optimization) +#define GGML_METAL_FUSION_MAX 16 + +typedef enum ggml_metal_fusion_mode { + // structural checks only; used by the graph optimizer, at which point the graph + // tensors are not allocated yet, so buffer placement cannot be verified + GGML_METAL_FUSION_STRUCTURAL = 0, + // full checks, including buffer placement; used by the op encoders + GGML_METAL_FUSION_FULL, +} ggml_metal_fusion_mode; + +// identifier of each fusion pattern so the op encoders know which kernel to use +typedef enum ggml_metal_fusion_id { + GGML_METAL_FUSION_NONE = 0, + GGML_METAL_FUSION_NORM_MUL, // NORM/RMS_NORM + MUL + GGML_METAL_FUSION_NORM_MUL_ADD, // NORM/RMS_NORM + MUL + ADD + GGML_METAL_FUSION_ADD_CHAIN, // ADD x N (N in [2, 7]) + GGML_METAL_FUSION_SNAKE, // MUL + SIN + SQR + MUL + ADD + GGML_METAL_FUSION_GDN_CACHE, // GATED_DELTA_NET + CPY (write snapshots into the recurrent cache) +} ggml_metal_fusion_id; + +struct ggml_metal_fusion { + ggml_metal_fusion_id id; + + const enum ggml_op * ops; // op sequence (fixed length) + int n_ops; // number of ops + + // if unsafe: the generic chain/shape + ggml_can_fuse_subgraph checks are skipped and the + // check callback below is the sole validator (used for patterns that are not elision chains, + // e.g. the gdn + cache-cpy write-through fusion) + bool unsafe; + + // extra backend constraints on top of ggml_can_fuse_subgraph + // nodes[j] is the j-th node of the pattern + bool (*check)(const struct ggml_metal_fusion * fusion, + const struct ggml_tensor * const * nodes, + ggml_metal_fusion_mode mode); +}; + +typedef struct ggml_metal_fusion ggml_metal_fusion; + +// the single table of all fusions supported by the Metal backend +const ggml_metal_fusion * ggml_metal_fusion_all(int * n); + +// ---- shared fusion info --------------------------------------------------- + +// shared fusion debugging context, owned by the device; newly created backend contexts for that +// device register with it so the fusion counters are race-free and accumulate across contexts. +struct ggml_metal_fusion_info; // defined in ggml-metal-fusion.cpp + +struct ggml_metal_fusion_info * ggml_metal_fusion_info_init(bool enabled, int debug); +void ggml_metal_fusion_info_free(struct ggml_metal_fusion_info * finfo); + +bool ggml_metal_fusion_info_enabled(const struct ggml_metal_fusion_info * finfo); +bool ggml_metal_fusion_info_stats (const struct ggml_metal_fusion_info * finfo); +int ggml_metal_fusion_info_debug (const struct ggml_metal_fusion_info * finfo); + +int ggml_metal_fusion_info_n_fusions(const struct ggml_metal_fusion_info * finfo); +const char * ggml_metal_fusion_info_label (const struct ggml_metal_fusion_info * finfo, int idx); +uint64_t ggml_metal_fusion_info_count (const struct ggml_metal_fusion_info * finfo, int idx); + +void ggml_metal_fusion_info_count_fusion(struct ggml_metal_fusion_info * finfo, const struct ggml_metal_fusion * fusion); +void ggml_metal_fusion_info_set_enabled (struct ggml_metal_fusion_info * finfo, bool enabled); + +void ggml_metal_fusion_info_stats_init ( struct ggml_metal_fusion_info * finfo); +void ggml_metal_fusion_info_stats_reset( struct ggml_metal_fusion_info * finfo); +int ggml_metal_fusion_info_stats_get (const struct ggml_metal_fusion_info * finfo, const char ** labels, uint64_t * counts, int n); +void ggml_metal_fusion_info_labels_init( struct ggml_metal_fusion_info * finfo); + +// compute phase: longest fusion starting at idx (a position in node_idxs) that matches in `mode`. +// returns the matching pattern (nullptr if no fusion) and sets *n_out to the number of nodes consumed. +const ggml_metal_fusion * ggml_metal_fusion_next( + const struct ggml_cgraph * gf, + const int * node_idxs, + int n_idxs, + int idx, + ggml_metal_fusion_mode mode, + int * n_out); + +// optimize phase: maximum number of nodes starting at idx (a raw sequential graph index) that +// could be fused, chaining patterns back-to-back. returns at least 1. +int ggml_metal_fusion_max(const struct ggml_cgraph * gf, int idx); + +#ifdef __cplusplus +} +#endif diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index 1fe947633..7ad21341e 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -62,18 +62,23 @@ #define N_R0_IQ1_S 4 #define N_SG_IQ1_S 2 +#define N_R0_IQ1_S_SPLIT 8 #define N_R0_IQ1_M 4 #define N_SG_IQ1_M 2 +#define N_R0_IQ1_M_SPLIT 8 #define N_R0_IQ2_XXS 4 #define N_SG_IQ2_XXS 2 +#define N_R0_IQ2_XXS_SPLIT 8 #define N_R0_IQ2_XS 4 #define N_SG_IQ2_XS 2 +#define N_R0_IQ2_XS_SPLIT 8 #define N_R0_IQ2_S 4 #define N_SG_IQ2_S 2 +#define N_R0_IQ2_S_SPLIT 8 #define N_R0_IQ3_XXS 4 #define N_SG_IQ3_XXS 2 @@ -81,6 +86,7 @@ #define N_R0_IQ3_S 4 #define N_SG_IQ3_S 2 +#define N_R0_IQ3_S_SPLIT 8 #define N_R0_IQ4_NL 2 #define N_SG_IQ4_NL 2 @@ -979,6 +985,7 @@ typedef struct { uint64_t nb1; uint64_t nb2; uint64_t nb3; + uint64_t nb_out; // 0 => snapshots are appended after the attn scores (unfused) } ggml_metal_kargs_gated_delta_net; typedef struct { diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index 3db8bca43..b4e87cb2c 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -7,6 +7,7 @@ #include "ggml-metal-impl.h" #include "ggml-metal-common.h" #include "ggml-metal-device.h" +#include "ggml-metal-fusion.h" #include "ggml-metal-tuning.h" #include @@ -31,24 +32,22 @@ struct ggml_metal_op { ggml_metal_device_t dev, ggml_metal_cmd_buf_t cmd_buf, ggml_cgraph * gf, + ggml_metal_fusion_info * finfo, int idx_start, int idx_end, - bool use_fusion, bool use_concurrency, bool use_capture, - int debug_graph, - int debug_fusion) { + int debug_graph) { this->dev = dev; this->lib = ggml_metal_device_get_library(dev); this->enc = ggml_metal_encoder_init(cmd_buf, use_concurrency); this->mem_ranges = ggml_mem_ranges_init(debug_graph); + this->finfo = finfo; this->idx_start = idx_start; this->idx_end = idx_end; - this->use_fusion = use_fusion; this->use_concurrency = use_concurrency; this->use_capture = use_capture; this->debug_graph = debug_graph; - this->debug_fusion = debug_fusion; this->gf = gf; idxs.reserve(gf->n_nodes); @@ -78,15 +77,24 @@ struct ggml_metal_op { return ggml_graph_node(gf, idxs[i]); } - bool can_fuse(int i0, const ggml_op * ops, int n_ops) const { - assert(use_fusion); + // consult the fusion table for the longest pattern starting at i0 + // returns the matching pattern (nullptr if no fusion) and sets *n_out to the number of nodes + const ggml_metal_fusion * can_fuse(int i0, enum ggml_metal_fusion_mode mode, int * n_out) const { + assert(use_fusion()); assert(i0 >= 0 && i0 < n_nodes()); - if (i0 + n_ops > n_nodes()) { - return false; - } + return ggml_metal_fusion_next(gf, idxs.data(), (int) idxs.size(), i0, mode, n_out); + } - return ggml_can_fuse_ext(gf, idxs.data() + i0, ops, n_ops); + // whether to attempt fusion; the toggle lives in the shared fusion debugging context owned + // by the device (initialized from GGML_METAL_FUSION_DISABLE, overridable by the test) + bool use_fusion() const { + return ggml_metal_fusion_info_enabled(finfo); + } + + // record that a fusion fired, indexed by the matching table entry + void count_fusions(const ggml_metal_fusion * fusion) const { + ggml_metal_fusion_info_count_fusion(finfo, fusion); } ggml_metal_device_t dev; @@ -94,12 +102,13 @@ struct ggml_metal_op { ggml_metal_encoder_t enc; ggml_mem_ranges_t mem_ranges; - bool use_fusion; + // shared fusion debugging context + ggml_metal_fusion_info * finfo; + bool use_concurrency; bool use_capture; int debug_graph; - int debug_fusion; private: ggml_cgraph * gf; @@ -115,24 +124,22 @@ ggml_metal_op_t ggml_metal_op_init( ggml_metal_device_t dev, ggml_metal_cmd_buf_t cmd_buf, ggml_cgraph * gf, + ggml_metal_fusion_info * finfo, int idx_start, int idx_end, - bool use_fusion, bool use_concurrency, bool use_capture, - int debug_graph, - int debug_fusion) { + int debug_graph) { ggml_metal_op_t res = new ggml_metal_op( dev, cmd_buf, gf, + finfo, idx_start, idx_end, - use_fusion, use_concurrency, use_capture, - debug_graph, - debug_fusion); + debug_graph); return res; } @@ -1868,6 +1875,8 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; + const bool use_fusion = ctx->use_fusion(); + const int debug_fusion = ggml_metal_fusion_info_debug(ctx->finfo); GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -1880,6 +1889,31 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) { auto pipeline = ggml_metal_library_get_pipeline_gated_delta_net(lib, op); + // when fused with the trailing cache cpy, the snapshots are written straight into the + // recurrent cache and the cpy is skipped (see GGML_METAL_FUSION_GDN_CACHE) + ggml_metal_buffer_id bid_out = ggml_metal_get_buffer_id(op); + uint64_t nb_out = 0; + int n_fuse = 1; + + if (use_fusion) { + int n = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); + + if (fusion && fusion->id == GGML_METAL_FUSION_GDN_CACHE) { + const ggml_tensor * dst_cache = ctx->node(idx + 1)->src[1]; // cache view + + bid_out = ggml_metal_get_buffer_id(dst_cache); + nb_out = dst_cache->nb[2]/sizeof(float); + n_fuse = 2; + + ctx->count_fusions(fusion); + + if (debug_fusion > 1) { + GGML_LOG_DEBUG("%s: fuse: GATED_DELTA_NET + CPY\n", __func__); + } + } + } + int ida = 0; ggml_metal_kargs_gated_delta_net args = { @@ -1918,23 +1952,25 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) { /*.nb1 =*/ nb1, /*.nb2 =*/ nb2, /*.nb3 =*/ nb3, + /*.nb_out =*/ nb_out, }; ggml_metal_encoder_set_pipeline(enc, pipeline); - ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), ida++); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), ida++); // args ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), ida++); // q ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), ida++); // k ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), ida++); // v ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[3]), ida++); // gate ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[4]), ida++); // beta ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[5]), ida++); // state - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++); // dst + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++); // dst (attn) + ggml_metal_encoder_set_buffer (enc, bid_out, ida++); // state_out const int nsg = pipeline.nsg; ggml_metal_encoder_dispatch_threadgroups(enc, op->src[2]->ne[0]/nsg, op->src[2]->ne[1], op->src[2]->ne[3], 32, nsg, 1); - return 1; + return n_fuse; } int ggml_metal_op_solve_tri(ggml_metal_op_t ctx, int idx) { @@ -3718,56 +3754,20 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { return 1; } -// Snake activation autofuse: mul -> sin -> sqr -> mul -> add -static bool ggml_metal_op_can_fuse_snake(ggml_metal_op_t ctx, int idx) { - static constexpr ggml_op snake_ops[5] = { GGML_OP_MUL, GGML_OP_SIN, GGML_OP_SQR, GGML_OP_MUL, GGML_OP_ADD }; - - if (ctx->node(idx)->op != GGML_OP_MUL || !ctx->can_fuse(idx, snake_ops, 5)) { - return false; - } - - const ggml_tensor * mul0 = ctx->node(idx + 0); - const ggml_tensor * sin_node = ctx->node(idx + 1); - const ggml_tensor * sqr = ctx->node(idx + 2); - const ggml_tensor * mul1 = ctx->node(idx + 3); - const ggml_tensor * add = ctx->node(idx + 4); - - // x carries the full activation shape, a is the broadcast operand - const ggml_tensor * x = ggml_are_same_shape(mul0, mul0->src[0]) ? mul0->src[0] : mul0->src[1]; - const ggml_tensor * a = (x == mul0->src[0]) ? mul0->src[1] : mul0->src[0]; - - // mul1 reads sqr and inv_b in either operand order - const ggml_tensor * inv_b = (mul1->src[0] == sqr) ? mul1->src[1] : mul1->src[0]; - - // closure check: the trailing add reads the same x as the leading mul - const ggml_tensor * x_in_add = (add->src[0] == mul1) ? add->src[1] : add->src[0]; - - // x is in the supported whitelist and every chain intermediate shares x's type. - // a and inv_b bind as device const float * in the kernel, so they stay F32. - const bool types_ok = - (x->type == GGML_TYPE_F32 || x->type == GGML_TYPE_F16 || x->type == GGML_TYPE_BF16) && - (a->type == GGML_TYPE_F32) && (inv_b->type == GGML_TYPE_F32) && - (mul0->type == x->type) && (sin_node->type == x->type) && - (sqr->type == x->type) && (mul1->type == x->type) && - (add->type == x->type); - // a / inv_b collapse to [1, C, 1, 1], x and add stay 2D - const bool shape_ok = ggml_are_same_shape(a, inv_b) && a->ne[0] == 1 && a->ne[1] == x->ne[1]; - const bool dim_ok = - (x->ne[2] == 1) && (x->ne[3] == 1) && - (add->ne[2] == 1) && (add->ne[3] == 1) && - (a->ne[2] == 1) && (a->ne[3] == 1) && - (inv_b->ne[2] == 1) && (inv_b->ne[3] == 1); - // kernel reads x[idx] and a[c] / inv_b[c] linearly, so every operand is contiguous - const bool contig_ok = - ggml_is_contiguous(x) && ggml_is_contiguous(add) && - ggml_is_contiguous(a) && ggml_is_contiguous(inv_b); - - return types_ok && shape_ok && dim_ok && contig_ok && x_in_add == x; -} - int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { - if (ctx->use_fusion && ggml_metal_op_can_fuse_snake(ctx, idx)) { - return ggml_metal_op_snake_fused(ctx, idx); + int n_fuse = 1; + const ggml_metal_fusion * fusion = nullptr; + + if (ctx->use_fusion()) { + int n = 1; + fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); + n_fuse = n; + + // snake activation autofuse: mul -> sin -> sqr -> mul -> add + if (fusion && fusion->id == GGML_METAL_FUSION_SNAKE) { + ctx->count_fusions(fusion); + return ggml_metal_op_snake_fused(ctx, idx); + } } ggml_tensor * op = ctx->node(idx); @@ -3775,9 +3775,9 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; - const bool use_fusion = ctx->use_fusion; + const bool use_fusion = ctx->use_fusion(); - const int debug_fusion = ctx->debug_fusion; + const int debug_fusion = ggml_metal_fusion_info_debug(ctx->finfo); GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -3822,57 +3822,19 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { /*.o1 =*/ { bid_src1.offs }, }; - ggml_op fops[8]; - - int n_fuse = 1; - // c[0] = add(a, b[0]) // c[1] = add(c[0], b[1]) // c[2] = add(c[1], b[2]) // ... - if (use_fusion) { - fops[0] = GGML_OP_ADD; - fops[1] = GGML_OP_ADD; - fops[2] = GGML_OP_ADD; - fops[3] = GGML_OP_ADD; - fops[4] = GGML_OP_ADD; - fops[5] = GGML_OP_ADD; - fops[6] = GGML_OP_ADD; - fops[7] = GGML_OP_ADD; - - // note: in metal, we sometimes encode the graph in parallel so we have to avoid fusing ops - // across splits. idx_end indicates the last node in the current split - for (n_fuse = 0; n_fuse <= 6; ++n_fuse) { - if (!ctx->can_fuse(idx + n_fuse, fops + n_fuse, 2)) { - break; - } - - ggml_tensor * f0 = ctx->node(idx + n_fuse); - ggml_tensor * f1 = ctx->node(idx + n_fuse + 1); - - if (f0 != f1->src[0]) { - break; - } - - // b[0] === b[1] === ... - if (!ggml_are_same_layout(f0->src[1], f1->src[1])) { - break; - } - - // only fuse ops if src1 is in the same Metal buffer - ggml_metal_buffer_id bid_fuse = ggml_metal_get_buffer_id(f1->src[1]); - if (bid_fuse.metal != bid_src1.metal) { - break; - } - - //ctx->fuse_cnt[ops[n_fuse + 1]->op]++; - - args.o1[n_fuse + 1] = bid_fuse.offs; + if (use_fusion && fusion && fusion->id == GGML_METAL_FUSION_ADD_CHAIN) { + // the offsets of the fused addends are relative to the start of the src1 buffer + for (int i = 1; i < n_fuse; i++) { + args.o1[i] = ggml_metal_get_buffer_id(ctx->node(idx + i)->src[1]).offs; } - ++n_fuse; + ctx->count_fusions(fusion); - if (debug_fusion > 1 && n_fuse > 1) { + if (debug_fusion > 1) { GGML_LOG_DEBUG("%s: fuse: ADD x %d\n", __func__, n_fuse); } } @@ -4080,9 +4042,9 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; - const bool use_fusion = ctx->use_fusion; + const bool use_fusion = ctx->use_fusion(); - const int debug_fusion = ctx->debug_fusion; + const int debug_fusion = ggml_metal_fusion_info_debug(ctx->finfo); GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -4110,8 +4072,6 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) { /*.nbf3 =*/ { nb03 }, }; - ggml_op fops[8]; - int n_fuse = 1; ggml_metal_buffer_id bid_fuse[2] = { bid_src0, bid_src0 }; @@ -4120,55 +4080,35 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) { // d[1] = mul(d[0], b) // d[2] = add(d[1], c) if (use_fusion) { - fops[0] = op->op; - fops[1] = GGML_OP_MUL; - fops[2] = GGML_OP_ADD; + int n = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); - for (n_fuse = 0; n_fuse <= 1; ++n_fuse) { - if (!ctx->can_fuse(idx + n_fuse, fops + n_fuse, 2)) { - break; + if (fusion && (fusion->id == GGML_METAL_FUSION_NORM_MUL || fusion->id == GGML_METAL_FUSION_NORM_MUL_ADD)) { + n_fuse = n; + + ctx->count_fusions(fusion); + + for (int i = 1; i < n_fuse; i++) { + const ggml_tensor * fn = ctx->node(idx + i); + + bid_fuse[i - 1] = ggml_metal_get_buffer_id(fn->src[1]); + + args.nef1[i] = fn->src[1]->ne[1]; + args.nef2[i] = fn->src[1]->ne[2]; + args.nef3[i] = fn->src[1]->ne[3]; + + args.nbf1[i] = fn->src[1]->nb[1]; + args.nbf2[i] = fn->src[1]->nb[2]; + args.nbf3[i] = fn->src[1]->nb[3]; } - ggml_tensor * f0 = ctx->node(idx + n_fuse); - ggml_tensor * f1 = ctx->node(idx + n_fuse + 1); - - if (f0 != f1->src[0]) { - break; - } - - if (f1->src[1]->ne[0] != op->ne[0]) { - break; - } - - if (!ggml_is_contiguous_rows(f1->src[1])) { - break; - } - - if (f1->type != GGML_TYPE_F32) { - break; - } - - //ctx->fuse_cnt[f1->op]++; - - bid_fuse[n_fuse] = ggml_metal_get_buffer_id(f1->src[1]); - - args.nef1[n_fuse + 1] = f1->src[1]->ne[1]; - args.nef2[n_fuse + 1] = f1->src[1]->ne[2]; - args.nef3[n_fuse + 1] = f1->src[1]->ne[3]; - - args.nbf1[n_fuse + 1] = f1->src[1]->nb[1]; - args.nbf2[n_fuse + 1] = f1->src[1]->nb[2]; - args.nbf3[n_fuse + 1] = f1->src[1]->nb[3]; - } - - ++n_fuse; - - if (debug_fusion > 1 && n_fuse > 1) { - if (n_fuse == 2) { - GGML_LOG_DEBUG("%s: fuse: %s + MUL\n", __func__, ggml_op_name(op->op)); - } - if (n_fuse == 3) { - GGML_LOG_DEBUG("%s: fuse: %s + MUL + ADD\n", __func__, ggml_op_name(op->op)); + if (debug_fusion > 1) { + if (n_fuse == 2) { + GGML_LOG_DEBUG("%s: fuse: %s + MUL\n", __func__, ggml_op_name(op->op)); + } + if (n_fuse == 3) { + GGML_LOG_DEBUG("%s: fuse: %s + MUL + ADD\n", __func__, ggml_op_name(op->op)); + } } } } diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.h index f8fe50b46..4dd8ce7af 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.h +++ b/ggml/src/ggml-metal/ggml-metal-ops.h @@ -8,17 +8,18 @@ extern "C" { typedef struct ggml_metal_op * ggml_metal_op_t; +struct ggml_metal_fusion; // forward decl (ggml-metal-device.h) + ggml_metal_op_t ggml_metal_op_init( ggml_metal_device_t dev, ggml_metal_cmd_buf_t cmd_buf, struct ggml_cgraph * gf, + struct ggml_metal_fusion_info * finfo, int idx_start, int idx_end, - bool use_fusion, bool use_concurrency, bool use_capture, - int debug_graph, - int debug_fusion); + int debug_graph); void ggml_metal_op_free(ggml_metal_op_t ctx); diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp index 3bd6abd06..4cbec8645 100644 --- a/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp @@ -4,6 +4,7 @@ #include "ggml-backend-impl.h" #include "ggml-metal-device.h" +#include "ggml-metal-fusion.h" #include "ggml-metal-context.h" #include "ggml-metal-ops.h" #include "ggml-metal-tuning.h" @@ -906,6 +907,30 @@ static const char * ggml_backend_metal_tuning_device_token(ggml_backend_dev_t de return ggml_metal_device_id_token(ggml_metal_device_get_props(ctx_dev)->device_id); } +// generic fusion debugging API (ad-hoc proc-address mechanism): the test resolves the device +// fusion context once and passes that opaque handle to the rest of the functions +typedef void * ggml_backend_fusion_t; + +static ggml_backend_fusion_t ggml_backend_metal_fusion_get(ggml_backend_dev_t dev) { + return ggml_metal_device_get_fusion_info((ggml_metal_device_t)dev->context); +} + +static void ggml_backend_metal_fusion_stats_init(ggml_backend_fusion_t finfo) { + ggml_metal_fusion_info_stats_init((struct ggml_metal_fusion_info *) finfo); +} + +static void ggml_backend_metal_fusion_stats_reset(ggml_backend_fusion_t finfo) { + ggml_metal_fusion_info_stats_reset((struct ggml_metal_fusion_info *) finfo); +} + +static int ggml_backend_metal_fusion_stats_get(ggml_backend_fusion_t finfo, const char ** labels, uint64_t * counts, int n) { + return ggml_metal_fusion_info_stats_get((struct ggml_metal_fusion_info *) finfo, labels, counts, n); +} + +static void ggml_backend_metal_fusion_set_enabled(ggml_backend_fusion_t finfo, bool enabled) { + ggml_metal_fusion_info_set_enabled((struct ggml_metal_fusion_info *) finfo, enabled); +} + static void * ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, const char * name) { if (strcmp(name, "ggml_backend_get_features") == 0) { return (void *)ggml_backend_metal_get_features; @@ -928,6 +953,23 @@ static void * ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, const if (strcmp(name, "ggml_backend_metal_tuning_device_token") == 0) { return (void *)ggml_backend_metal_tuning_device_token; } + // generic fusion debugging API (ad-hoc proc-address mechanism, not part of the official + // ggml backend interface yet; a backend that adopts it exports these exact names) + if (strcmp(name, "ggml_backend_fusion_get") == 0) { + return (void *)ggml_backend_metal_fusion_get; + } + if (strcmp(name, "ggml_backend_fusion_stats_init") == 0) { + return (void *)ggml_backend_metal_fusion_stats_init; + } + if (strcmp(name, "ggml_backend_fusion_stats_reset") == 0) { + return (void *)ggml_backend_metal_fusion_stats_reset; + } + if (strcmp(name, "ggml_backend_fusion_stats_get") == 0) { + return (void *)ggml_backend_metal_fusion_stats_get; + } + if (strcmp(name, "ggml_backend_fusion_set_enabled") == 0) { + return (void *)ggml_backend_metal_fusion_set_enabled; + } return NULL; diff --git a/ggml/src/ggml-metal/kernels/gated_delta_net.metal b/ggml/src/ggml-metal/kernels/gated_delta_net.metal index 8422d8e29..5e4861ece 100644 --- a/ggml/src/ggml-metal/kernels/gated_delta_net.metal +++ b/ggml/src/ggml-metal/kernels/gated_delta_net.metal @@ -15,6 +15,7 @@ kernel void kernel_gated_delta_net_impl( device const char * b, device const char * s, device char * dst, + device char * dst_fuse, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tpitg[[thread_position_in_threadgroup]], uint3 ntg[[threads_per_threadgroup]]) { @@ -65,6 +66,12 @@ kernel void kernel_gated_delta_net_impl( // per-(seq,head) offset within a slot const uint state_out_base = (i23*args.ne21 + i21)*S_v*S_v + i20*S_v; + // when fused with the cache cpy, write the snapshots straight into the cache buffer using + // the slot stride; otherwise append them after the attn scores (nb_out == 0) + const bool fused = args.nb_out > 0; + const device float * state_out = fused ? (device float *)dst_fuse : (device float *)dst + attn_size; + const uint slot_stride = fused ? (uint)args.nb_out : state_size_per_snap; + for (short t = 0; t < args.ne22; t++) { float s_k = 0.0f; @@ -116,7 +123,7 @@ kernel void kernel_gated_delta_net_impl( if (K > 1) { const int target_slot = (int)args.ne22 - 1 - (int)t; if (target_slot >= 0 && target_slot < (int)K) { - device float * dst_state = (device float *) (dst) + attn_size + (uint)target_slot * state_size_per_snap + state_out_base; + device float * dst_state = (device float *)state_out + (uint)target_slot * slot_stride + state_out_base; FOR_UNROLL (short j = 0; j < NSG; j++) { const short is = tx*NSG + j; dst_state[is] = ls[j]; @@ -126,7 +133,7 @@ kernel void kernel_gated_delta_net_impl( } if (K == 1) { - device float * dst_state = (device float *) (dst) + attn_size + state_out_base; + device float * dst_state = (device float *)state_out + state_out_base; FOR_UNROLL (short j = 0; j < NSG; j++) { const short is = tx*NSG + j; dst_state[is] = ls[j]; @@ -158,6 +165,7 @@ kernel void kernel_gated_delta_net_impl( device const char * b, device const char * s, device char * dst, + device char * dst_fuse, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tpitg[[thread_position_in_threadgroup]], uint3 ntg[[threads_per_threadgroup]]) { @@ -230,7 +238,13 @@ kernel void kernel_gated_delta_net_impl( dst_attn += args.ne21*S_v; } - device float * dst_state = (device float *) (dst) + args.ne23*args.ne22*args.ne21*S_v + (i23*args.ne21 + i21)*S_v*S_v + i20; + // when fused with the cache cpy, write the snapshots straight into the cache buffer using + // the slot stride; otherwise append them after the attn scores (nb_out == 0) + const bool fused = args.nb_out > 0; + const device float * state_out = fused ? (device float *)dst_fuse : (device float *)dst + args.ne23*args.ne22*args.ne21*S_v; + const uint slot_stride = fused ? (uint)args.nb_out : S_v*S_v; + + device float * dst_state = (device float *)state_out + (i23*args.ne21 + i21)*slot_stride + i20; device T * dstt_state = (device T *) (dst_state); FOR_UNROLL (short j = 0; j < NSG; j++) { diff --git a/ggml/src/ggml-metal/kernels/mul_mm.metal b/ggml/src/ggml-metal/kernels/mul_mm.metal index ee848eed6..0a45bb1bb 100644 --- a/ggml/src/ggml-metal/kernels/mul_mm.metal +++ b/ggml/src/ggml-metal/kernels/mul_mm.metal @@ -496,6 +496,13 @@ kernel void kernel_mul_mm_id( + args.nb11*i11 + args.nb10*iy); + // skip the upper half of the token tile when the expert did not fill it + constexpr short NR1H = NR1/2; + + const bool has_hi = nr1 > NR1H; + + const short lb1 = (short) tiitg/NL1; // 0 .. NR1-1, this thread's row of the B tile + #ifndef GGML_METAL_HAS_TENSOR S0_8x8 ma[4]; S1_8x8 mb[2]; @@ -505,15 +512,22 @@ kernel void kernel_mul_mm_id( for (short i = 0; i < 8; i++){ mc[i] = make_filled_simdgroup_matrix(0.f); } + + // simdgroups 2,3 own rows NR1H..NR1-1 + const bool sg_active = has_hi || sgitg < 2; #else - auto tA = tensor, tensor_inline>(sa, dextents(NK, NR0)); - auto tB = tensor, tensor_inline>(sb, dextents(NR1, NK )); + auto tA = tensor, tensor_inline>(sa, dextents(NK, NR0)); + + // sb is [NR1][NK] row-major + auto tB0 = tensor, tensor_inline>(sb, dextents(NK, NR1H)); + auto tB1 = tensor, tensor_inline>(sb + NR1H*NK, dextents(NK, NR1H)); mpp::tensor_ops::matmul2d< - mpp::tensor_ops::matmul2d_descriptor(NR1, NR0, NK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate), + mpp::tensor_ops::matmul2d_descriptor(NR1H, NR0, NK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate), execution_simdgroups<4>> mm; - auto cT = mm.get_destination_cooperative_tensor(); + auto cT0 = mm.get_destination_cooperative_tensor(); + auto cT1 = mm.get_destination_cooperative_tensor(); #endif for (int loop_k = 0; loop_k < args.ne00; loop_k += NK) { @@ -656,37 +670,45 @@ kernel void kernel_mul_mm_id( threadgroup_barrier(mem_flags::mem_threadgroup); #ifndef GGML_METAL_HAS_TENSOR - // load matrices from threadgroup memory and conduct outer products - threadgroup const S0 * lsma = (sa + 4*64*(sgitg%2)); - threadgroup const S1 * lsmb = (sb + 2*64*(sgitg/2)); + if (sg_active) { + // load matrices from threadgroup memory and conduct outer products + threadgroup const S0 * lsma = (sa + 4*64*(sgitg%2)); + threadgroup const S1 * lsmb = (sb + 2*64*(sgitg/2)); - FOR_UNROLL (short ik = 0; ik < NK/8; ik++) { - simdgroup_barrier(mem_flags::mem_none); + FOR_UNROLL (short ik = 0; ik < NK/8; ik++) { + simdgroup_barrier(mem_flags::mem_none); - FOR_UNROLL (short i = 0; i < 4; i++) { - simdgroup_load(ma[i], lsma + 64*i, 8, 0, false); + FOR_UNROLL (short i = 0; i < 4; i++) { + simdgroup_load(ma[i], lsma + 64*i, 8, 0, false); + } + + simdgroup_barrier(mem_flags::mem_none); + + FOR_UNROLL (short i = 0; i < 2; i++) { + simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false); + } + + simdgroup_barrier(mem_flags::mem_none); + + FOR_UNROLL (short i = 0; i < 8; i++){ + simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]); + } + + lsma += 8*64; + lsmb += 4*64; } - - simdgroup_barrier(mem_flags::mem_none); - - FOR_UNROLL (short i = 0; i < 2; i++) { - simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false); - } - - simdgroup_barrier(mem_flags::mem_none); - - FOR_UNROLL (short i = 0; i < 8; i++){ - simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]); - } - - lsma += 8*64; - lsmb += 4*64; } #else - auto sA = tA.slice(0, 0); - auto sB = tB.slice(0, 0); + auto sA = tA.slice(0, 0); + auto sB0 = tB0.slice(0, 0); - mm.run(sB, sA, cT); + mm.run(sB0, sA, cT0); + + if (has_hi) { + auto sB1 = tB1.slice(0, 0); + + mm.run(sB1, sA, cT1); + } #endif } @@ -694,13 +716,20 @@ kernel void kernel_mul_mm_id( threadgroup_barrier(mem_flags::mem_threadgroup); #ifdef GGML_METAL_HAS_TENSOR - auto tC = tensor, tensor_inline>(sc, dextents(NR0, NR1)); - cT.store(tC); -#else - threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; + auto tC0 = tensor, tensor_inline>(sc, dextents(NR0, NR1H)); + cT0.store(tC0); - for (short i = 0; i < 8; i++) { - simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); + if (has_hi) { + auto tC1 = tensor, tensor_inline>(sc + NR1H*NR0, dextents(NR0, NR1H)); + cT1.store(tC1); + } +#else + if (sg_active) { + threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; + + for (short i = 0; i < 8; i++) { + simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); + } } #endif diff --git a/ggml/src/ggml-metal/kernels/mul_mv.metal b/ggml/src/ggml-metal/kernels/mul_mv.metal index fbe8398ea..8e2df2765 100644 --- a/ggml/src/ggml-metal/kernels/mul_mv.metal +++ b/ggml/src/ggml-metal/kernels/mul_mv.metal @@ -1889,8 +1889,19 @@ void kernel_mul_mv_iq2_xxs_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq2_xxs * x = (device const block_iq2_xxs *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -1898,8 +1909,6 @@ void kernel_mul_mv_iq2_xxs_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem); threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 256); { @@ -1912,11 +1921,9 @@ void kernel_mul_mv_iq2_xxs_f32_impl( threadgroup_barrier(mem_flags::mem_threadgroup); } - const int ix = tiisg; - device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { for (short i = 0; i < 32; ++i) { yl[i] = y4[i]; } @@ -1928,7 +1935,7 @@ void kernel_mul_mv_iq2_xxs_f32_impl( device const uint16_t * q2 = xr->qs + 4 * ib; device const half * dh = &xr->d; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { const float db = dh[0]; device const uint8_t * aux8 = (device const uint8_t *)q2; const uint32_t aux32 = q2[2] | (q2[3] << 16); @@ -1948,7 +1955,7 @@ void kernel_mul_mv_iq2_xxs_f32_impl( q2 += args.nb01/2; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -1961,6 +1968,23 @@ void kernel_mul_mv_iq2_xxs_f32_impl( } } +template +void kernel_mul_mv_iq2_xxs_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq2_xxs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq2_xxs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq2_xxs_f32")]] kernel void kernel_mul_mv_iq2_xxs_f32( constant ggml_metal_kargs_mul_mv & args, @@ -1971,7 +1995,7 @@ kernel void kernel_mul_mv_iq2_xxs_f32( uint3 tgpig[[threadgroup_position_in_grid]], ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq2_xxs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + kernel_mul_mv_iq2_xxs_f32_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); } template @@ -1997,8 +2021,19 @@ void kernel_mul_mv_iq2_xs_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq2_xs * x = (device const block_iq2_xs *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2006,8 +2041,6 @@ void kernel_mul_mv_iq2_xs_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem); threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 512); { @@ -2020,11 +2053,9 @@ void kernel_mul_mv_iq2_xs_f32_impl( threadgroup_barrier(mem_flags::mem_threadgroup); } - const int ix = tiisg; - device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { for (short i = 0; i < 32; ++i) { yl[i] = y4[i]; } @@ -2037,7 +2068,7 @@ void kernel_mul_mv_iq2_xs_f32_impl( device const uint8_t * sc = xr->scales + ib; device const half * dh = &xr->d; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { const float db = dh[0]; const uint8_t ls1 = sc[0] & 0xf; const uint8_t ls2 = sc[0] >> 4; @@ -2066,7 +2097,7 @@ void kernel_mul_mv_iq2_xs_f32_impl( sc += args.nb01; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -2079,6 +2110,23 @@ void kernel_mul_mv_iq2_xs_f32_impl( } } +template +void kernel_mul_mv_iq2_xs_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq2_xs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq2_xs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq2_xs_f32")]] kernel void kernel_mul_mv_iq2_xs_f32( constant ggml_metal_kargs_mul_mv & args, @@ -2090,7 +2138,7 @@ kernel void kernel_mul_mv_iq2_xs_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq2_xs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + kernel_mul_mv_iq2_xs_f32_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); } // FC_mul_mv_split: for nb32 < 32 (nb32 divides 32), 32/nb32 threads share each chunk and each takes a slice of the rows @@ -2117,8 +2165,19 @@ void kernel_mul_mv_iq3_xxs_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq3_xxs * x = (device const block_iq3_xxs *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2126,8 +2185,6 @@ void kernel_mul_mv_iq3_xxs_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - threadgroup uint32_t * svalues = (threadgroup uint32_t *)(shmem); threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 256); { @@ -2140,15 +2197,6 @@ void kernel_mul_mv_iq3_xxs_f32_impl( threadgroup_barrier(mem_flags::mem_threadgroup); } - const short ntx = FC_mul_mv_split ? nb32 : 32; - const short nrep = 32 / ntx; - - const short ix = tiisg % ntx; - const short irep = tiisg / ntx; - - const short row0 = (nr0 * irep ) / nrep; - const short row1 = (nr0 * (irep + 1)) / nrep; - device const float * y4 = y + 32 * ix; for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { @@ -2160,9 +2208,9 @@ void kernel_mul_mv_iq3_xxs_f32_impl( const int ib = ib32 % (QK_K / 32); device const block_iq3_xxs * xr = x + ibl; - device const uint8_t * q3 = xr->qs + 8 * ib + (uint64_t) row0*args.nb01; - device const uint16_t * gas = (device const uint16_t *)(xr->qs + QK_K/4) + 2 * ib + (uint64_t) row0*args.nb01/2; - device const half * dh = &xr->d + (uint64_t) row0*args.nb01/2; + device const uint8_t * q3 = xr->qs + 8 * ib; + device const uint16_t * gas = (device const uint16_t *)(xr->qs + QK_K/4) + 2 * ib; + device const half * dh = &xr->d; for (short row = row0; row < row1; row++) { const float db = dh[0]; @@ -2253,8 +2301,19 @@ void kernel_mul_mv_iq3_s_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq3_s * x = (device const block_iq3_s *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2262,8 +2321,6 @@ void kernel_mul_mv_iq3_s_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - threadgroup uint32_t * svalues = (threadgroup uint32_t *) shmem; { int nval = 8; @@ -2272,11 +2329,9 @@ void kernel_mul_mv_iq3_s_f32_impl( threadgroup_barrier(mem_flags::mem_threadgroup); } - const int ix = tiisg; - device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { for (short i = 0; i < 32; ++i) { yl[i] = y4[i]; } @@ -2291,7 +2346,7 @@ void kernel_mul_mv_iq3_s_f32_impl( device const uint8_t * signs = xr->signs + 4 * ib; device const half * dh = &xr->d; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { const float db = dh[0]; const float d = db * (1 + 2*((sc[0] >> 4*(ib%2)) & 0xf)); @@ -2315,7 +2370,7 @@ void kernel_mul_mv_iq3_s_f32_impl( signs += args.nb01; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -2328,6 +2383,23 @@ void kernel_mul_mv_iq3_s_f32_impl( } } +template +void kernel_mul_mv_iq3_s_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq3_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq3_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq3_s_f32")]] kernel void kernel_mul_mv_iq3_s_f32( constant ggml_metal_kargs_mul_mv & args, @@ -2339,7 +2411,7 @@ kernel void kernel_mul_mv_iq3_s_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq3_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + kernel_mul_mv_iq3_s_f32_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); } template @@ -2365,8 +2437,19 @@ void kernel_mul_mv_iq2_s_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq2_s * x = (device const block_iq2_s *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2374,8 +2457,6 @@ void kernel_mul_mv_iq2_s_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - //threadgroup uint64_t * svalues = (threadgroup uint64_t *) shmem; //{ // int nval = 32; @@ -2384,11 +2465,9 @@ void kernel_mul_mv_iq2_s_f32_impl( // threadgroup_barrier(mem_flags::mem_threadgroup); //} - const short ix = tiisg; - device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { for (short i = 0; i < 32; ++i) { yl[i] = y4[i]; } @@ -2403,7 +2482,7 @@ void kernel_mul_mv_iq2_s_f32_impl( device const uint8_t * signs = qs + QK_K/8; device const half * dh = &xr->d; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { const float db = dh[0]; const float d1 = db * (0.5f + (sc[0] & 0xf)); const float d2 = db * (0.5f + (sc[0] >> 4)); @@ -2428,7 +2507,7 @@ void kernel_mul_mv_iq2_s_f32_impl( signs += args.nb01; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -2441,6 +2520,23 @@ void kernel_mul_mv_iq2_s_f32_impl( } } +template +void kernel_mul_mv_iq2_s_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq2_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq2_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq2_s_f32")]] kernel void kernel_mul_mv_iq2_s_f32( constant ggml_metal_kargs_mul_mv & args, @@ -2452,7 +2548,7 @@ kernel void kernel_mul_mv_iq2_s_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq2_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + kernel_mul_mv_iq2_s_f32_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); } template @@ -2478,8 +2574,19 @@ void kernel_mul_mv_iq1_s_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq1_s * x = (device const block_iq1_s *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2487,13 +2594,9 @@ void kernel_mul_mv_iq1_s_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - - const short ix = tiisg; - device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { float sumy = 0; for (short i = 0; i < 32; ++i) { yl[i] = y4[i]; @@ -2508,7 +2611,7 @@ void kernel_mul_mv_iq1_s_f32_impl( device const uint16_t * qh = xr->qh + ib; device const half * dh = &xr->d; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700))); constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 5) & 0x700))); constant uint8_t * grid3 = (constant uint8_t *)(iq1s_grid_gpu + (qs[2] | ((qh[0] << 2) & 0x700))); @@ -2528,7 +2631,7 @@ void kernel_mul_mv_iq1_s_f32_impl( qh += args.nb01/2; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -2541,6 +2644,23 @@ void kernel_mul_mv_iq1_s_f32_impl( } } +template +void kernel_mul_mv_iq1_s_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq1_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq1_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq1_s_f32")]] kernel void kernel_mul_mv_iq1_s_f32( constant ggml_metal_kargs_mul_mv & args, @@ -2551,7 +2671,7 @@ kernel void kernel_mul_mv_iq1_s_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq1_s_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); + kernel_mul_mv_iq1_s_f32_disp(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); } template @@ -2577,8 +2697,19 @@ void kernel_mul_mv_iq1_m_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq1_m * x = (device const block_iq1_m *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2586,15 +2717,11 @@ void kernel_mul_mv_iq1_m_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - - const short ix = tiisg; - device const float * y4 = y + 32 * ix; iq1m_scale_t scale; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { float4 sumy = {0.f}; for (short i = 0; i < 8; ++i) { yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0]; @@ -2611,7 +2738,7 @@ void kernel_mul_mv_iq1_m_f32_impl( device const uint8_t * qh = xr->qh + 2 * ib; device const uint16_t * sc = (device const uint16_t *)xr->scales; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700))); @@ -2637,7 +2764,7 @@ void kernel_mul_mv_iq1_m_f32_impl( qh += args.nb01; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -2650,6 +2777,23 @@ void kernel_mul_mv_iq1_m_f32_impl( } } +template +void kernel_mul_mv_iq1_m_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq1_m_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq1_m_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq1_m_f32")]] kernel void kernel_mul_mv_iq1_m_f32( constant ggml_metal_kargs_mul_mv & args, @@ -2660,7 +2804,7 @@ kernel void kernel_mul_mv_iq1_m_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq1_m_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); + kernel_mul_mv_iq1_m_f32_disp(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); } template @@ -3239,13 +3383,13 @@ template [[host_name("kernel_mul_mv_id_q3_K_f32")]] kernel kernel_mul_mv_id_t template [[host_name("kernel_mul_mv_id_q4_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_q5_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_q6_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq1_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq1_m_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq2_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq2_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq1_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq1_m_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq2_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq2_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_iq3_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq3_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq2_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq3_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq2_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_iq4_nl_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_iq4_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_tq2_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index d0ff6df5c..59d47d370 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -5430,8 +5430,9 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { bool prefer_large = tiles_m > shader_core_count || tiles_l > shader_core_count || (tiles_l <= shader_core_count / 3 && tiles_m > shader_core_count / 2); if (n > crossover_large && prefer_large) return last; - uint32_t crossover_medium = configs[0].unaligned->wg_denoms[1]; - if (n > crossover_medium) return 1; + uint32_t crossover_medium_m = configs[0].unaligned->wg_denoms[0]; + uint32_t crossover_medium_n = configs[0].unaligned->wg_denoms[1]; + if (m > crossover_medium_m && n > crossover_medium_n) return 1; return 0; }; device->matmul_id_tile_selector = [](uint32_t /*m*/, uint32_t n, uint32_t /*k*/, uint32_t /*shader_core_count*/, @@ -9061,7 +9062,7 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m, } uint32_t split_k = 1; - if (ctx->device->shader_core_count != 0 && m >= pipeline->wg_denoms[0] && n >= pipeline->wg_denoms[1]) { + if (ctx->device->shader_core_count != 0 && n >= pipeline->wg_denoms[1]) { // If k is 'large' and the SMs will fill less than halfway, use split_k. uint32_t m_tiles = CEIL_DIV(m, pipeline->wg_denoms[0]); uint32_t n_tiles = CEIL_DIV(n, pipeline->wg_denoms[1]); @@ -9814,10 +9815,10 @@ static bool ggml_vk_should_use_mmvq(const vk_device& device, uint32_t m, uint32_ GGML_UNUSED(m); } -static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { +static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx, bool swap_inputs = false) { ggml_tensor * dst = cgraph->nodes[node_idx]; - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; + const ggml_tensor * src0 = dst->src[swap_inputs ? 1 : 0]; + const ggml_tensor * src1 = dst->src[swap_inputs ? 0 : 1]; VK_LOG_DEBUG("ggml_vk_mul_mat_vec_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; @@ -9836,8 +9837,8 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& const uint64_t ne12 = src1->ne[2]; const uint64_t ne13 = src1->ne[3]; - const uint64_t ne20 = dst->ne[0]; - const uint64_t ne21 = dst->ne[1]; + const uint64_t ne20 = dst->ne[swap_inputs ? 1 : 0]; + const uint64_t ne21 = dst->ne[swap_inputs ? 0 : 1]; // const uint64_t ne22 = dst->ne[2]; // const uint64_t ne23 = dst->ne[3]; @@ -10451,6 +10452,16 @@ static void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context& subctx, c src0->ne[1] <= ctx->device->properties.limits.maxComputeWorkGroupCount[1] && src1->ne[2] <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]) { ggml_vk_mul_mat_vec_nc_f16_f32(ctx, subctx, cgraph, node_idx); + // With one output row, B^T*A has the same flat output as A^T*B. + } else if (ctx->num_additional_fused_ops == 0 && + (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) && + (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_BF16 || ggml_is_quantized(src1->type)) && + dst->ne[0] == 1 && dst->ne[1] > mul_mat_vec_max_cols && + src0->ne[2] == 1 && src0->ne[3] == 1 && + src1->ne[2] == 1 && src1->ne[3] == 1 && + ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst) && + get_misalign_bytes(ctx, src0) == 0 && get_misalign_bytes(ctx, src1) == 0 && get_misalign_bytes(ctx, dst) == 0) { + ggml_vk_mul_mat_vec_q_f16(ctx, subctx, cgraph, node_idx, true); // mul_mat_vec supports batching ne12*ne13 when ne11==1, or treating ne11 as the batch size (up to four) // when ne12 and ne13 are one. } else if ((dst->ne[1] == 1 || (dst->ne[1] <= mul_mat_vec_max_cols && src1->ne[2] * src1->ne[3] == 1)) && @@ -17158,6 +17169,22 @@ static bool ggml_backend_vk_cpy_tensor_async(ggml_backend_t backend_src, ggml_ba return false; } + // If the backend is idle, use a CPU copy to avoid GPU synchronization overhead. + static constexpr size_t max_cpu_copy_size = 128 * 1024; + const bool src_backend_synchronous = backend_src->iface.synchronize == nullptr; + const bool transfer_idle = !ctx->device->async_use_transfer_queue || + ctx->transfer_semaphore_last_submitted == ctx->transfer_semaphore.value; + const bool backend_idle = ctx->compute_ctx.expired() && ctx->transfer_ctx.expired() && + !ctx->submit_pending && !ctx->almost_ready_fence_pending && transfer_idle; + const bool dst_host_coherent = + (dst_buf->memory_property_flags & (vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent)) == + (vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent); + + if ((backend_src == backend_dst || src_backend_synchronous) && backend_idle && dst_host_coherent && ggml_nbytes(src) <= max_cpu_copy_size) { + ggml_vk_buffer_write(dst_buf, vk_tensor_offset(dst) + dst->view_offs, src->data, ggml_nbytes(src)); + return true; + } + vk_context cpy_ctx; if (ctx->device->async_use_transfer_queue) { cpy_ctx = ggml_vk_get_transfer_ctx(ctx); @@ -17170,7 +17197,6 @@ static bool ggml_backend_vk_cpy_tensor_async(ggml_backend_t backend_src, ggml_ba src->data, ggml_nbytes(src)); } - GGML_UNUSED(backend_src); return false; } @@ -18351,38 +18377,30 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg bool need_disable = false; - // topk_moe often overwrites the source, but for a given row all the src values are - // loaded before anything is stored. If there's only one row, this is safe, so treat - // this as a special case. - bool is_topk_moe_single_row = ctx->fused_topk_moe_mode != TOPK_MOE_COUNT && - ggml_nrows(cgraph->nodes[i]->src[0]) == 1; - - if (!is_topk_moe_single_row) { - for (int j = 0; j < 2; ++j) { - ggml_tensor *dst = output_nodes[j]; - if (!dst) { - continue; - } - // Loop over all srcs of all nodes in the fusion. If the src overlaps - // the destination and the src is not an intermediate node that's being - // elided, then disable fusion. - for (int k = 0; k <= ctx->num_additional_fused_ops; ++k) { - for (uint32_t s = 0; s < GGML_MAX_SRC; ++s) { - ggml_tensor *src = cgraph->nodes[i + k]->src[s]; - if (!src || src->op == GGML_OP_NONE) { - continue; + for (int j = 0; j < 2; ++j) { + ggml_tensor *dst = output_nodes[j]; + if (!dst) { + continue; + } + // Loop over all srcs of all nodes in the fusion. If the src overlaps + // the destination and the src is not an intermediate node that's being + // elided, then disable fusion. + for (int k = 0; k <= ctx->num_additional_fused_ops; ++k) { + for (uint32_t s = 0; s < GGML_MAX_SRC; ++s) { + ggml_tensor *src = cgraph->nodes[i + k]->src[s]; + if (!src || src->op == GGML_OP_NONE) { + continue; + } + if (ggml_vk_tensors_overlap(src, dst, op_srcs_fused_elementwise[k])) { + bool found = false; + for (int n = 0; n < k; ++n) { + if (cgraph->nodes[i + n] == src) { + found = true; + break; + } } - if (ggml_vk_tensors_overlap(src, dst, op_srcs_fused_elementwise[k])) { - bool found = false; - for (int n = 0; n < k; ++n) { - if (cgraph->nodes[i + n] == src) { - found = true; - break; - } - } - if (!found) { - need_disable = true; - } + if (!found) { + need_disable = true; } } } @@ -18395,6 +18413,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg ctx->fused_topk_moe_scale = false; ctx->fused_topk_qsa = false; ctx->fused_rms_norm_mode = RMS_NORM_COUNT; + fusion_string = nullptr; } } @@ -18497,7 +18516,6 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg // Sort the graph for improved parallelism. static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, struct ggml_backend_graph_optimize_params * params) { - GGML_UNUSED(params); VK_LOG_DEBUG("ggml_vk_graph_optimize(" << graph->n_nodes << " nodes)"); ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; @@ -18583,19 +18601,50 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * return false; }; - if (keep_pattern(topk_moe_early_softmax_norm)) { + auto const &add_pattern_alloc_deps = [&](const std::initializer_list &pattern, int last_node) { + // Keep external inputs alive through the fused output. + std::set seen; + for (size_t j = 0; j < pattern.size(); ++j) { + ggml_tensor * node = graph->nodes[first_unused + j]; + for (uint32_t s = 0; s < GGML_MAX_SRC; ++s) { + ggml_tensor * src = node->src[s]; + if (src && seen.insert(src).second) { + params->add_alloc_dep(params->user_data, src, graph->nodes[last_node]); + } + } + seen.insert(node); + } + }; + + auto const &keep_topk_moe_pattern = [&](const std::initializer_list &pattern) -> bool { + if (!match_pattern(pattern, first_unused)) { + return false; + } + + int last_node = first_unused + (int) pattern.size() - 1; + // Some TOPK_MOE variants fuse a trailing scale. + if (last_node + 1 < graph->n_nodes && graph->nodes[last_node + 1]->op == GGML_OP_SCALE) { + last_node++; + } + + add_pattern_alloc_deps(pattern, last_node); + + return keep_pattern(pattern); + }; + + if (keep_topk_moe_pattern(topk_moe_early_softmax_norm)) { continue; } - if (keep_pattern(topk_moe_sigmoid_norm_bias)) { + if (keep_topk_moe_pattern(topk_moe_sigmoid_norm_bias)) { continue; } - if (keep_pattern(topk_moe_sqrt_softplus_norm_bias)) { + if (keep_topk_moe_pattern(topk_moe_sqrt_softplus_norm_bias)) { continue; } - if (keep_pattern(topk_moe_early_softmax)) { + if (keep_topk_moe_pattern(topk_moe_early_softmax)) { continue; } - if (keep_pattern(topk_moe_late_softmax)) { + if (keep_topk_moe_pattern(topk_moe_late_softmax)) { continue; } if (keep_pattern(snake_pattern)) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp b/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp index 0fc2b9b72..4ba63f7ae 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp @@ -33,7 +33,11 @@ void argsort(bool needs_bounds_check, const uint row) { const uint row_offset = row * p.ncols; // initialize indices - dst_row[col] = ivec2(col, floatBitsToInt(data_a[row_offset + col])); + ivec2 value = ivec2(col, 0); + if (!needs_bounds_check || col < p.ncols) { + value.y = floatBitsToInt(data_a[row_offset + col]); + } + dst_row[col] = value; barrier(); uint num_outer_loop_iters = NCOLS_PADDED_LOG2; @@ -42,18 +46,20 @@ void argsort(bool needs_bounds_check, const uint row) { [[unroll]] for (uint j = k / 2, inner_idx = 0; inner_idx < num_inner_loop_iters; j /= 2, inner_idx++) { const int ixj = int(col ^ j); - int idx_0 = (col & k) == 0 ? col : ixj; - int idx_1 = (col & k) == 0 ? ixj : col; + if (ixj > col) { + int idx_0 = (col & k) == 0 ? col : ixj; + int idx_1 = (col & k) == 0 ? ixj : col; - ivec2 sh_idx_0 = dst_row[idx_0]; - ivec2 sh_idx_1 = dst_row[idx_1]; - bool idx_0_oob = needs_bounds_check ? sh_idx_0.x >= p.ncols : false; - bool idx_1_oob = needs_bounds_check ? sh_idx_1.x >= p.ncols : false; + ivec2 sh_idx_0 = dst_row[idx_0]; + ivec2 sh_idx_1 = dst_row[idx_1]; + bool idx_0_oob = needs_bounds_check ? sh_idx_0.x >= p.ncols : false; + bool idx_1_oob = needs_bounds_check ? sh_idx_1.x >= p.ncols : false; - if ((idx_0_oob || - (!idx_1_oob && intBitsToFloat(sh_idx_0.y) > intBitsToFloat(sh_idx_1.y))) && (ixj > col)) { - dst_row[idx_0] = sh_idx_1; - dst_row[idx_1] = sh_idx_0; + if (idx_0_oob || + (!idx_1_oob && intBitsToFloat(sh_idx_0.y) > intBitsToFloat(sh_idx_1.y))) { + dst_row[idx_0] = sh_idx_1; + dst_row[idx_1] = sh_idx_0; + } } barrier(); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp b/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp index 920bac6bb..b2df44137 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp @@ -42,7 +42,10 @@ void argsort(bool needs_bounds_check, const uint row) { [[unroll]] for (int u = 0; u < WG_UNROLL_FACTOR; ++u) { uint c = u*BLOCK_SIZE + col; if (c < p.ncols_padded) { - ivec2 v = ivec2(c, floatBitsToInt(data_a[row_offset + c])); + ivec2 v = ivec2(c, 0); + if (!needs_bounds_check || c < p.ncols) { + v.y = floatBitsToInt(data_a[row_offset + c]); + } tmp_idx[idx_offset + c] = v; } } diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index ad3497997..7ae78b36b 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -1114,7 +1114,7 @@ static speculative_draft_result speculative_decoding_eval_chunk(llama_context * auto & dp = common_speculative_get_draft_params(draft_spec, 0); dp.drafting = true; dp.n_max = n_draft_max; - dp.n_past = n_past; + dp.pos0 = n_past; dp.id_last = embd[0]; dp.prompt = &prompt_tokens; dp.result = &drafted_ids; diff --git a/src/llama-context.cpp b/src/llama-context.cpp index b99e02856..963338bfc 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -675,7 +675,9 @@ void llama_context::sched_reserve() { // need to implement a more robust mechanism that tries a few different inputs and analyzes the results ggml_cgraph * gf = nullptr; switch (model.arch) { + case LLM_ARCH_KIMI_LINEAR: case LLM_ARCH_MINIMAX_01: + // [TAG_RESERVE_DIAG_DECAY] // the `inp_diag_decay` tensor size scales with `n_seq_tokens^2` which // makes `n_seqs == 1` use more memory for the compute graph compared to `n_seqs > 1` gf = graph_reserve(n_tokens, 1, n_outputs_pp, mctx.get(), model.hparams.no_alloc); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 55feb6684..91c8ea9ea 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2798,9 +2798,9 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, filter = [&](uint32_t il) { return il >= hparams.n_layer(); }; } - if ((arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_HY_V3 || arch == LLM_ARCH_GLM_DSA || - arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_DEEPSEEK32) && - hparams.n_layer_nextn > 0) { + // don't filter when n_layer_nextn is repurposed for a router layer the trunk attends + // or when a model is entirely n_layer_nextn layers and has no trunk + if (hparams.n_layer_nextn > 0 && hparams.n_layer() > 0 && hparams.router_layer < 0) { if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) { filter = [&](uint32_t il) { return il >= hparams.n_layer(); }; } else { diff --git a/src/models/gemma3n.cpp b/src/models/gemma3n.cpp index ea616db3b..bb628203a 100644 --- a/src/models/gemma3n.cpp +++ b/src/models/gemma3n.cpp @@ -82,7 +82,7 @@ std::unique_ptr llama_model_gemma3n::build_arch_graph(const l } // get 2D slice view from a 3D tensor, the idx corresponds to the 3rd dim -static ggml_tensor * ggml_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) { +static ggml_tensor * gemma3n_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) { GGML_ASSERT(idx < (int) x->ne[2]); return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], ggml_row_size(x->type, x->ne[0]), idx * x->ne[0] * x->ne[1] * ggml_element_size(x)); @@ -139,7 +139,7 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par ggml_tensor * predictions = altup_predict(cur, il); // [n_embd, n_tokens, n_altup] // predicted value will go through self-attention and laurel - ggml_tensor * active_prediction = ggml_view_2d_slice(ctx0, predictions, i_altup_act); // [n_embd, n_tokens] + ggml_tensor * active_prediction = gemma3n_view_2d_slice(ctx0, predictions, i_altup_act); // [n_embd, n_tokens] cur = active_prediction; cb(cur, "active_prediction", il); @@ -236,13 +236,13 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par ggml_tensor * first_prediction; // [n_embd, n_tokens] { - first_prediction = ggml_view_2d_slice(ctx0, corrected, i_altup_act); // [n_embd, n_tokens] + first_prediction = gemma3n_view_2d_slice(ctx0, corrected, i_altup_act); // [n_embd, n_tokens] first_prediction = ggml_mul(ctx0, first_prediction, model.layers[il].altup_correct_scale); first_prediction = build_lora_mm(model.layers[il].per_layer_inp_gate, first_prediction); first_prediction = ggml_gelu(ctx0, first_prediction); // [n_embd_altup, n_tokens] cb(first_prediction, "first_prediction_gated", il); - ggml_tensor * inp_this_layer = ggml_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_altup, n_tokens] + ggml_tensor * inp_this_layer = gemma3n_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_altup, n_tokens] first_prediction = ggml_mul(ctx0, first_prediction, inp_this_layer); // [n_embd_altup, n_tokens] cb(first_prediction, "first_prediction_scaled", il); @@ -253,7 +253,7 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par } // equivalent to python code: corrected_predictions[1:] += first_prediction { - ggml_tensor * slice_first = ggml_view_2d_slice(ctx0, corrected, 0); + ggml_tensor * slice_first = gemma3n_view_2d_slice(ctx0, corrected, 0); ggml_tensor * slice_rest = ggml_view_3d( ctx0, corrected, n_embd, n_tokens, n_altup - 1, ggml_row_size(corrected->type, n_embd), ggml_row_size(corrected->type, n_embd * n_tokens), n_embd * n_tokens * ggml_element_size(corrected)); @@ -271,7 +271,7 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par // cur now has multiple altup(s), we want to merge them back to 1 altup { - ggml_tensor * target_magnitude = calc_magnitude(ggml_view_2d_slice(ctx0, cur, i_altup_act)); // [n_embd, n_tokens] + ggml_tensor * target_magnitude = calc_magnitude(gemma3n_view_2d_slice(ctx0, cur, i_altup_act)); // [n_embd, n_tokens] // do a view to skip the first slice (active altup) ggml_tensor * alt_slice = ggml_view_3d(ctx0, cur, n_embd, n_tokens, n_altup - 1, ggml_row_size(cur->type, n_embd), @@ -283,9 +283,9 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par cb(altup_unembd, "altup_unembd", -1); // equivalent to torch.mean(hidden_states, dim=0) - cur = ggml_view_2d_slice(ctx0, cur, 0); // [n_embd, n_tokens] + cur = gemma3n_view_2d_slice(ctx0, cur, 0); // [n_embd, n_tokens] for (int i = 0; i < n_altup - 1; ++i) { - cur = ggml_add(ctx0, cur, ggml_view_2d_slice(ctx0, altup_unembd, i)); + cur = ggml_add(ctx0, cur, gemma3n_view_2d_slice(ctx0, altup_unembd, i)); } cur = ggml_scale(ctx0, cur, 1.0f / float(n_altup)); // [n_embd, n_tokens] cb(cur, "unembd_merged", -1); @@ -419,7 +419,7 @@ ggml_tensor * llama_model_gemma3n::graph::altup_compute_router_modalities(ggml_t // input cur shape: [n_embd, n_tokens, n_altup] // output shape: [n_embd, n_tokens, n_altup] ggml_tensor * llama_model_gemma3n::graph::altup_predict(ggml_tensor * cur, int il) { - ggml_tensor * activated = ggml_view_2d_slice(ctx0, cur, i_altup_act); // [n_embd, n_tokens] + ggml_tensor * activated = gemma3n_view_2d_slice(ctx0, cur, i_altup_act); // [n_embd, n_tokens] ggml_tensor * modalities = altup_compute_router_modalities(activated, il); // [n_altup, n_tokens] cb(modalities, "modalities", il); @@ -447,7 +447,7 @@ ggml_tensor * llama_model_gemma3n::graph::altup_correct(ggml_tensor * prediction ggml_tensor * modalities = altup_compute_router_modalities(activated, il); // [n_altup, n_tokens] cb(modalities, "modalities", il); - ggml_tensor * active_prediction = ggml_view_2d_slice(ctx0, predictions, i_altup_act); + ggml_tensor * active_prediction = gemma3n_view_2d_slice(ctx0, predictions, i_altup_act); ggml_tensor * innovation = ggml_sub(ctx0, activated, active_prediction); // [n_embd, n_tokens] cb(innovation, "innovation", il); diff --git a/src/models/gemma4.cpp b/src/models/gemma4.cpp index b8ae9623e..39e899aa6 100644 --- a/src/models/gemma4.cpp +++ b/src/models/gemma4.cpp @@ -145,11 +145,11 @@ std::unique_ptr llama_model_gemma4::build_arch_graph(const ll } // get 2D slice view from a 3D tensor, the idx corresponds to the 3rd dim -// static ggml_tensor * ggml_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) { -// GGML_ASSERT(idx < (int) x->ne[2]); -// return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], ggml_row_size(x->type, x->ne[0]), -// idx * x->ne[0] * x->ne[1] * ggml_element_size(x)); -// } +static ggml_tensor * gemma4_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) { + GGML_ASSERT(idx < (int) x->ne[2]); + return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], ggml_row_size(x->type, x->ne[0]), + idx * x->ne[0] * x->ne[1] * ggml_element_size(x)); +} llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params), @@ -372,7 +372,7 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para cur = build_lora_mm(model.layers[il].per_layer_inp_gate, cur); // [n_embd_per_layer, n_tokens] cur = ggml_gelu(ctx0, cur); - ggml_tensor * inp_this_layer = ggml_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_per_layer, n_tokens] + ggml_tensor * inp_this_layer = gemma4_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_per_layer, n_tokens] // TODO @ngxson : improve this if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) { diff --git a/src/models/minimax-01.cpp b/src/models/minimax-01.cpp index 361114acc..9fa2e8fc0 100644 --- a/src/models/minimax-01.cpp +++ b/src/models/minimax-01.cpp @@ -229,6 +229,7 @@ llama_model_minimax_01::graph::graph(const llama_model & model, const llm_graph_ ggml_set_input(inp->inp_k_decay); cb(inp->inp_k_decay, "k_decay_exp", -1); + // [TAG_RESERVE_DIAG_DECAY] inp->inp_diag_decay = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_seq_tokens, n_seq_tokens, n_head, n_seqs); ggml_set_input(inp->inp_diag_decay); cb(inp->inp_diag_decay, "diag_decay_exp", -1); diff --git a/src/models/plamo2.cpp b/src/models/plamo2.cpp index d946b3cff..ba1cea146 100644 --- a/src/models/plamo2.cpp +++ b/src/models/plamo2.cpp @@ -142,6 +142,11 @@ llama_model_plamo2::graph::graph(const llama_model & model, const llm_graph_para cur = build_plamo2_attn_layer(inp_hybrid->get_attn(), inp_pos, cur, model, il); } + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + residual = ggml_get_rows(ctx0, residual, inp_out_ids); + } + // post_mixer_norm cur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il); cb(cur, "attn_post_norm", il); @@ -167,11 +172,6 @@ llama_model_plamo2::graph::graph(const llama_model & model, const llm_graph_para cur = build_norm(cur, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, il); cb(cur, "ffn_post_norm", il); - if (il == n_layer - 1 && inp_out_ids) { - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - residual = ggml_get_rows(ctx0, residual, inp_out_ids); - } - // residual connection cur = ggml_add(ctx0, cur, residual); cb(cur, "ffn_residual", il); diff --git a/src/models/qwen3vl.cpp b/src/models/qwen3vl.cpp index 5596620f0..30c08ed35 100644 --- a/src/models/qwen3vl.cpp +++ b/src/models/qwen3vl.cpp @@ -18,6 +18,7 @@ void llama_model_qwen3vl::load_arch_tensors(llama_model_loader &) { int64_t n_vocab_out = n_vocab; if (arch == LLM_ARCH_QWEN3TTS) { + // [TAG_LLAMA_N_VOCAB_OUT] n_vocab_out = 3072; } diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index 2ac98b6fd..483391333 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -15,6 +15,23 @@ #include #include #include +#include +#include + +#ifdef _WIN32 +// windows.h defines min and max as macros, which breaks std::min and std::max +#define WIN32_LEAN_AND_MEAN +#ifndef NOMINMAX +# define NOMINMAX +#endif +#include +#include +#else +#include +#include +#include +#include +#endif json format_error_response(const std::string & message, const enum error_type type) { std::string type_str; @@ -1832,3 +1849,133 @@ server_tokens format_prompt_rerank( return result; } + +// +// server_subproc +// + +bool server_subproc::has_output() { + if (out_handle >= 0) { + return true; + } + FILE * f = sproc.stdout_file(); // combined stdout/stderr + if (!f) { + return false; + } +#ifdef _WIN32 + HANDLE h = (HANDLE) _get_osfhandle(_fileno(f)); + if (h != INVALID_HANDLE_VALUE) { + out_handle = (intptr_t) h; + } +#else + int fd = fileno(f); + if (fd >= 0) { + fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK); + out_handle = fd; + } +#endif + return out_handle >= 0; +} + +int server_subproc::read_output(char * buf, size_t len) { + if (!has_output()) { + return -1; + } +#ifdef _WIN32 + HANDLE h = (HANDLE) out_handle; + DWORD avail = 0; + if (!PeekNamedPipe(h, NULL, 0, NULL, &avail, NULL)) { + return -1; // pipe broken, child gone + } + if (avail == 0) { + return 0; + } + DWORD to_read = avail < (DWORD) len ? avail : (DWORD) len; + DWORD got = 0; + if (!ReadFile(h, buf, to_read, &got, NULL) || got == 0) { + return -1; + } + return (int) got; +#else + while (true) { + ssize_t r = read((int) out_handle, buf, len); + if (r > 0) { + return (int) r; + } + if (r == 0) { + return -1; // EOF + } + if (errno == EINTR) { + continue; + } + if (errno == EAGAIN || errno == EWOULDBLOCK) { + return 0; + } + return -1; + } +#endif +} + +server_subproc::waiter::waiter() { +#ifndef _WIN32 + int fds[2]; + GGML_ASSERT(pipe(fds) == 0); + for (int fd : fds) { + fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK); + } + wake_fd[0] = fds[0]; + wake_fd[1] = fds[1]; +#endif +} + +server_subproc::waiter::~waiter() { +#ifndef _WIN32 + close((int) wake_fd[0]); + close((int) wake_fd[1]); +#endif +} + +void server_subproc::waiter::wake() { +#ifndef _WIN32 + char c = 1; + (void) !write((int) wake_fd[1], &c, 1); +#endif +} + +void server_subproc::waiter::wait(const std::vector & procs, std::vector & ready, int64_t timeout_ms) { + ready.assign(procs.size(), false); +#ifdef _WIN32 + // no waitable wait exists for anonymous pipes, so poll them in 50 ms steps + bool any = false; + for (size_t i = 0; i < procs.size(); i++) { + DWORD avail = 0; + if (!procs[i]->has_output() || !PeekNamedPipe((HANDLE) procs[i]->out_handle, NULL, 0, NULL, &avail, NULL) || avail > 0) { + ready[i] = true; // data or broken pipe, read_output() tells which + any = true; + } + } + if (!any) { + int64_t step = timeout_ms < 0 ? 50 : std::min(timeout_ms, 50); + std::this_thread::sleep_for(std::chrono::milliseconds(step)); + } +#else + std::vector pfds; + pfds.reserve(procs.size() + 1); + pfds.push_back({ (int) wake_fd[0], POLLIN, 0 }); + for (auto * p : procs) { + pfds.push_back({ p->has_output() ? (int) p->out_handle : -1, POLLIN, 0 }); // poll() skips negative fds + } + int timeout = timeout_ms < 0 ? -1 : (int) std::min(timeout_ms, std::numeric_limits::max()); + int r = poll(pfds.data(), pfds.size(), timeout); + if (r < 0 && errno != EINTR) { + LOG_ERR("%s: poll() failed: %s\n", __func__, strerror(errno)); + } + if (pfds[0].revents) { + char buf[64]; + while (read((int) wake_fd[0], buf, sizeof(buf)) > 0) {} + } + for (size_t i = 0; i < procs.size(); i++) { + ready[i] = pfds[i + 1].fd < 0 || pfds[i + 1].revents != 0; + } +#endif +} diff --git a/tools/server/server-common.h b/tools/server/server-common.h index 6c681a2cf..9894f5f06 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -6,6 +6,7 @@ #include "chat.h" #include "mtmd.h" #include "mtmd-helper.h" +#include "subproc.h" #include "json.h" @@ -13,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -611,3 +613,39 @@ struct server_pipe { return true; } }; + +// wrapper around common_subproc to manage a child server process +// mainly used by router mode +struct server_subproc { + common_subproc sproc; + std::atomic stopped{false}; // set by the monitor once the process exited and was reaped + + bool is_alive() { return sproc.alive(); } + void terminate() { sproc.terminate(); } + int join() { return sproc.join(); } + + // true if the child's combined stdout/stderr pipe is available (call after create()) + bool has_output(); + + // non-blocking read + // returns the number of bytes read, 0 when nothing is available, -1 when the pipe is closed or broken + int read_output(char * buf, size_t len); + + // wait until one of a set of children has output, wake() is called, or a timeout passes + struct waiter { + waiter(); + ~waiter(); + + // thread-safe; on Windows this is a no-op, wait() returns within 50 ms anyway + void wake(); + + // timeout_ms < 0 waits until data or wake(); ready[i] is set for each child with data (or a broken pipe) + void wait(const std::vector & procs, std::vector & ready, int64_t timeout_ms); + + private: + intptr_t wake_fd[2] = { -1, -1 }; // POSIX self-pipe + }; + +private: + intptr_t out_handle = -1; // fd on POSIX, HANDLE on Windows; taken lazily from sproc +}; diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index fe068d3e9..b6835e434 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3028,7 +3028,7 @@ private: common_speculative_get_draft_params(spec.get(), slot.id) = { /* .drafting = */ true, /* .n_max = */ n_draft_max, - /* .n_past = */ slot.prompt.n_tokens(), + /* .pos0 = */ slot.prompt.tokens.pos_next(), /* .id_last = */ slot.sampled, /* .prompt = */ &slot.spec_prompt, /* .result = */ &slot.spec_draft, diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 4d2592b25..3d134acf3 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -44,30 +44,215 @@ extern char **environ; #define CMD_ROUTER_TO_CHILD_EXIT "cmd_router_to_child:exit" #define CMD_CHILD_TO_ROUTER_STATE "cmd_child_to_router:state:" // followed by json string +// note: SIGPIPE is ignored by the server +static void request_child_exit(server_subproc & proc) { + FILE * stdin_file = proc.sproc.stdin_file(); + if (stdin_file) { + fprintf(stdin_file, "%s\n", CMD_ROUTER_TO_CHILD_EXIT); + fflush(stdin_file); + } +} + // address for child process, this is needed because router may run on 0.0.0.0 // ref: https://github.com/ggml-org/llama.cpp/issues/17862 #define CHILD_ADDR "127.0.0.1" -struct server_subproc { - common_subproc sproc; // not yet spawned while in DOWNLOADING state - std::atomic stopped{false}; // set to cancel a download or signal child process exit - - bool is_alive() { - return sproc.alive(); +// single-threaded, watching all child processes at once +struct server_monitor { + server_monitor(server_models & models) : models(models) { + th = std::thread([this]() { run(); }); } - void request_exit() { - FILE * stdin_file = sproc.stdin_file(); - if (stdin_file) { - fprintf(stdin_file, "%s\n", CMD_ROUTER_TO_CHILD_EXIT); - fflush(stdin_file); + ~server_monitor() { + push({ cmd_t::QUIT, {}, "", 0, false }); + th.join(); + } + + // thread-safe + void watch(const std::string & name, std::shared_ptr proc, server_child_mode mode, int port) { + child_t c; + c.name = name; + c.proc = std::move(proc); + c.mode = mode; + c.port = port; + if (!c.proc->has_output()) { + SRV_ERR("failed to get stdout/stderr of child process for name=%s\n", name.c_str()); + c.eof = true; } - stopped.store(true, std::memory_order_relaxed); + push({ cmd_t::WATCH, std::move(c), "", 0, false }); } - void terminate() { - sproc.terminate(); + // thread-safe + void stop(const std::string & name, int stop_timeout, bool send_exit) { + push({ cmd_t::STOP, {}, name, stop_timeout, send_exit }); } + +private: + struct child_t { + std::string name; + std::shared_ptr proc; + server_child_mode mode = SERVER_CHILD_MODE_NORMAL; + int port = 0; + std::string buf; // partial line + bool eof = false; // output closed, waiting for the process to be reaped + int64_t deadline = 0; // force-kill time in ms, 0 when no stop is pending + }; + + struct cmd_t { + enum { WATCH, STOP, QUIT } type; + child_t child; + std::string name; + int stop_timeout; + bool send_exit; + }; + + void push(cmd_t && cmd) { + { + std::lock_guard lk(mu); + cmds.push_back(std::move(cmd)); + } + waiter.wake(); + } + + // returns true if the loop should exit + bool handle_commands() { + std::deque batch; + { + std::lock_guard lk(mu); + batch.swap(cmds); + } + for (auto & cmd : batch) { + switch (cmd.type) { + case cmd_t::WATCH: + children.push_back(std::move(cmd.child)); + break; + case cmd_t::STOP: + // the newest child with this name is the one the registry knows + for (auto it = children.rbegin(); it != children.rend(); ++it) { + if (it->name != cmd.name) { + continue; + } + if (cmd.send_exit && !it->eof) { + request_child_exit(*it->proc); + } + it->deadline = ggml_time_ms() + (int64_t) cmd.stop_timeout * 1000; + break; + } + break; + case cmd_t::QUIT: + return true; + } + } + return false; + } + + // read what the child wrote, forward complete lines + void read_output(child_t & c) { + char chunk[4096]; + while (!c.eof) { + int n = c.proc->read_output(chunk, sizeof(chunk)); + if (n < 0) { + c.eof = true; + break; + } + if (n == 0) { + break; + } + c.buf.append(chunk, (size_t) n); + size_t start = 0; + while (true) { + size_t nl = c.buf.find('\n', start); + if (nl == std::string::npos) { + break; + } + std::string line = c.buf.substr(start, nl + 1 - start); + start = nl + 1; + on_line(c, line); + } + c.buf.erase(0, start); + if (c.buf.size() > max_line) { + c.buf.clear(); // a child that never writes a newline must not grow this without bound + } + } + if (c.eof && !c.buf.empty()) { + on_line(c, c.buf); + c.buf.clear(); + } + } + + void on_line(child_t & c, const std::string & line) { + if (string_starts_with(line, CMD_CHILD_TO_ROUTER_STATE)) { + LOG_DBG("[%5d] %s", c.port, line.c_str()); // prevent spamming the log + models.handle_child_state(c.name, line); + } else { + LOG("[%5d] %s", c.port, line.c_str()); // forward log + } + } + + void run() { + while (true) { + if (handle_commands()) { + return; + } + + // wait for output, a wakeup, or the next deadline; + // a child whose output closed is polled for its exit every 50 ms + int64_t now = ggml_time_ms(); + int64_t timeout = -1; + for (const auto & c : children) { + if (c.eof) { + timeout = timeout < 0 ? 50 : std::min(timeout, 50); + } + if (c.deadline) { + int64_t d = std::max(0, c.deadline - now); + timeout = timeout < 0 ? d : std::min(timeout, d); + } + } + std::vector procs; + std::vector owners; + for (auto & c : children) { + if (!c.eof) { + procs.push_back(c.proc.get()); + owners.push_back(&c); + } + } + std::vector ready; + waiter.wait(procs, ready, timeout); + for (size_t i = 0; i < owners.size(); i++) { + if (ready[i]) { + read_output(*owners[i]); + } + } + + // deadlines and exits + now = ggml_time_ms(); + for (auto it = children.begin(); it != children.end();) { + if (it->deadline && now >= it->deadline && !it->proc->stopped.load(std::memory_order_acquire)) { + SRV_WRN("force-killing model instance name=%s after timeout\n", it->name.c_str()); + it->proc->terminate(); + it->deadline = 0; + } + if (it->eof && !it->proc->is_alive()) { + int exit_code = it->proc->join(); + it->proc->stopped.store(true, std::memory_order_release); + models.on_child_exit(it->name, it->proc, it->mode, exit_code); + SRV_INF("instance name=%s exited with status %d\n", it->name.c_str(), exit_code); + it = children.erase(it); + } else { + ++it; + } + } + } + } + + static constexpr size_t max_line = 1024 * 1024; + + server_models & models; + std::mutex mu; + std::deque cmds; + std::vector children; // monitor thread only + server_subproc::waiter waiter; + std::thread th; }; struct server_lru_sched { @@ -395,7 +580,8 @@ server_models::server_models( base_params(params), base_env(get_environment()), base_preset(ctx_preset.load_from_args(argc, argv)), - sched(std::make_unique(*this)) { + sched(std::make_unique(*this)), + monitor(std::make_unique(*this)) { // clean up base preset unset_reserved_args(base_preset, true); // set binary path @@ -412,6 +598,10 @@ server_models::server_models( server_models::~server_models() = default; +void server_models::instance_t::request_exit() const { + request_child_exit(*subproc); +} + void server_models::add_model(server_model_meta && meta) { if (mapping.find(meta.name) != mapping.end()) { throw std::runtime_error(string_format("model '%s' appears multiple times", meta.name.c_str())); @@ -466,7 +656,6 @@ void server_models::add_model(server_model_meta && meta) { std::string name = meta.name; mapping[name] = instance_t{ /* subproc */ std::make_shared(), - /* th */ std::thread(), /* meta */ std::move(meta) }; } @@ -621,9 +810,7 @@ void server_models::load_models() { }; // Phase 2: acquire the lock once for all mapping mutations. - // We temporarily release it only when calling functions that acquire it internally - // (unload, load) or when joining threads (the monitoring thread calls update_status - // which locks the mutex, so joining while holding it would deadlock). + // We temporarily release it only when calling functions that acquire it internally (unload) std::unique_lock lk(mutex); need_reload = false; @@ -708,49 +895,15 @@ void server_models::load_models() { return true; }); - // collect all threads to join in one pass while the lock is held: - // - monitoring threads from just-unloaded models (to_unload) - // - threads of finished downloads (DOWNLOADED), they acquire the mutex on exit - // - threads of already-UNLOADED models that are being removed from source - std::vector threads_to_join; - for (const auto & name : to_unload) { - auto it = mapping.find(name); - if (it != mapping.end() && it->second.th.joinable()) { - threads_to_join.push_back(std::move(it->second.th)); - } - } - for (auto & [name, inst] : mapping) { - if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { - continue; // downloading models are not from config sources, leave them alone - } - if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADED) { - // joining this thread under the lock deadlocks: it locks the mutex on its way out - if (inst.th.joinable()) { - threads_to_join.push_back(std::move(inst.th)); - } - continue; - } - if (final_presets.find(name) == final_presets.end() && !inst.meta.is_running() && inst.th.joinable()) { - threads_to_join.push_back(std::move(inst.th)); - } - } - - // join outside the lock - monitoring thread calls update_status (needs lock) - lk.unlock(); - for (auto & th : threads_to_join) th.join(); - lk.lock(); - // erase models no longer in any source for (auto it = mapping.begin(); it != mapping.end(); ) { if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { ++it; // download thread is still busy, skip } else if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADED) { - // download finished, thread is joined above, safe to erase - GGML_ASSERT(!it->second.th.joinable()); + // download finished, safe to erase it = mapping.erase(it); } else if (final_presets.find(it->first) == final_presets.end()) { SRV_INF("(reload) removing model name=%s (no longer in source)\n", it->first.c_str()); - GGML_ASSERT(!it->second.th.joinable()); // must have been joined above it = mapping.erase(it); } else { ++it; @@ -976,7 +1129,8 @@ void server_models::load(const std::string & name, const load_options & opts) { // exceeding models_max. Without this, the window between unload_lru() // releasing its lock and this lock_guard acquiring allows multiple // threads to each observe capacity and all proceed to load. - if (base_params.models_max > 0) { + // Download workers do not use models_max slots. + if (opts.mode == SERVER_CHILD_MODE_NORMAL && base_params.models_max > 0) { size_t count_active = 0; for (const auto & m : mapping) { if (m.second.meta.is_running()) { @@ -1030,117 +1184,12 @@ void server_models::load(const std::string & name, const load_options & opts) { } } - // start a thread to manage the child process - // captured variables are guaranteed to be destroyed only after the thread is joined - inst.th = std::thread([ - this, name, - child_proc = inst.subproc, - port = inst.meta.port, - stop_timeout = inst.meta.stop_timeout, - child_mode = opts.mode - ]() { - FILE * stdin_file = child_proc->sproc.stdin_file(); - FILE * stdout_file = child_proc->sproc.stdout_file(); // combined stdout/stderr - - std::thread log_thread([&]() { - // read stdout/stderr and forward to main server log - // also handle status report from child process - std::vector vec_buf(128 * 1024); // large buffer for storing info - char * buffer = vec_buf.data(); - if (stdout_file) { - while (fgets(buffer, vec_buf.size(), stdout_file) != nullptr) { - std::string str(buffer); - if (string_starts_with(buffer, CMD_CHILD_TO_ROUTER_STATE)) { - LOG_DBG("[%5d] %s", port, buffer); // prevent spamming the log - this->handle_child_state(name, str); - } else { - // forward log - LOG("[%5d] %s", port, buffer); - } - } - } else { - SRV_ERR("failed to get stdout/stderr of child process for name=%s\n", name.c_str()); - } - }); - - std::thread stopping_thread([&]() { - // thread to monitor explicit stop requests; child crash is signalled via child_proc->stopped - auto is_stopping = [this, &name]() { - return this->stopping_models.find(name) != this->stopping_models.end(); - }; - { - std::unique_lock lk(this->mutex); - this->cv_stop.wait(lk, [&]() { - return is_stopping() || child_proc->stopped.load(std::memory_order_acquire); - }); - } - // child crashed or finished on its own, skip graceful shutdown sequence - if (child_proc->stopped.load(std::memory_order_acquire)) { - return; - } - SRV_INF("stopping model instance name=%s\n", name.c_str()); - fprintf(stdin_file, "%s\n", CMD_ROUTER_TO_CHILD_EXIT); - fflush(stdin_file); - int64_t start_time = ggml_time_ms(); - while (true) { - std::unique_lock lk(this->mutex); - if (!is_stopping() || child_proc->stopped.load(std::memory_order_acquire)) { - return; - } - int64_t elapsed = ggml_time_ms() - start_time; - if (elapsed >= stop_timeout * 1000) { - lk.unlock(); - SRV_WRN("force-killing model instance name=%s after %d seconds timeout\n", name.c_str(), stop_timeout); - child_proc->terminate(); - return; - } - this->cv_stop.wait_for(lk, std::chrono::seconds(1), [&]() { - return !is_stopping() || child_proc->stopped.load(std::memory_order_acquire); - }); - } - }); - - // we reach here when the child process exits (stdout EOF) - // note: we cannot join() prior to this point because it will close stdin_file - if (log_thread.joinable()) { - log_thread.join(); - } - - child_proc->stopped.store(true, std::memory_order_release); - { - std::lock_guard lk(this->mutex); - stopping_models.erase(name); - cv_stop.notify_all(); - } - if (stopping_thread.joinable()) { - stopping_thread.join(); - } - - // get the exit code - int exit_code = child_proc->sproc.join(); - - // update status and exit code - if (child_mode == SERVER_CHILD_MODE_DOWNLOAD) { - // instance will be cleaned up on next load_models() call - } else { - this->update_status(name, { - SERVER_MODEL_STATUS_UNLOADED, - exit_code - }); - } - SRV_INF("instance name=%s exited with status %d\n", name.c_str(), exit_code); - }); - - // clean up old process/thread if exists + // old process should have exited already, but just in case, we clean it up here { - auto & old_instance = mapping[name]; - // old process should have exited already, but just in case, we clean it up here - if (old_instance.subproc && old_instance.subproc->is_alive()) { + auto it = mapping.find(name); + if (it != mapping.end() && it->second.subproc && it->second.subproc->is_alive()) { SRV_WRN("old process for model name=%s is still alive, this is unexpected\n", name.c_str()); - old_instance.subproc->terminate(); // force kill - } - if (old_instance.th.joinable()) { - old_instance.th.join(); + it->second.subproc->terminate(); // force kill } } @@ -1148,13 +1197,41 @@ void server_models::load(const std::string & name, const load_options & opts) { {"status", server_model_status_to_string(inst.meta.status)}, }); + auto proc = inst.subproc; + int port = inst.meta.port; mapping[name] = std::move(inst); + monitor->watch(name, proc, opts.mode, port); cv.notify_all(); } -void server_models::request_stop(const std::string & name) { +void server_models::request_stop(const std::string & name, bool send_exit) { + auto it = mapping.find(name); + if (it == mapping.end() || stopping_models.count(name)) { + return; + } stopping_models.insert(name); - cv_stop.notify_all(); + monitor->stop(name, it->second.meta.stop_timeout, send_exit); +} + +void server_models::on_child_exit(const std::string & name, const std::shared_ptr & proc, server_child_mode mode, int exit_code) { + { + std::lock_guard lk(mutex); + stopping_models.erase(name); + auto it = mapping.find(name); + if (it == mapping.end() || it->second.subproc != proc) { + return; // entry erased, or a newer instance took the name + } + } + if (mode == SERVER_CHILD_MODE_DOWNLOAD) { + // instance will be cleaned up on next load_models() call + std::lock_guard lk(mutex); + cv.notify_all(); + } else { + update_status(name, { + SERVER_MODEL_STATUS_UNLOADED, + exit_code + }); + } } void server_models::unload(const std::string & name) { @@ -1163,20 +1240,21 @@ void server_models::unload(const std::string & name) { if (it != mapping.end()) { if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { SRV_INF("cancelling download for model name=%s\n", name.c_str()); - it->second.subproc->request_exit(); + it->second.request_exit(); // for convenience, we wait the status change here wait(lk, name, [](const server_model_meta & new_meta) { return new_meta.status != SERVER_MODEL_STATUS_DOWNLOADING; }); } else if (it->second.meta.is_running()) { SRV_INF("stopping model instance name=%s\n", name.c_str()); - if (it->second.meta.status == SERVER_MODEL_STATUS_LOADING) { + bool loading = it->second.meta.status == SERVER_MODEL_STATUS_LOADING; + if (loading) { // special case: if model is in loading state, unloading means force-killing it SRV_WRN("model name=%s is still loading, force-killing\n", name.c_str()); it->second.subproc->terminate(); } - request_stop(name); - // status change will be handled by the managing thread + request_stop(name, !loading); + // status change will be handled by the monitor } else { SRV_WRN("model instance name=%s is not running\n", name.c_str()); } @@ -1184,27 +1262,29 @@ void server_models::unload(const std::string & name) { } void server_models::unload_all() { - std::vector to_join; - { - std::lock_guard lk(mutex); - for (auto & [name, inst] : mapping) { - if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { - SRV_INF("cancelling download for model name=%s\n", name.c_str()); - inst.subproc->stopped.store(true, std::memory_order_relaxed); - } else if (inst.meta.is_running()) { - SRV_INF("stopping model instance name=%s\n", name.c_str()); - request_stop(name); - // status change will be handled by the managing thread + std::unique_lock lk(mutex); + for (auto & [name, inst] : mapping) { + if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { + SRV_INF("cancelling download for model name=%s\n", name.c_str()); + inst.request_exit(); + } else if (inst.meta.is_running()) { + SRV_INF("stopping model instance name=%s\n", name.c_str()); + bool loading = inst.meta.status == SERVER_MODEL_STATUS_LOADING; + if (loading) { + inst.subproc->terminate(); } - // moving the thread to join list to avoid deadlock - to_join.push_back(std::move(inst.th)); + request_stop(name, !loading); } } - for (auto & th : to_join) { - if (th.joinable()) { - th.join(); + // wait for every child to exit, the monitor force-kills the ones that ignore the exit command + cv.wait(lk, [this]() { + for (const auto & [name, inst] : mapping) { + if (inst.meta.is_running() || inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { + return false; + } } - } + return true; + }); } void server_models::update_status(const std::string & name, const update_status_args & args) { @@ -1291,18 +1371,18 @@ bool server_models::remove(const std::string & name) { if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { // cancel in-flight download SRV_INF("cancelling download for model name=%s\n", name.c_str()); - it->second.subproc->request_exit(); + it->second.request_exit(); } else if (it->second.meta.is_running()) { // stop running instance SRV_INF("stopping model instance name=%s\n", name.c_str()); - stopping_models.insert(name); - if (it->second.meta.status == SERVER_MODEL_STATUS_LOADING) { + bool loading = it->second.meta.status == SERVER_MODEL_STATUS_LOADING; + if (loading) { it->second.subproc->terminate(); } - cv_stop.notify_all(); + request_stop(name, !loading); } - // wait until the monitoring thread finishes + // wait until the child is gone wait(lk, name, [](const server_model_meta & meta) { return meta.status == SERVER_MODEL_STATUS_UNLOADED || meta.status == SERVER_MODEL_STATUS_DOWNLOADED; @@ -1311,8 +1391,7 @@ bool server_models::remove(const std::string & name) { // re-find after wait - load_models() may have erased the entry during the wait it = mapping.find(name); if (it == mapping.end()) { - // load_models() already joined the thread and erased the entry; - // we just need to clean up the cached files on disk + // load_models() already erased the entry; we just need to clean up the cached files on disk lk.unlock(); bool ok = common_download_remove(name); SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "partial"); @@ -1320,11 +1399,6 @@ bool server_models::remove(const std::string & name) { return true; } - // join before erasing - thread no longer acquires this mutex - if (it->second.th.joinable()) { - it->second.th.join(); - } - // remove from disk (best-effort: cancelled downloads may have no cached files) bool ok = common_download_remove(name); mapping.erase(name); @@ -1539,7 +1613,7 @@ void server_models::handle_child_state(const std::string & name, const std::stri std::lock_guard lk(mutex); auto it = mapping.find(name); if (it != mapping.end()) { - return it->second.subproc->request_exit(); + return it->second.request_exit(); } }; if (result == "download_finished") { @@ -1713,7 +1787,10 @@ void server_child::notify_to_router(const std::string & state, const json & payl std::lock_guard lk(mtx_stdout); common_log_pause(common_log_main()); fflush(stdout); - fprintf(stdout, "%s%s\n", CMD_CHILD_TO_ROUTER_STATE, safe_json_to_str(data).c_str()); + // the router matches the command on a line prefix, so the leading newline + // closes whatever the logger left open on the shared pipe, down to the + // trailing color reset that carries no newline of its own + fprintf(stdout, "\n%s%s\n", CMD_CHILD_TO_ROUTER_STATE, safe_json_to_str(data).c_str()); fflush(stdout); common_log_resume(common_log_main()); } diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 7f6c26b35..90161bf34 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -107,27 +108,29 @@ struct server_model_meta { }; struct server_models_routes; -struct server_subproc; // defined in server-models.cpp struct server_lru_sched; // defined in server-models.cpp +struct server_monitor; // defined in server-models.cpp struct server_models { friend struct server_models_routes; friend struct server_lru_sched; + friend struct server_monitor; private: struct instance_t { - std::shared_ptr subproc; // shared between main thread and monitoring thread - std::thread th; + std::shared_ptr subproc; // shared with the monitor thread server_model_meta meta; int req_count = 0; // number of active proxy requests + + // ask the child to exit (it handles the command on its stdin, see server_child::setup) + void request_exit() const; }; std::mutex mutex; std::condition_variable cv; std::map mapping; - // for stopping models - std::condition_variable cv_stop; + // models asked to stop, still counted as running until the monitor records their exit std::set stopping_models; // set to true while load_models() is executing a reload; load() will wait until clear @@ -216,9 +219,12 @@ private: // not thread-safe, caller must hold mutex void add_model(server_model_meta && meta); - // ask the monitoring thread to stop a running instance + // ask the monitor to stop a running instance; send_exit is false for a child that was already force-killed // not thread-safe, caller must hold mutex - void request_stop(const std::string & name); + void request_stop(const std::string & name, bool send_exit = true); + + // called by the monitor once a child exited and was reaped + void on_child_exit(const std::string & name, const std::shared_ptr & proc, server_child_mode mode, int exit_code); // notify SSE clients void notify_sse(const std::string & event, const std::string & model_id, const json & data = nullptr); @@ -297,12 +303,16 @@ public: // handle message sent from server_child::notify_to_router() // raw input must starts with CMD_CHILD_TO_ROUTER_STATE, followed by a JSON string - // this function is not thread-safe, must be called from instance's monitoring thread + // called from the monitor thread // payload per state: // state = loading -> payload = {} (TODO: add progress info) // state = ready -> payload = model_info (json), or {} if wakeup from sleeping // state = sleeping -> payload = {} void handle_child_state(const std::string & name, const std::string & raw_input); + +private: + // one thread watching every child; keep last, the destructor joins the thread + std::unique_ptr monitor; }; struct server_child { diff --git a/tools/server/tests/unit/test_completion.py b/tools/server/tests/unit/test_completion.py index 9375e0110..01732eb16 100644 --- a/tools/server/tests/unit/test_completion.py +++ b/tools/server/tests/unit/test_completion.py @@ -394,7 +394,12 @@ def test_completion_unified(n_ctx, n_slots, n_predict_vals, expected_success): results = parallel_function_calls(tasks) for res, n_predict, expect_ok in zip(results, n_predict_vals, expected_success): if expect_ok: - assert res.status_code == 200 + # the pool is aborted as a whole, so a request that fits on its own + # is still dropped when the slots overlap, and it says so explicitly + assert res.status_code == 200 or ( + res.status_code == 500 + and "context size has been exceeded" in res.body["error"]["message"].lower() + ) # note: https://github.com/ggml-org/llama.cpp/pull/18700#issuecomment-3728695581 if res.status_code == 200: diff --git a/tools/server/tests/unit/test_router.py b/tools/server/tests/unit/test_router.py index e4b7f9fe4..bae156517 100644 --- a/tools/server/tests/unit/test_router.py +++ b/tools/server/tests/unit/test_router.py @@ -540,13 +540,17 @@ def _wait_for_sse_event(collected: list, event_type: str, model: str, timeout: i def test_router_download_model(): - """Case 1: download a model, verify SSE events and GET /models.""" + """Case 1: download a model at the model limit, verify SSE events and GET /models.""" global server + server.models_max = 1 server.start() # Ensure the model is not present before we start server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}") + # A download worker must not consume or evict a model slot + _load_model_and_wait(MODEL_B, timeout=120) + sse_events: list = [] stop = threading.Event() sse_ready = threading.Event() @@ -580,6 +584,7 @@ def test_router_download_model(): # Model should now appear in GET /models ids = _get_model_ids(is_reload=False) assert MODEL_DOWNLOAD_ID in ids, f"{MODEL_DOWNLOAD_ID} not found in /models after download" + assert _get_model_status(MODEL_B) == "loaded" def test_router_delete_model(): diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp index 7fd10b393..c82ff1e71 100644 --- a/vendor/cpp-httplib/httplib.cpp +++ b/vendor/cpp-httplib/httplib.cpp @@ -912,17 +912,42 @@ bool write_websocket_frame(Stream &strm, ws::Opcode opcode, namespace ws { namespace impl { -bool read_websocket_frame(Stream &strm, Opcode &opcode, - std::string &payload, bool &fin, - bool expect_masked, size_t max_len) { - // Read first 2 bytes +// Read exactly `size` bytes. Stream::read may return less than asked for -- it +// hands back whatever its buffer already holds -- so every multi-byte field has +// to loop. Reading a 2-byte header with a single read() fails whenever the +// header straddles the read buffer's boundary. +// +// Timeout is reported only when nothing at all was consumed. Once a byte has +// been taken the stream sits mid-field and cannot be resumed, so a timeout +// there is a failure like any other. (When read() fails it always records why, +// so the error belongs to this call and not to an earlier one.) +FrameRead read_exact(Stream &strm, void *buf, size_t size) { + auto p = static_cast(buf); + size_t total = 0; + while (total < size) { + auto n = strm.read(p + total, size - total); + if (n <= 0) { + auto timed_out = total == 0 && strm.get_error() == Error::Timeout; + return timed_out ? FrameRead::Timeout : FrameRead::Fail; + } + total += static_cast(n); + } + return FrameRead::Ok; +} + +FrameRead read_websocket_frame(Stream &strm, Opcode &opcode, + std::string &payload, bool &fin, + bool expect_masked, size_t max_len) { + // Read first 2 bytes. This is the only read that may report a timeout: it + // sits on a frame boundary, where nothing has been consumed yet. uint8_t header[2]; - if (strm.read(reinterpret_cast(header), 2) != 2) { return false; } + FrameRead first = read_exact(strm, header, 2); + if (first != FrameRead::Ok) { return first; } fin = (header[0] & 0x80) != 0; // RSV1, RSV2, RSV3 must be 0 when no extension is negotiated - if (header[0] & 0x70) { return false; } + if (header[0] & 0x70) { return FrameRead::Fail; } opcode = static_cast(header[0] & 0x0F); bool masked = (header[1] & 0x80) != 0; @@ -932,46 +957,44 @@ bool read_websocket_frame(Stream &strm, Opcode &opcode, // MUST have a payload length of 125 bytes or less bool is_control = (static_cast(opcode) & 0x08) != 0; if (is_control) { - if (!fin) { return false; } - if (payload_len > 125) { return false; } + if (!fin) { return FrameRead::Fail; } + if (payload_len > 125) { return FrameRead::Fail; } } - if (masked != expect_masked) { return false; } + if (masked != expect_masked) { return FrameRead::Fail; } // Extended payload length if (payload_len == 126) { uint8_t ext[2]; - if (strm.read(reinterpret_cast(ext), 2) != 2) { return false; } + if (read_exact(strm, ext, 2) != FrameRead::Ok) { return FrameRead::Fail; } payload_len = (static_cast(ext[0]) << 8) | ext[1]; } else if (payload_len == 127) { uint8_t ext[8]; - if (strm.read(reinterpret_cast(ext), 8) != 8) { return false; } + if (read_exact(strm, ext, 8) != FrameRead::Ok) { return FrameRead::Fail; } // RFC 6455 Section 5.2: the most significant bit MUST be 0 - if (ext[0] & 0x80) { return false; } + if (ext[0] & 0x80) { return FrameRead::Fail; } payload_len = 0; for (int i = 0; i < 8; i++) { payload_len = (payload_len << 8) | ext[i]; } } - if (payload_len > max_len) { return false; } + if (payload_len > max_len) { return FrameRead::Fail; } // Read mask key if present uint8_t mask_key[4] = {0}; if (masked) { - if (strm.read(reinterpret_cast(mask_key), 4) != 4) { return false; } + if (read_exact(strm, mask_key, 4) != FrameRead::Ok) { + return FrameRead::Fail; + } } // Read payload payload.resize(static_cast(payload_len)); - if (payload_len > 0) { - size_t total_read = 0; - while (total_read < payload_len) { - auto n = strm.read(&payload[total_read], - static_cast(payload_len - total_read)); - if (n <= 0) { return false; } - total_read += static_cast(n); - } + if (payload_len > 0 && + read_exact(strm, &payload[0], static_cast(payload_len)) != + FrameRead::Ok) { + return FrameRead::Fail; } // Unmask if needed @@ -981,7 +1004,7 @@ bool read_websocket_frame(Stream &strm, Opcode &opcode, } } - return true; + return FrameRead::Ok; } } // namespace impl @@ -1728,7 +1751,9 @@ ssize_t select_impl(socket_t sock, short events, time_t sec, pfd.events = events; pfd.revents = 0; - auto timeout = static_cast(sec * 1000 + usec / 1000); + // A negative timeout waits forever, poll's own convention. 0 keeps meaning + // "return immediately", which callers here rely on to probe a socket. + auto timeout = sec < 0 ? -1 : static_cast(sec * 1000 + usec / 1000); return handle_EINTR([&]() { return poll_wrapper(&pfd, 1, timeout); }); } @@ -1810,8 +1835,11 @@ private: bool ensure_readable(); socket_t sock_; - time_t read_timeout_sec_; - time_t read_timeout_usec_; + // Atomic because ws::WebSocket::set_read_timeout() reaches this from another + // thread while a read is in flight -- that is the point of it, for a caller + // holding one connection and wanting control back to send on it. + std::atomic read_timeout_sec_; + std::atomic read_timeout_usec_; time_t write_timeout_sec_; time_t write_timeout_usec_; time_t max_timeout_msec_; @@ -2204,12 +2232,10 @@ int getaddrinfo_with_timeout(const char *node, const char *service, // actually finish before letting the stack frame go. The trade-off is that // a wedged DNS server can hold this thread for the system resolver timeout // (~30s by default) past the caller's connection timeout. - struct gaicb request {}; + struct gaicb request{}; struct gaicb *requests[1] = {&request}; - struct sigevent sevp {}; - struct timespec timeout { - timeout_sec, 0 - }; + struct sigevent sevp{}; + struct timespec timeout{timeout_sec, 0}; request.ar_name = node; request.ar_service = service; @@ -2948,8 +2974,21 @@ EncodingType encoding_type(const Request &req, return best; } +// `content_type` is taken separately because a file-backed response has not +// been given one yet when its coding has to be decided. +EncodingType encoding_type(const Request &req, const Response &res, + const std::string &content_type) { + // The response already names a content coding of its own: a handler serving + // a body it encoded itself (pre-compressed static assets, say), or a mount + // point whose headers name the coding its files are stored in. Applying one + // on top of that would double-encode the body and append a second + // `Content-Encoding` field line. + if (res.has_header("Content-Encoding")) { return EncodingType::None; } + return encoding_type(req, content_type); +} + EncodingType encoding_type(const Request &req, const Response &res) { - return encoding_type(req, res.get_header_value("Content-Type")); + return encoding_type(req, res, res.get_header_value("Content-Type")); } std::unique_ptr make_compressor(EncodingType type) { @@ -3677,6 +3716,17 @@ bool is_chunked_transfer_encoding(const Headers &headers) { return case_ignore::equal(last_coding, "chunked"); } +bool has_conflicting_content_length(const Headers &headers) { + // RFC 9112 §6.3: a message carrying both Transfer-Encoding and a non-zero + // Content-Length is framed ambiguously. The body readers here delimit it by + // the transfer coding and drop Content-Length, while an intermediary may do + // the reverse, so the two disagree on where the body ends and a reused + // connection is desynchronised (request/response smuggling). Content-Length: + // 0 is tolerated for compatibility with existing peers. + return has_header(headers, "Transfer-Encoding") && + get_header_value_u64(headers, "Content-Length", 0, 0) > 0; +} + template bool prepare_content_receiver(T &x, int &status, ContentReceiverWithProgress receiver, @@ -4035,7 +4085,7 @@ void set_file_content_provider(Response &res, return true; }); - res.file_content_encoding_ = encoding; + res.content_coding_ = encoding; } template @@ -4361,13 +4411,20 @@ bool parse_range_header(const std::string &s, Ranges &ranges) try { ssize_t first = -1; if (!lhs.empty()) { - ssize_t v; - auto res = detail::from_chars(lhs.data(), lhs.data() + lhs.size(), v); - if (res.ec == std::errc{}) { first = v; } + // Reject an overflowing first-byte-pos; treating it as absent (-1) + // would turn the range into a suffix range. + auto res = + detail::from_chars(lhs.data(), lhs.data() + lhs.size(), first); + if (res.ec != std::errc{}) { + all_valid_ranges = false; + return; + } } ssize_t last = -1; if (!rhs.empty()) { + // An overflowing last-byte-pos is past any content length, so keeping + // -1 ("remainder", RFC 9110 14.1.2) is correct here. ssize_t v; auto res = detail::from_chars(rhs.data(), rhs.data() + rhs.size(), v); if (res.ec == std::errc{}) { last = v; } @@ -6902,7 +6959,7 @@ void Response::set_content(const char *s, size_t n, auto rng = headers.equal_range("Content-Type"); headers.erase(rng.first, rng.second); set_header("Content-Type", content_type); - file_content_encoding_ = detail::EncodingType::None; + content_coding_ = detail::EncodingType::None; } void Response::set_content(const std::string &s, @@ -6917,7 +6974,7 @@ void Response::set_content(std::string &&s, auto rng = headers.equal_range("Content-Type"); headers.erase(rng.first, rng.second); set_header("Content-Type", content_type); - file_content_encoding_ = detail::EncodingType::None; + content_coding_ = detail::EncodingType::None; } void Response::set_content_provider( @@ -6928,7 +6985,7 @@ void Response::set_content_provider( if (in_length > 0) { content_provider_ = std::move(provider); } content_provider_resource_releaser_ = std::move(resource_releaser); is_chunked_content_provider_ = false; - file_content_encoding_ = detail::EncodingType::None; + content_coding_ = detail::EncodingType::None; } void Response::set_content_provider( @@ -6939,7 +6996,7 @@ void Response::set_content_provider( content_provider_ = detail::ContentProviderAdapter(std::move(provider)); content_provider_resource_releaser_ = std::move(resource_releaser); is_chunked_content_provider_ = false; - file_content_encoding_ = detail::EncodingType::None; + content_coding_ = detail::EncodingType::None; } void Response::set_chunked_content_provider( @@ -6950,7 +7007,7 @@ void Response::set_chunked_content_provider( content_provider_ = detail::ContentProviderAdapter(std::move(provider)); content_provider_resource_releaser_ = std::move(resource_releaser); is_chunked_content_provider_ = true; - file_content_encoding_ = detail::EncodingType::None; + content_coding_ = detail::EncodingType::None; } void Response::set_file_content(const std::string &path, @@ -7991,12 +8048,19 @@ ssize_t WebSocketSSLStream::read(char *ptr, size_t size) { needs_readable || (err.code == tls::ErrorCode::SyscallError && WSAGetLastError() == WSAETIMEDOUT); #endif - if (!needs_readable && err.code != tls::ErrorCode::WantWrite) { return -1; } + if (!needs_readable && err.code != tls::ErrorCode::WantWrite) { + error_ = Error::Read; + return -1; + } if (!(needs_readable ? wait_readable() : wait_writable())) { error_ = Error::Timeout; return -1; } } + // Out of retries. Recording a reason matters: a caller that reads get_error() + // to tell a timeout from a close would otherwise see whatever the previous + // failure left behind (error_ is never cleared on success). + error_ = Error::Read; return -1; } @@ -8653,9 +8717,10 @@ Server::write_content_with_provider(Stream &strm, const Request &req, } } else { if (res.is_chunked_content_provider_) { - auto type = detail::encoding_type(req, res); - - auto compressor = detail::make_compressor(type); + // Use the coding `apply_ranges()` chose when it wrote the headers; + // re-negotiating here would disagree with them, e.g. once a handler's + // own Content-Encoding header suppresses the negotiation. + auto compressor = detail::make_compressor(res.content_coding_); if (!compressor) { compressor = detail::make_unique(); } @@ -8881,7 +8946,8 @@ bool Server::handle_file_request(Request &req, Response &res) { auto encoding = detail::EncodingType::None; if (static_file_compression_) { content_type = content_type_of(); - encoding = static_file_encoding(req, content_type, stat.size()); + encoding = + static_file_encoding(req, res, content_type, stat.size()); } // The ETag names the representation actually sent, so a client that @@ -9296,8 +9362,10 @@ bool Server::dispatch_request(Request &req, Response &res, // the ETag, which has to name the representation actually sent, and // `apply_static_file_compression()` go through this, so the two cannot drift // apart. -detail::EncodingType Server::static_file_encoding( - const Request &req, const std::string &content_type, size_t length) const { +detail::EncodingType +Server::static_file_encoding(const Request &req, const Response &res, + const std::string &content_type, + size_t length) const { if (!static_file_compression_) { return detail::EncodingType::None; } // Nothing to compress, and an empty file already answers with @@ -9322,14 +9390,14 @@ detail::EncodingType Server::static_file_encoding( return detail::EncodingType::None; } - return detail::encoding_type(req, content_type); + return detail::encoding_type(req, res, content_type); } // Compresses a file-backed content provider into `res.body` and takes over the // framing headers. Returns false when the response is left untouched. bool Server::apply_static_file_compression(const Request &req, Response &res) const { - auto type = res.file_content_encoding_; + auto type = res.content_coding_; if (type == detail::EncodingType::None || !res.content_provider_) { return false; } @@ -9353,7 +9421,7 @@ bool Server::apply_static_file_compression(const Request &req, res.content_provider_success_ = true; res.content_provider_ = nullptr; res.content_length_ = 0; - res.file_content_encoding_ = detail::EncodingType::None; + res.content_coding_ = detail::EncodingType::None; res.set_header("Content-Encoding", detail::encoding_name(type)); res.set_header("Vary", "Accept-Encoding"); @@ -9412,6 +9480,7 @@ void Server::apply_ranges(const Request &req, Response &res, if (res.content_provider_) { if (res.is_chunked_content_provider_) { res.set_header("Transfer-Encoding", "chunked"); + res.content_coding_ = type; if (type != detail::EncodingType::None) { res.set_header("Content-Encoding", detail::encoding_name(type)); res.set_header("Vary", "Accept-Encoding"); @@ -9568,8 +9637,8 @@ Server::process_request(Stream &strm, const std::string &remote_addr, // coding is not chunked, which leaves the body length undeterminable. The // latter must not fall through to the "no body" path, or the body bytes are // parsed as the next request on a persistent connection. - if (req.has_header("Transfer-Encoding") && - (req.get_header_value_u64("Content-Length") > 0 || + if (detail::has_conflicting_content_length(req.headers) || + (req.has_header("Transfer-Encoding") && !detail::is_chunked_transfer_encoding(req.headers))) { connection_closed = true; res.status = StatusCode::BadRequest_400; @@ -9734,7 +9803,7 @@ Server::process_request(Stream &strm, const std::string &remote_addr, auto ws_strm = std::unique_ptr(new detail::WebSocketSSLStream( strm.socket(), const_cast(req.ssl), - CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND, 0, + CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND, 0, write_timeout_sec_, write_timeout_usec_)); ws::WebSocket ws(std::move(ws_strm), req, true, websocket_ping_interval_sec_, @@ -9744,7 +9813,8 @@ Server::process_request(Stream &strm, const std::string &remote_addr, } #endif // Use WebSocket-specific read timeout instead of HTTP timeout - strm.set_read_timeout(CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND, 0); + strm.set_read_timeout(CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND, + 0); ws::WebSocket ws(strm, req, true, websocket_ping_interval_sec_, websocket_max_missed_pongs_); entry.handler(req, ws); @@ -9808,7 +9878,7 @@ Server::process_request(Stream &strm, const std::string &remote_addr, detail::set_file_content_provider( res, mm, content_type, - static_file_encoding(req, content_type, mm->size())); + static_file_encoding(req, res, content_type, mm->size())); } } @@ -10228,8 +10298,12 @@ Result ClientImpl::send_(Request &&req) { void ClientImpl::prepare_default_headers(Request &r, bool for_stream, const std::string &ct) { (void)for_stream; - for (const auto &header : default_headers_) { - if (!r.has_header(header.first)) { r.headers.insert(header); } + // Default headers are meant for the origin and may carry its credentials, so + // keep them off the CONNECT request the proxy reads. + if (r.method != "CONNECT") { + for (const auto &header : default_headers_) { + if (!r.has_header(header.first)) { r.headers.insert(header); } + } } // RFC 9110 5.3 recommends sending control data such as Host first, so @@ -10379,6 +10453,17 @@ ClientImpl::open_stream(const std::string &method, const std::string &path, return handle; } + // Same framing check as ClientImpl::process_request(). A HEAD or bodyless + // (204/304) response legitimately carries framing headers with no body. + if (method != "HEAD" && + handle.response->status != StatusCode::NoContent_204 && + handle.response->status != StatusCode::NotModified_304 && + detail::has_conflicting_content_length(handle.response->headers)) { + handle.error = Error::Read; + handle.response.reset(); + return handle; + } + handle.body_reader_.stream = handle.stream_; handle.body_reader_.payload_max_length = payload_max_length_; @@ -10910,24 +10995,24 @@ bool ClientImpl::write_request(Stream &strm, Request &req, } } - if (!basic_auth_password_.empty() || !basic_auth_username_.empty()) { - if (!req.has_header("Authorization")) { + // A CONNECT request is read by the proxy; everything sent through the tunnel + // it opens is read by the origin. Each credential goes only to its own hop. + auto is_connect = req.method == "CONNECT"; + + if (!is_connect && !req.has_header("Authorization")) { + if (!basic_auth_password_.empty() || !basic_auth_username_.empty()) { req.headers.insert(make_basic_authentication_header( basic_auth_username_, basic_auth_password_, false)); - } - } - - if (!bearer_token_auth_token_.empty()) { - if (!req.has_header("Authorization")) { + } else if (!bearer_token_auth_token_.empty()) { req.headers.insert(make_bearer_token_authentication_header( bearer_token_auth_token_, false)); } } - // Proxy-Authorization is only sent when the proxy is actually used for - // this target — otherwise NO_PROXY-matched requests would leak proxy - // credentials directly to the destination server. - if (is_proxy_enabled_for_host(host_)) { + // Proxy-Authorization is only sent when the proxy reads this message — + // otherwise NO_PROXY-matched requests, and requests inside a TLS tunnel, + // would leak proxy credentials to the destination server. + if (is_proxy_enabled_for_host(host_) && (!is_ssl() || is_connect)) { if (!proxy_basic_auth_username_.empty() && !proxy_basic_auth_password_.empty() && !req.has_header("Proxy-Authorization")) { @@ -11323,6 +11408,17 @@ bool ClientImpl::process_request(Stream &strm, Request &req, // Body if ((res.status != StatusCode::NoContent_204) && req.method != "HEAD" && req.method != "CONNECT") { + // Reject ambiguous framing (RFC 9112 §6.3). Unlike a request, a response + // whose final transfer coding is not chunked is not ambiguous: its body + // runs until the server closes the connection, so it is not rejected. + // HEAD/204 are excluded above and a 304 carries no body. + if (res.status != StatusCode::NotModified_304 && + detail::has_conflicting_content_length(res.headers)) { + error = Error::Read; + output_error_log(error, &req); + return false; + } + auto redirect = 300 < res.status && res.status < 400 && res.status != StatusCode::NotModified_304 && follow_location_; @@ -17562,8 +17658,16 @@ ReadResult WebSocket::read(std::string &msg) { std::string payload; bool fin; - if (!impl::read_websocket_frame(strm_, opcode, payload, fin, is_server_, - CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH)) { + impl::FrameRead r = + impl::read_websocket_frame(strm_, opcode, payload, fin, is_server_, + CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH); + // A timeout landed on a frame boundary: the connection is untouched and + // still usable, so hand control back without closing it. That is only + // useful to a caller who asked for the timeout; the compile-time default + // is a backstop against a peer gone quiet, and elapsing it closes the + // connection so a plain `while (ws.read(msg))` loop ends. + if (r == impl::FrameRead::Timeout && read_timeout_set_) { return Timeout; } + if (r != impl::FrameRead::Ok) { closed_ = true; return Fail; } @@ -17600,9 +17704,14 @@ ReadResult WebSocket::read(std::string &msg) { Opcode cont_opcode; std::string cont_payload; bool cont_fin; - if (!impl::read_websocket_frame( + // A timeout is not reportable here: half of a fragmented message is + // already in `msg` and read() has no way to resume it, so it is a + // failure like any other. Timeouts are only ever seen on a message + // boundary. + if (impl::read_websocket_frame( strm_, cont_opcode, cont_payload, cont_fin, is_server_, - CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH)) { + CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH) != + impl::FrameRead::Ok) { closed_ = true; return Fail; } @@ -17696,7 +17805,8 @@ void WebSocket::close(CloseStatus status, const std::string &reason) { Opcode op; std::string resp; bool fin; - while (impl::read_websocket_frame(strm_, op, resp, fin, is_server_, 125)) { + while (impl::read_websocket_frame(strm_, op, resp, fin, is_server_, 125) == + impl::FrameRead::Ok) { if (op == Opcode::Close) { break; } } } @@ -17741,6 +17851,15 @@ const Request &WebSocket::request() const { return req_; } bool WebSocket::is_open() const { return !closed_; } +void WebSocket::set_read_timeout(time_t sec, time_t usec) { + // 0 waits forever here, as it does for SO_RCVTIMEO. The stream waits with + // poll(), where 0 would instead mean "return immediately", so hand it the + // negative poll uses for an unbounded wait. + if (sec == 0 && usec == 0) { sec = -1; } + strm_.set_read_timeout(sec, usec); + read_timeout_set_ = true; +} + // WebSocketClient implementation WebSocketClient::WebSocketClient( const std::string &scheme_host_port_path, const Headers &headers) @@ -17843,6 +17962,16 @@ void WebSocketClient::shutdown_and_close() { bool WebSocketClient::create_stream(std::unique_ptr &strm, Error &error, int &ssl_error, uint64_t &ssl_backend_error) { + // A read timeout of 0 means "wait forever", the way SO_RCVTIMEO reads it. + // The streams wait with poll(), where 0 instead means "return immediately", + // so they are given the negative poll uses for an unbounded wait. + auto unbounded = read_timeout_sec_ == 0 && read_timeout_usec_ == 0; + time_t strm_read_sec = unbounded ? -1 : read_timeout_sec_; + time_t strm_read_usec = unbounded ? 0 : read_timeout_usec_; + // The handshake belongs to establishing the connection, so an unset read + // timeout leaves it bounded by the connection timeout instead of forever. + time_t hs_sec = unbounded ? connection_timeout_sec_ : read_timeout_sec_; + time_t hs_usec = unbounded ? connection_timeout_usec_ : read_timeout_usec_; #ifdef CPPHTTPLIB_SSL_ENABLED if (is_ssl_) { // A plain flag rather than SSLClient::load_certs()'s call_once: connect() @@ -17862,8 +17991,8 @@ bool WebSocketClient::create_stream(std::unique_ptr &strm, detail::ClientTlsSessionError tls_error; if (!detail::setup_client_tls_session(host_, tls_ctx_, tls_session_, sock_, server_certificate_verification_, - read_timeout_sec_, read_timeout_usec_, - &tls_error, options)) { + hs_sec, hs_usec, &tls_error, + options)) { error = tls_error.error; ssl_error = tls_error.ssl_error; ssl_backend_error = tls_error.backend_error; @@ -17871,17 +18000,19 @@ bool WebSocketClient::create_stream(std::unique_ptr &strm, } strm = std::unique_ptr(new detail::WebSocketSSLStream( - sock_, tls_session_, read_timeout_sec_, read_timeout_usec_, - write_timeout_sec_, write_timeout_usec_)); + sock_, tls_session_, strm_read_sec, strm_read_usec, write_timeout_sec_, + write_timeout_usec_)); return true; } #else (void)error; (void)ssl_error; (void)ssl_backend_error; + (void)hs_sec; + (void)hs_usec; #endif strm = std::unique_ptr( - new detail::SocketStream(sock_, read_timeout_sec_, read_timeout_usec_, + new detail::SocketStream(sock_, strm_read_sec, strm_read_usec, write_timeout_sec_, write_timeout_usec_)); return true; } @@ -17951,6 +18082,9 @@ Result WebSocketClient::connect() { ws_ = std::unique_ptr(new WebSocket(std::move(strm), req, false, websocket_ping_interval_sec_, websocket_max_missed_pongs_)); + // The stream was created with the timeout already; tell the WebSocket + // whether it came from the caller, so read() knows to report it as Timeout. + ws_->read_timeout_set_ = read_timeout_set_; return Result{Error::Success, upgrade.status, std::move(upgrade.headers)}; } @@ -17983,6 +18117,10 @@ const std::string &WebSocketClient::subprotocol() const { void WebSocketClient::set_read_timeout(time_t sec, time_t usec) { read_timeout_sec_ = sec; read_timeout_usec_ = usec; + read_timeout_set_ = true; + // The members above only seed the next connect(); read() consults the + // stream, so an already-open connection has to be told directly. + if (ws_) { ws_->set_read_timeout(sec, usec); } } void WebSocketClient::set_write_timeout(time_t sec, time_t usec) { diff --git a/vendor/cpp-httplib/httplib.h b/vendor/cpp-httplib/httplib.h index ca7c96a41..a3a2ff45a 100644 --- a/vendor/cpp-httplib/httplib.h +++ b/vendor/cpp-httplib/httplib.h @@ -8,8 +8,8 @@ #ifndef CPPHTTPLIB_HTTPLIB_H #define CPPHTTPLIB_HTTPLIB_H -#define CPPHTTPLIB_VERSION "0.54.1" -#define CPPHTTPLIB_VERSION_NUM "0x003601" +#define CPPHTTPLIB_VERSION "0.56.0" +#define CPPHTTPLIB_VERSION_NUM "0x003800" #ifdef _WIN32 #if defined(_WIN32_WINNT) && _WIN32_WINNT < 0x0A00 @@ -215,8 +215,36 @@ #define CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH 16777216 #endif -#ifndef CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND -#define CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND 300 +// One macro used to set the read timeout for both sides. They want different +// defaults: a client's read timeout is the caller's own tool (it waits forever +// until asked not to), while a server keeps a ceiling that reclaims a worker +// from a peer that has gone quiet. The old name still works and sets both. +#ifdef CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND +#pragma message( \ + "CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND is deprecated; define " \ + "CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND and/or " \ + "CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND instead") +#ifndef CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND \ + CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND +#endif +#ifndef CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND \ + CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND +#endif +#endif + +// 0 waits forever. A read timeout is how a caller gets control back to send on +// the same connection; it is not a liveness check (that is ping/pong). Only a +// timeout set at runtime through set_read_timeout() is reported as +// ws::Timeout; when one of these compile-time defaults elapses, read() returns +// ws::Fail and closes the connection. +#ifndef CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND 0 +#endif + +#ifndef CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND 300 #endif #ifndef CPPHTTPLIB_WEBSOCKET_CLOSE_TIMEOUT_SECOND @@ -1817,10 +1845,12 @@ struct Response { std::string file_content_path_; std::string file_content_content_type_; - // Content coding chosen for a file-backed content provider, decided once - // where the file is opened so that the ETag and the body cannot disagree. - // `EncodingType::None` for every other kind of response. - detail::EncodingType file_content_encoding_ = detail::EncodingType::None; + // Content coding chosen for the response body, decided once so that the + // headers and the body cannot disagree: where the file is opened for a + // file-backed content provider (keeping the ETag honest), and in + // `apply_ranges()` for a chunked content provider. `EncodingType::None` + // for every other kind of response. + detail::EncodingType content_coding_ = detail::EncodingType::None; }; enum class Error { @@ -2359,6 +2389,7 @@ private: bool parse_request_line(const char *s, Request &req) const; detail::EncodingType static_file_encoding(const Request &req, + const Response &res, const std::string &content_type, size_t length) const; bool apply_static_file_compression(const Request &req, Response &res) const; @@ -3663,6 +3694,9 @@ ssize_t read_socket(socket_t sock, void *ptr, size_t size, int flags); EncodingType encoding_type(const Request &req, const std::string &content_type); +EncodingType encoding_type(const Request &req, const Response &res, + const std::string &content_type); + EncodingType encoding_type(const Request &req, const Response &res); class BufferStream final : public Stream { @@ -4345,7 +4379,11 @@ enum class CloseStatus : uint16_t { InternalError = 1011, }; -enum ReadResult : int { Fail = 0, Text = 1, Binary = 2 }; +// Timeout is returned only when a read timeout was set and it elapsed before +// any byte of a frame arrived: nothing was consumed and the connection is +// still open, so the caller can send on it and read again. `msg` is left +// untouched, so a `while (ws.read(msg))` loop must not treat it as a message. +enum ReadResult : int { Fail = 0, Text = 1, Binary = 2, Timeout = 3 }; // Result of WebSocketClient::connect(). Truthy only when the WebSocket // upgrade handshake fully succeeded. On failure error() identifies the @@ -4405,6 +4443,18 @@ public: const Request &request() const; bool is_open() const; + // Bound how long read() waits before returning Timeout. 0 waits forever. + // A server handler owns its connection's timeout this way; a client sets it + // through WebSocketClient. Safe to call while another thread is in read(). + // + // Only a timeout set here is reported as Timeout. The compile-time default + // (CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND) is a backstop rather + // than a request for control, so when it elapses read() returns Fail and + // closes the connection, and `while (ws.read(msg))` ends as it always has. + void set_read_timeout(time_t sec, time_t usec = 0); + template + void set_read_timeout(const std::chrono::duration &duration); + private: friend class httplib::Server; friend class WebSocketClient; @@ -4440,6 +4490,10 @@ private: int max_missed_pongs_; int unacked_pings_ = 0; std::atomic closed_{false}; + // Set once the caller has bounded read() through set_read_timeout(). Until + // then the timeout in effect is the compile-time default, and elapsing it + // is a failure that closes the connection, not a Timeout. + std::atomic read_timeout_set_{false}; std::mutex write_mutex_; // Owned by whichever thread is parsing frames off strm_. Only one thread // may do so: read_websocket_frame() reads a payload until it has the whole @@ -4527,8 +4581,9 @@ private: bool is_valid_ = false; socket_t sock_ = INVALID_SOCKET; std::unique_ptr ws_; - time_t read_timeout_sec_ = CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND; + time_t read_timeout_sec_ = CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND; time_t read_timeout_usec_ = 0; + bool read_timeout_set_ = false; // see WebSocket::read_timeout_set_ time_t write_timeout_sec_ = CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_SECOND; time_t write_timeout_usec_ = CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_USECOND; time_t websocket_ping_interval_sec_ = @@ -4560,6 +4615,13 @@ private: #endif }; +template +inline void WebSocket::set_read_timeout( + const std::chrono::duration &duration) { + detail::duration_to_sec_and_usec( + duration, [&](time_t sec, time_t usec) { set_read_timeout(sec, usec); }); +} + template inline void WebSocketClient::set_read_timeout( const std::chrono::duration &duration) { @@ -4586,8 +4648,14 @@ namespace impl { bool is_valid_utf8(const std::string &s); -bool read_websocket_frame(Stream &strm, Opcode &opcode, std::string &payload, - bool &fin, bool expect_masked, size_t max_len); +// Three states, because a failure that consumed bytes and one that consumed +// none are not the same thing: the first has left the stream in the middle of +// a frame and the connection cannot be reused, the second can just be retried. +enum class FrameRead { Ok, Fail, Timeout }; + +FrameRead read_websocket_frame(Stream &strm, Opcode &opcode, + std::string &payload, bool &fin, + bool expect_masked, size_t max_len); } // namespace impl