Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-13 18:28:51 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge commit 'ae9afff8d2c012ca760eb9c2adf41961cf6f6232' into concedo_experimental

Browse Source 
# Conflicts:
#	.github/workflows/build-openvino.yml
#	.github/workflows/build-self-hosted.yml
#	.github/workflows/build-vulkan.yml
#	.github/workflows/build-wasm.yml
#	.github/workflows/build-webgpu.yml
#	.github/workflows/server-sanitize.yml
#	CMakeLists.txt
#	ci/run.sh
#	common/CMakeLists.txt
#	docs/backend/snapdragon/README.md
#	docs/backend/snapdragon/developer.md
#	docs/build.md
#	examples/speculative-simple/speculative-simple.cpp
#	ggml/src/ggml-cpu/CMakeLists.txt
#	ggml/src/ggml-hexagon/ggml-hexagon.cpp
#	ggml/src/ggml-hexagon/htp-opnode.h
#	ggml/src/ggml-hexagon/htp/act-ops.c
#	ggml/src/ggml-hexagon/htp/allreduce-ops.c
#	ggml/src/ggml-hexagon/htp/allreduce-ops.h
#	ggml/src/ggml-hexagon/htp/argsort-ops.c
#	ggml/src/ggml-hexagon/htp/binary-ops.c
#	ggml/src/ggml-hexagon/htp/concat-ops.c
#	ggml/src/ggml-hexagon/htp/cpy-ops.c
#	ggml/src/ggml-hexagon/htp/cumsum-ops.c
#	ggml/src/ggml-hexagon/htp/diag-ops.c
#	ggml/src/ggml-hexagon/htp/fill-ops.c
#	ggml/src/ggml-hexagon/htp/flash-attn-ops.c
#	ggml/src/ggml-hexagon/htp/flash-attn-ops.h
#	ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c
#	ggml/src/ggml-hexagon/htp/get-rows-ops.c
#	ggml/src/ggml-hexagon/htp/hex-common.h
#	ggml/src/ggml-hexagon/htp/hex-utils.h
#	ggml/src/ggml-hexagon/htp/hmx-utils.h
#	ggml/src/ggml-hexagon/htp/htp-ctx.h
#	ggml/src/ggml-hexagon/htp/htp-ops.h
#	ggml/src/ggml-hexagon/htp/htp-tensor.c
#	ggml/src/ggml-hexagon/htp/htp-tensor.h
#	ggml/src/ggml-hexagon/htp/hvx-arith.h
#	ggml/src/ggml-hexagon/htp/hvx-div.h
#	ggml/src/ggml-hexagon/htp/hvx-inverse.h
#	ggml/src/ggml-hexagon/htp/hvx-scale.h
#	ggml/src/ggml-hexagon/htp/hvx-sigmoid.h
#	ggml/src/ggml-hexagon/htp/im2col-ops.c
#	ggml/src/ggml-hexagon/htp/main.c
#	ggml/src/ggml-hexagon/htp/matmul-ops.c
#	ggml/src/ggml-hexagon/htp/pad-ops.c
#	ggml/src/ggml-hexagon/htp/repeat-ops.c
#	ggml/src/ggml-hexagon/htp/rope-ops.c
#	ggml/src/ggml-hexagon/htp/set-rows-ops.c
#	ggml/src/ggml-hexagon/htp/softmax-ops.c
#	ggml/src/ggml-hexagon/htp/solve-tri-ops.c
#	ggml/src/ggml-hexagon/htp/ssm-conv.c
#	ggml/src/ggml-hexagon/htp/sum-rows-ops.c
#	ggml/src/ggml-hexagon/htp/unary-ops.c
#	ggml/src/ggml-metal/CMakeLists.txt
#	ggml/src/ggml-opencl/CMakeLists.txt
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	ggml/src/ggml-rpc/CMakeLists.txt
#	ggml/src/ggml-sycl/ggml-sycl.cpp
#	ggml/src/ggml-webgpu/CMakeLists.txt
#	ggml/src/ggml-webgpu/ggml-webgpu.cpp
#	scripts/snapdragon/run.py
#	scripts/sync_vendor.py
#	scripts/ui-assets.cmake
#	src/CMakeLists.txt
#	src/models/gemma4.cpp
#	tests/.gitignore
#	tests/CMakeLists.txt
#	tests/test-backend-ops.cpp
#	tests/test-jinja.cpp
#	tests/test-llama-archs.cpp
#	tests/test-quantize-fns.cpp
#	tests/test-save-load-state.cpp
#	tools/mtmd/CMakeLists.txt
#	tools/server/CMakeLists.txt
This commit is contained in:
Concedo
2026-09-13 23:53:41 +08:00
parent 7610e171a8 ae9afff8d2
commit 1c3cbac085
46 changed files with 2507 additions and 759 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+4 -1
View File
@@ -327,7 +327,7 @@ ifdef LLAMA_METAL
CFLAGS += -DGGML_USE_METAL -DGGML_METAL_NDEBUG
CXXFLAGS += -DGGML_USE_METAL
LDFLAGS += -framework Foundation -framework Metal -framework MetalKit -framework MetalPerformanceShaders
OBJS += ggml-metal.o ggml-metal-device.o ggml-metal-device-m.o ggml-metal-context-m.o ggml-metal-common.o ggml-metal-ops.o ggml-metal-tuning.o
OBJS += ggml-metal.o ggml-metal-device.o ggml-metal-device-m.o ggml-metal-context-m.o ggml-metal-common.o ggml-metal-ops.o ggml-metal-tuning.o ggml-metal-fusion.o
ggml-metal-common.o: ggml/src/ggml-metal/ggml-metal-common.cpp ggml/src/ggml-metal/ggml-metal-common.h
$(CXX) $(CXXFLAGS) -c $< -o $@
@@ -338,6 +338,9 @@ ggml-metal-ops.o: ggml/src/ggml-metal/ggml-metal-ops.cpp ggml/src/ggml-metal/ggm
ggml-metal-tuning.o: ggml/src/ggml-metal/ggml-metal-tuning.cpp ggml/src/ggml-metal/ggml-metal-tuning.h
$(CXX) $(CXXFLAGS) -c $< -o $@
ggml-metal-fusion.o: ggml/src/ggml-metal/ggml-metal-fusion.cpp ggml/src/ggml-metal/ggml-metal-fusion.h
$(CXX) $(CXXFLAGS) -c $< -o $@
ggml-metal.o: ggml/src/ggml-metal/ggml-metal.cpp
$(CXX) $(CXXFLAGS) -c $< -o $@
common/jinja/runtime.cpp
+6
View File
@@ -845,6 +845,12 @@ value member_expression::execute_impl(context & ctx) {
} else {
property = this->property->execute(ctx);
}
} else if (is_stmt<integer_literal>(this->property)) {
// syntax: obj.index
property = mk_val<value_int>(cast_stmt<integer_literal>(this->property)->val);
if (property->as_int() < 0) {
throw std::runtime_error("Static member property cannot be negative");
}
} else {
// syntax: obj.prop
if (!is_stmt<identifier>(this->property)) {
common/speculative.cpp
+11 -11
View File
@@ -296,7 +296,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl {
drafting[seq_id] = true;
common_sampler_reset(smpls[seq_id].get());
common_batch_add(batch, dp.id_last, dp.n_past, { seq_id }, true);
common_batch_add(batch, dp.id_last, dp.pos0, { seq_id }, true);
}
int ret = llama_decode(ctx_dft, batch);
@@ -355,7 +355,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl {
continue;
}
common_batch_add(batch, id, dp.n_past + i + 1, { seq_id }, true);
common_batch_add(batch, id, dp.pos0 + i + 1, { seq_id }, true);
}
if (batch.n_tokens == 0) {
@@ -1197,7 +1197,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
common_sampler_reset(smpls[seq_id].get());
const int32_t n = (int32_t) dp.n_past;
const int32_t n = (int32_t) dp.pos0;
const int32_t n_draft = params.n_max;
@@ -1493,7 +1493,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
const int32_t n_tokens = batch_in.n_tokens;
// remember the frist and last batch index for each sequence
// remember the first and last batch index for each sequence
std::fill(i_batch_beg.begin(), i_batch_beg.end(), -1);
std::fill(i_batch_end.begin(), i_batch_end.end(), -1);
@@ -1621,7 +1621,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
drafting[seq_id] = true;
common_sampler_reset(smpls[seq_id].get());
common_batch_add(batch, dp.id_last, dp.n_past, { seq_id }, true);
common_batch_add(batch, dp.id_last, dp.pos0, { seq_id }, true);
std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, pending_h[seq_id].data(), row_bytes);
i_last[seq_id] = batch.n_tokens - 1;
@@ -1635,16 +1635,16 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
while (n_drafting > 0) {
// each step decodes under a different head, i.e. a different decoder layer, and
// KV is per layer. process() filled this layer's KV only for positions < n_past
// KV is per layer. process() filled this layer's KV only for positions < pos0
// (prompt + accepted prefix) — nothing in the draft region yet. so reset the
// draft region (the seq_rm lower bound is n_past, leaving the prompt KV intact)
// draft region (the seq_rm lower bound is pos0, leaving the prompt KV intact)
// and select head i so it rebuilds its own layer's KV there; decoding just the
// latest token would leave its attention reading cells only another head wrote.
if (chain_heads) {
auto * mem_dft = llama_get_memory(ctx_dft);
for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) {
if (drafting[seq_id]) {
llama_memory_seq_rm(mem_dft, seq_id, dparams[seq_id].n_past, -1);
llama_memory_seq_rm(mem_dft, seq_id, dparams[seq_id].pos0, -1);
}
}
llama_set_nextn_layer_offset(ctx_dft, i);
@@ -1710,17 +1710,17 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
const int n_rows = (int) result.size() + 1; // id_last + tokens drafted so far
for (int t = 0; t < n_rows; ++t) {
const llama_token tok = (t == 0) ? dp.id_last : result[t - 1];
common_batch_add(batch, tok, dp.n_past + t, { seq_id }, t == n_rows - 1);
common_batch_add(batch, tok, dp.pos0 + t, { seq_id }, t == n_rows - 1);
std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd,
chain_h[seq_id].data() + (size_t) t * n_embd, row_bytes);
}
} else if (is_mem_shared) {
// note: with shared memory (e.g. Gemma4 assistants) we use the same position for all draft tokens
// ref: https://github.com/huggingface/transformers/blob/effde20942e3f82a1b97449f60b3a48c5ff96145/docs/source/en/model_doc/gemma4_assistant.md?plain=1#L36-L37
common_batch_add(batch, id, dp.n_past, { seq_id }, true);
common_batch_add(batch, id, dp.pos0, { seq_id }, true);
std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, h_row, row_bytes);
} else {
common_batch_add(batch, id, dp.n_past + i + 1, { seq_id }, true);
common_batch_add(batch, id, dp.pos0 + i + 1, { seq_id }, true);
std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, h_row, row_bytes);
}
common/speculative.h
+1 -1
View File
@@ -61,7 +61,7 @@ struct common_speculative_draft_params {
// can be used to constraint the max draft based on the remaining context size
int32_t n_max = -1;
llama_pos n_past;
llama_pos pos0;
llama_token id_last;
// TODO: remove in the future by keeping track of the prompt from the _begin() call and the consecutive accept calls
ggml/src/ggml-backend.cpp
+4
View File
@@ -1712,6 +1712,10 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
ggml_tensor * ids_tensor = node->src[2];
ggml_backend_t ids_backend = split_backend;
if (ggml_nelements(ids_tensor) == 0) {
continue;
}
// if the ids tensor is also an input of the split, it may not have been copied yet to the split backend
// in that case, we use the original ids tensor
for (int i = input_id + 1; i < split->n_inputs; i++) {
ggml/src/ggml-cpu/ops.h
+8
View File
@@ -18,7 +18,15 @@
#endif
#endif
// -Winterference-size was introduced in GCC 12
#if defined(__cplusplus) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Winterference-size"
#endif
static const size_t CACHE_LINE_SIZE_F32 = CACHE_LINE_SIZE/sizeof(float);
#if defined(__cplusplus) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12
#pragma GCC diagnostic pop
#endif
// Work buffer size for im2col operations in CONV2D
#define GGML_IM2COL_WORK_SIZE (16 * 1024 * 1024)
ggml/src/ggml-cuda/fattn-common.cuh
+14 -5
View File
@@ -1129,12 +1129,21 @@ void launch_fattn(
dim3 blocks_num;
if (stream_k) {
// For short contexts it can be faster to have the SMs work on whole tiles because this lets us skip the fixup.
const int max_blocks = max_blocks_per_sm*nsm;
const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks;
const int tiles_efficiency_percent = 100 * ntiles_dst / (max_blocks*tiles_nwaves);
auto should_use_stream_k = [](const int cc, const int ntiles_dst, const int max_blocks, const int DKQ) {
const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks;
const int tiles_efficiency_percent = 100 * ntiles_dst / (max_blocks*tiles_nwaves);
const bool use_stream_k = cc >= GGML_CUDA_CC_ADA_LOVELACE || amd_wmma_available(cc) || tiles_efficiency_percent < 75;
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_ADA_LOVELACE) {
return true;
}
if (amd_wmma_available(cc) && DKQ == 64) {
return true; // TODO better configuration
}
return tiles_efficiency_percent < 75;
};
const int max_blocks = max_blocks_per_sm*nsm;
const bool use_stream_k = should_use_stream_k(cc, ntiles_dst, max_blocks, Q->ne[0]);
blocks_num.x = ntiles_dst;
blocks_num.y = 1;
ggml/src/ggml-cuda/fattn-mma-f16.cuh
+3 -3
View File
@@ -158,8 +158,8 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co
GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 8, 64, 2, 32, 128, 128, 128, 1, true);
GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 64, 2, 32, 128, 128, 128, 1, true);
GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2, 64, 128, 128, 64, 1, true);
GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2, 64, 128, 128, 64, 1, true);
GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 256, 2, 64, 128, 128, 64, 1, true);
GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 256, 2, 64, 128, 128, 64, 1, true);
GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 32, 128, 2, 32, 160, 128, 128, 1, true);
GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 64, 128, 2, 32, 160, 128, 128, 1, true);
@@ -1826,7 +1826,7 @@ static __global__ void flash_attn_ext_f16(
#endif // __CUDA_ARCH__ == GGML_CUDA_CC_TURING
#if defined(AMD_WMMA_AVAILABLE)
if (ncols1*ncols2 < 16 || ncols2 == 1 || DKQ > 128) {
if (ncols1*ncols2 < 16 || ncols2 == 1 || DKQ > 256) {
NO_DEVICE_CODE;
return;
}
ggml/src/ggml-cuda/fattn.cu
+21 -2
View File
@@ -279,6 +279,24 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2(ggml_backend_cuda_con
}
}
// On RDNA it is preferable to minimize wasted compute vs. duplicate I/O for the mask.
if (amd_wmma_available(cc)) {
if (use_gqa_opt && gqa_ratio % 8 == 0) {
ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<DKQ, DV, 8>(ctx, dst);
return;
}
if (use_gqa_opt && gqa_ratio % 4 == 0) {
ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<DKQ, DV, 4>(ctx, dst);
return;
}
if (use_gqa_opt && gqa_ratio % 2 == 0) {
ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<DKQ, DV, 2>(ctx, dst);
return;
}
}
if (use_gqa_opt && gqa_ratio > 4) {
ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<DKQ, DV, 8>(ctx, dst);
return;
@@ -704,8 +722,9 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
}
}
// AMD WMMA is always faster than the tile kernel if the full tile width of 16 can be utilized.
if ((amd_wmma_available(cc) && gqa_opt_applies && Q->ne[0] <= 128) && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[1] * gqa_ratio_eff > 8) {
// AMD WMMA is faster than the tile kernel if the wide tiles with high arithmetic intensity can be utilized.
if ((amd_wmma_available(cc) && gqa_opt_applies && Q->ne[0] <= 256) && Q->ne[0] != 40 && Q->ne[0] != 72 &&
Q->ne[1] * gqa_ratio_eff > (Q->ne[0] <= 128 ? 8 : 16)) {
return BEST_FATTN_KERNEL_MMA_F16;
}
ggml/src/ggml-cuda/mmq-config-gcn.cuh
+281
View File
@@ -0,0 +1,281 @@
static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_gcn(ggml_type type, int J, bool fallback) {
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q5_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q5_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q5_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q5_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q5_1, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q5_1, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q5_1, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q5_1, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q8_0, 512, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
// ---------------------------------------------------------------------------------------------
CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q3_K, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q3_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q3_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q3_K, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q3_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q3_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q3_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q4_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q4_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q5_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q5_K, 256, 3, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q5_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q5_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
// ---------------------------------------------------------------------------------------------
CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
// ---------------------------------------------------------------------------------------------
CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_MXFP4, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_MXFP4, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_MXFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_MXFP4, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_MXFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_MXFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_MXFP4, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_NVFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);
}
ggml/src/ggml-cuda/mmq.cuh
+7 -1
View File
@@ -219,6 +219,7 @@ struct ggml_cuda_mmq_config {
#include "mmq-config-ampere.cuh"
#include "mmq-config-blackwell.cuh"
#include "mmq-config-gcn.cuh"
#include "mmq-config-cdna.cuh"
#include "mmq-config-rdna2.cuh"
#include "mmq-config-rdna3.cuh"
@@ -229,6 +230,9 @@ struct ggml_cuda_mmq_config {
static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type type, const int J, const bool fallback, const int cc) {
if (GGML_CUDA_CC_IS_AMD(cc)) {
if (GGML_CUDA_CC_IS_GCN(cc)) {
return ggml_cuda_mmq_get_config_gcn(type, J, fallback);
}
if (GGML_CUDA_CC_IS_CDNA(cc)) {
return ggml_cuda_mmq_get_config_cdna(type, J, fallback);
}
@@ -257,7 +261,9 @@ static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type ty
static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback) {
#ifdef GGML_USE_HIP
#ifdef CDNA
#ifdef GCN
return ggml_cuda_mmq_get_config_gcn(type, J, fallback);
#elif defined(CDNA)
return ggml_cuda_mmq_get_config_cdna(type, J, fallback);
#elif defined(RDNA4)
return ggml_cuda_mmq_get_config_rdna4(type, J, fallback);
ggml/src/ggml-metal/ggml-metal-common.cpp
+10 -37
View File
@@ -1,4 +1,5 @@
#include "ggml-metal-common.h"
#include "ggml-metal-fusion.h"
#include "ggml.h"
#include "ggml-impl.h"
@@ -390,59 +391,31 @@ static std::vector<int> ggml_metal_graph_optimize_reorder(const std::vector<node
}
void ggml_graph_optimize(ggml_cgraph * gf) {
constexpr int MAX_FUSE = 16;
const int n = gf->n_nodes;
enum ggml_op ops[MAX_FUSE];
std::vector<node_info> nodes;
nodes.reserve(gf->n_nodes);
// fuse nodes:
// we don't want to make reorders that break fusing, so we first pack all fusable tensors
// and perform the reorder over the fused nodes. after the reorder is done, we unfuse
//
// the fusable sequences are declared in the fusion table (ggml-metal-fuse.cpp), so the
// packing here is driven by the same patterns that the op encoders will later use
for (int i = 0; i < n; i++) {
node_info node = {
/*.node =*/ gf->nodes[i],
/*.fused =*/ {},
};
// fuse only ops that start with these operations
// can be expanded when needed
if (node.op() == GGML_OP_ADD ||
node.op() == GGML_OP_NORM ||
node.op() == GGML_OP_RMS_NORM) {
ops[0] = node.op();
const int f = ggml_metal_fusion_max(gf, i);
int f = i + 1;
while (f < n && f < i + MAX_FUSE) {
// conservatively allow fusing only these ops
// can be expanded when needed
if (gf->nodes[f]->op != GGML_OP_ADD &&
gf->nodes[f]->op != GGML_OP_MUL &&
gf->nodes[f]->op != GGML_OP_NORM &&
gf->nodes[f]->op != GGML_OP_RMS_NORM) {
break;
}
ops[f - i] = gf->nodes[f]->op;
f++;
}
// add the fused tensors into the node info so we can unfuse them later
for (int k = 1; k < f; k++) {
++i;
f -= i;
for (; f > 1; f--) {
if (ggml_can_fuse(gf, i, ops, f)) {
break;
}
}
// add the fused tensors into the node info so we can unfuse them later
for (int k = 1; k < f; k++) {
++i;
// the .dst() becomes the last fused tensor
node.add_fused(gf->nodes[i]);
}
// the .dst() becomes the last fused tensor
node.add_fused(gf->nodes[i]);
}
nodes.push_back(std::move(node));
ggml/src/ggml-metal/ggml-metal-context.h
+1
View File
@@ -33,6 +33,7 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx);
void ggml_metal_set_n_cb (ggml_metal_t ctx, int n_cb);
void ggml_metal_set_abort_callback (ggml_metal_t ctx, ggml_abort_callback abort_callback, void * user_data);
bool ggml_metal_supports_family (ggml_metal_t ctx, int family);
void ggml_metal_capture_next_compute(ggml_metal_t ctx);
ggml/src/ggml-metal/ggml-metal-context.m
+33 -22
View File
@@ -6,6 +6,7 @@
#import "ggml-metal-impl.h"
#import "ggml-metal-common.h"
#import "ggml-metal-ops.h"
#import "ggml-metal-fusion.h"
#import <Foundation/Foundation.h>
@@ -36,15 +37,12 @@ struct ggml_metal {
// additional, inference-time compiled pipelines
ggml_metal_pipelines_t pipelines_ext;
bool use_fusion;
bool use_concurrency;
bool use_graph_optimize;
int debug_graph;
int debug_fusion;
// how many times a given op was fused
uint64_t fuse_cnt[GGML_OP_COUNT];
struct ggml_metal_fusion_info * finfo;
// capture state
int capture_compute;
@@ -139,7 +137,6 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) {
res->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT);
res->use_fusion = getenv("GGML_METAL_FUSION_DISABLE") == nil;
res->use_concurrency = getenv("GGML_METAL_CONCURRENCY_DISABLE") == nil;
{
@@ -147,20 +144,19 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) {
res->debug_graph = val ? atoi(val) : 0;
}
{
const char * val = getenv("GGML_METAL_FUSION_DEBUG");
res->debug_fusion = val ? atoi(val) : 0;
}
res->use_graph_optimize = true;
if (getenv("GGML_METAL_GRAPH_OPTIMIZE_DISABLE") != NULL) {
res->use_graph_optimize = false;
}
memset(res->fuse_cnt, 0, sizeof(res->fuse_cnt));
res->finfo = ggml_metal_device_get_fusion_info(dev);
if (ggml_metal_fusion_info_stats(res->finfo)) {
ggml_metal_fusion_info_labels_init(res->finfo);
res->n_cb = 0;
}
GGML_LOG_INFO("%s: use fusion = %s\n", __func__, res->use_fusion ? "true" : "false");
GGML_LOG_INFO("%s: use fusion = %s\n", __func__, ggml_metal_fusion_info_enabled(res->finfo) ? "true" : "false");
GGML_LOG_INFO("%s: use concurrency = %s\n", __func__, res->use_concurrency ? "true" : "false");
GGML_LOG_INFO("%s: use graph optimize = %s\n", __func__, res->use_graph_optimize ? "true" : "false");
@@ -222,15 +218,18 @@ void ggml_metal_free(ggml_metal_t ctx) {
ctx->pipelines_ext = nil;
}
if (ctx->debug_fusion > 0) {
if (ggml_metal_fusion_info_debug(ctx->finfo) > 0) {
GGML_LOG_DEBUG("%s: fusion stats:\n", __func__);
for (int i = 0; i < GGML_OP_COUNT; i++) {
if (ctx->fuse_cnt[i] == 0) {
const int n_fusions = ggml_metal_fusion_info_n_fusions(ctx->finfo);
for (int i = 0; i < n_fusions; i++) {
const uint64_t count = ggml_metal_fusion_info_count(ctx->finfo, i);
if (count == 0) {
continue;
}
// note: cannot use ggml_log here
GGML_LOG_DEBUG("%s: - %s: %" PRIu64 "\n", __func__, ggml_op_name((enum ggml_op) i), ctx->fuse_cnt[i]);
GGML_LOG_DEBUG("%s: - %s: %" PRIu64 "\n", __func__, ggml_metal_fusion_info_label(ctx->finfo, i), count);
}
}
@@ -481,10 +480,17 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph *
@autoreleasepool {
ctx->gf = gf;
ctx->n_nodes_0 = MIN(n_main, gf->n_nodes);
ctx->n_nodes_1 = gf->n_nodes - ctx->n_nodes_0;
if (ctx->n_cb == 0) {
// single-threaded encoding: the whole graph is encoded by one command buffer
ctx->n_nodes_0 = gf->n_nodes;
ctx->n_nodes_1 = 0;
ctx->n_nodes_per_cb = 0;
} else {
ctx->n_nodes_0 = MIN(n_main, gf->n_nodes);
ctx->n_nodes_1 = gf->n_nodes - ctx->n_nodes_0;
ctx->n_nodes_per_cb = (ctx->n_nodes_1 + ctx->n_cb - 1) / ctx->n_cb;
ctx->n_nodes_per_cb = (ctx->n_nodes_1 + ctx->n_cb - 1) / ctx->n_cb;
}
if (ctx->capture_compute >= 0) {
ctx->capture_compute--;
@@ -682,6 +688,12 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx) {
}
void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) {
// when fusion stats are collected the graph must be encoded by a single thread so the
// counters are race-free; override whatever the caller requested
if (ggml_metal_fusion_info_stats(ctx->finfo)) {
n_cb = 0;
}
if (ctx->n_cb != n_cb) {
ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_COMMAND_BUFFERS);
@@ -717,13 +729,12 @@ void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) {
ctx->dev,
cmd_buf,
ctx->gf,
ctx->finfo,
idx_start,
idx_end,
ctx->use_fusion,
ctx->use_concurrency,
ctx->capture_compute,
ctx->debug_graph,
ctx->debug_fusion);
ctx->debug_graph);
for (int idx = 0; idx < ggml_metal_op_n_nodes(ctx_op); ++idx) {
const int res = ggml_metal_op_encode(ctx_op, idx);
ggml/src/ggml-metal/ggml-metal-device.cpp
+72
View File
@@ -932,12 +932,24 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta
nsg = N_SG_IQ2_XXS;
nr0 = N_R0_IQ2_XXS;
smem = 256*8+128;
const int nb32 = ne00/32;
if (nb32 < 32 && (32 % nb32) == 0) {
nr0 = N_R0_IQ2_XXS_SPLIT;
split = true;
}
} break;
case GGML_TYPE_IQ2_XS:
{
nsg = N_SG_IQ2_XS;
nr0 = N_R0_IQ2_XS;
smem = 512*8+128;
const int nb32 = ne00/32;
if (nb32 < 32 && (32 % nb32) == 0) {
nr0 = N_R0_IQ2_XS_SPLIT;
split = true;
}
} break;
case GGML_TYPE_IQ3_XXS:
{
@@ -957,21 +969,45 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta
nsg = N_SG_IQ3_S;
nr0 = N_R0_IQ3_S;
smem = 512*4;
const int nb32 = ne00/32;
if (nb32 < 32 && (32 % nb32) == 0) {
nr0 = N_R0_IQ3_S_SPLIT;
split = true;
}
} break;
case GGML_TYPE_IQ2_S:
{
nsg = N_SG_IQ2_S;
nr0 = N_R0_IQ2_S;
const int nb32 = ne00/32;
if (nb32 < 32 && (32 % nb32) == 0) {
nr0 = N_R0_IQ2_S_SPLIT;
split = true;
}
} break;
case GGML_TYPE_IQ1_S:
{
nsg = N_SG_IQ1_S;
nr0 = N_R0_IQ1_S;
const int nb32 = ne00/32;
if (nb32 < 32 && (32 % nb32) == 0) {
nr0 = N_R0_IQ1_S_SPLIT;
split = true;
}
} break;
case GGML_TYPE_IQ1_M:
{
nsg = N_SG_IQ1_M;
nr0 = N_R0_IQ1_M;
const int nb32 = ne00/32;
if (nb32 < 32 && (32 % nb32) == 0) {
nr0 = N_R0_IQ1_M_SPLIT;
split = true;
}
} break;
case GGML_TYPE_IQ4_NL:
{
@@ -1177,12 +1213,24 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m
nsg = N_SG_IQ2_XXS;
nr0 = N_R0_IQ2_XXS;
smem = 256*8+128;
const int nb32 = ne00/32;
if (nb32 < 32 && (32 % nb32) == 0) {
nr0 = N_R0_IQ2_XXS_SPLIT;
split = true;
}
} break;
case GGML_TYPE_IQ2_XS:
{
nsg = N_SG_IQ2_XS;
nr0 = N_R0_IQ2_XS;
smem = 512*8+128;
const int nb32 = ne00/32;
if (nb32 < 32 && (32 % nb32) == 0) {
nr0 = N_R0_IQ2_XS_SPLIT;
split = true;
}
} break;
case GGML_TYPE_IQ3_XXS:
{
@@ -1202,21 +1250,45 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m
nsg = N_SG_IQ3_S;
nr0 = N_R0_IQ3_S;
smem = 512*4;
const int nb32 = ne00/32;
if (nb32 < 32 && (32 % nb32) == 0) {
nr0 = N_R0_IQ3_S_SPLIT;
split = true;
}
} break;
case GGML_TYPE_IQ2_S:
{
nsg = N_SG_IQ2_S;
nr0 = N_R0_IQ2_S;
const int nb32 = ne00/32;
if (nb32 < 32 && (32 % nb32) == 0) {
nr0 = N_R0_IQ2_S_SPLIT;
split = true;
}
} break;
case GGML_TYPE_IQ1_S:
{
nsg = N_SG_IQ1_S;
nr0 = N_R0_IQ1_S;
const int nb32 = ne00/32;
if (nb32 < 32 && (32 % nb32) == 0) {
nr0 = N_R0_IQ1_S_SPLIT;
split = true;
}
} break;
case GGML_TYPE_IQ1_M:
{
nsg = N_SG_IQ1_M;
nr0 = N_R0_IQ1_M;
const int nb32 = ne00/32;
if (nb32 < 32 && (32 % nb32) == 0) {
nr0 = N_R0_IQ1_M_SPLIT;
split = true;
}
} break;
case GGML_TYPE_IQ4_NL:
{
ggml/src/ggml-metal/ggml-metal-device.h
+5
View File
@@ -325,6 +325,11 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
const struct ggml_metal_device_props * ggml_metal_device_get_props(ggml_metal_device_t dev);
struct ggml_metal_fusion_info;
// the device-owned fusion debugging context (NULL unless fusion debugging is enabled)
struct ggml_metal_fusion_info * ggml_metal_device_get_fusion_info(ggml_metal_device_t dev);
//
// device buffers
//
ggml/src/ggml-metal/ggml-metal-device.m
+17
View File
@@ -1,4 +1,5 @@
#import "ggml-metal-device.h"
#import "ggml-metal-fusion.h"
#import "ggml-impl.h"
#import "ggml-backend-impl.h"
@@ -896,6 +897,9 @@ struct ggml_metal_device {
struct ggml_metal_device_props props;
// shared fusion debugging context
struct ggml_metal_fusion_info * finfo;
// virtual address for GPU memory allocations
atomic_uintptr_t addr_virt;
};
@@ -1280,6 +1284,13 @@ ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) {
dev->props.max_working_set_size = dev->mtl_device.maxBufferLength;
}
{
const char * val = getenv("GGML_METAL_FUSION_DEBUG");
dev->finfo = ggml_metal_fusion_info_init(
getenv("GGML_METAL_FUSION_DISABLE") == nil,
val ? atoi(val) : 0);
}
snprintf(dev->props.name, sizeof(dev->props.name), "%s%d", "MTL", device);
const char * gpu_name = [[dev->mtl_device name] UTF8String];
if (n_devices > 1) {
@@ -1354,6 +1365,8 @@ void ggml_metal_device_free(ggml_metal_device_t dev) {
assert(dev != NULL);
@autoreleasepool {
ggml_metal_fusion_info_free(dev->finfo);
ggml_metal_rsets_free(dev->rsets);
ggml_metal_library_free(dev->library);
@@ -1941,6 +1954,10 @@ static void ggml_metal_device_disable_tensor(ggml_metal_device_t dev) {
dev->props.has_tensor = false;
}
struct ggml_metal_fusion_info * ggml_metal_device_get_fusion_info(ggml_metal_device_t dev) {
return dev->finfo;
}
//
// device buffers
//
ggml/src/ggml-metal/ggml-metal-fusion.cpp
+502
View File
@@ -0,0 +1,502 @@
#include "ggml-metal-fusion.h"
#include "ggml-backend-impl.h"
#include "ggml-metal-device.h"
#include <algorithm>
#include <string>
#include <vector>
// ---- helpers -------------------------------------------------------------
// true if two tensors live in the same Metal buffer
static bool ggml_metal_fusion_same_buffer(const ggml_tensor * a, const ggml_tensor * b) {
if (!a || !b) {
return false;
}
ggml_backend_buffer_t ba = a->view_src ? a->view_src->buffer : a->buffer;
ggml_backend_buffer_t bb = b->view_src ? b->view_src->buffer : b->buffer;
ggml_metal_buffer_t ca = (ggml_metal_buffer_t) ba->context;
ggml_metal_buffer_t cb = (ggml_metal_buffer_t) bb->context;
return ggml_metal_buffer_get_id(ca, a).metal == ggml_metal_buffer_get_id(cb, b).metal;
}
// ---- pattern checks ------------------------------------------------------
// NORM/RMS_NORM + MUL + ADD: the weight/bias of each fused step must match the norm input
// width, be contiguous rows, and the fused outputs must stay F32
static bool ggml_metal_fusion_check_norm(
const ggml_metal_fusion * fusion,
const ggml_tensor * const * nodes,
ggml_metal_fusion_mode mode) {
GGML_UNUSED(mode);
GGML_ASSERT(fusion->n_ops >= 2);
for (int j = 1; j < fusion->n_ops; j++) {
// the fused MUL/ADD must read the previous node as src0
if (nodes[j]->src[0] != nodes[j - 1]) {
return false;
}
// the weight/bias must have the same row width as the norm input
if (nodes[j]->src[1]->ne[0] != nodes[0]->ne[0]) {
return false;
}
if (!ggml_is_contiguous_rows(nodes[j]->src[1])) {
return false;
}
if (nodes[j]->type != GGML_TYPE_F32) {
return false;
}
}
return true;
}
// ADD x N: each ADD reads the previous ADD as src0, and all addends must share layout
// (and, in FULL mode, live in the same Metal buffer)
static bool ggml_metal_fusion_check_add_chain(
const ggml_metal_fusion * fusion,
const ggml_tensor * const * nodes,
ggml_metal_fusion_mode mode) {
GGML_ASSERT(fusion->n_ops >= 2);
for (int j = 1; j < fusion->n_ops; j++) {
if (nodes[j]->src[0] != nodes[j - 1]) {
return false;
}
if (!ggml_are_same_layout(nodes[j]->src[1], nodes[j - 1]->src[1])) {
return false;
}
if (mode == GGML_METAL_FUSION_FULL) {
if (!ggml_metal_fusion_same_buffer(nodes[j]->src[1], nodes[0]->src[1])) {
return false;
}
}
}
return true;
}
// GATED_DELTA_NET + CPY: the trailing cpy scatters the gdn state snapshots into the recurrent
// cache, so the gdn kernel writes them straight to the cache and the cpy is elided.
// mirrors ggml_metal_op_can_fuse_gdn_cache (PR #25788). the gdn output has other consumers (the
// attn scores view), so unlike the other patterns this is not an elision chain: the structural
// checks live entirely in this callback (unsafe = true).
static bool ggml_metal_fusion_check_gdn_cache(
const ggml_metal_fusion * fusion,
const ggml_tensor * const * nodes,
ggml_metal_fusion_mode mode) {
GGML_UNUSED(fusion);
const ggml_tensor * gdn = nodes[0];
const ggml_tensor * cpy = nodes[1];
// the kernel skips the snapshot tail, so the gdn output must not be a graph output
if (gdn->type != GGML_TYPE_F32 || (gdn->flags & GGML_TENSOR_FLAG_OUTPUT)) {
return false;
}
if (cpy->op != GGML_OP_CPY || (cpy->flags & GGML_TENSOR_FLAG_OUTPUT)) {
return false;
}
const int64_t S_v = gdn->src[2]->ne[0];
const int64_t H = gdn->src[2]->ne[1];
const int64_t n_tokens = gdn->src[2]->ne[2];
const int64_t n_seqs = gdn->src[2]->ne[3];
const int64_t K = ggml_get_op_params_i32(gdn, 0);
const size_t tail_off = ggml_row_size(GGML_TYPE_F32, S_v * H * n_tokens * n_seqs);
const int64_t D = S_v * S_v * H;
const int64_t n_written = std::min<int64_t>(n_tokens, K);
const ggml_tensor * src = cpy->src[0]; // gdn snapshot tail view
const ggml_tensor * dst = cpy->src[1]; // cache view
// src must be this gdn's snapshot tail (contiguous, at the tail offset)
if (src->op != GGML_OP_VIEW || src->view_src != gdn ||
src->view_offs != tail_off || !ggml_is_contiguous(src)) {
return false;
}
const int64_t expected_ne[GGML_MAX_DIMS] = { D, n_seqs, n_written, 1 };
if (dst->type != GGML_TYPE_F32 ||
!std::equal(expected_ne, expected_ne + GGML_MAX_DIMS, dst->ne) ||
dst->nb[0] != ggml_type_size(GGML_TYPE_F32) ||
dst->nb[1] != ggml_row_size(GGML_TYPE_F32, D)) {
return false;
}
if (mode == GGML_METAL_FUSION_FULL) {
// the cache must be allocated so the kernel can write straight to its buffer
if (dst->data == nullptr) {
return false;
}
}
return true;
}
// MUL + SIN + SQR + MUL + ADD (snake activation)
static bool ggml_metal_fusion_check_snake(
const ggml_metal_fusion * fusion,
const ggml_tensor * const * nodes,
ggml_metal_fusion_mode mode) {
GGML_UNUSED(fusion);
GGML_UNUSED(mode);
const ggml_tensor * mul0 = nodes[0];
const ggml_tensor * sin_node = nodes[1];
const ggml_tensor * sqr = nodes[2];
const ggml_tensor * mul1 = nodes[3];
const ggml_tensor * add = nodes[4];
// x carries the full activation shape, a is the broadcast operand
const ggml_tensor * x = ggml_are_same_shape(mul0, mul0->src[0]) ? mul0->src[0] : mul0->src[1];
const ggml_tensor * a = (x == mul0->src[0]) ? mul0->src[1] : mul0->src[0];
// mul1 reads sqr and inv_b in either operand order
const ggml_tensor * inv_b = (mul1->src[0] == sqr) ? mul1->src[1] : mul1->src[0];
// closure check: the trailing add reads the same x as the leading mul
const ggml_tensor * x_in_add = (add->src[0] == mul1) ? add->src[1] : add->src[0];
// x is in the supported whitelist and every chain intermediate shares x's type.
// a and inv_b bind as device const float * in the kernel, so they stay F32.
const bool types_ok =
(x->type == GGML_TYPE_F32 || x->type == GGML_TYPE_F16 || x->type == GGML_TYPE_BF16) &&
(a->type == GGML_TYPE_F32) && (inv_b->type == GGML_TYPE_F32) &&
(mul0->type == x->type) && (sin_node->type == x->type) &&
(sqr->type == x->type) && (mul1->type == x->type) &&
(add->type == x->type);
// a / inv_b collapse to [1, C, 1, 1], x and add stay 2D
const bool shape_ok = ggml_are_same_shape(a, inv_b) && a->ne[0] == 1 && a->ne[1] == x->ne[1];
const bool dim_ok =
(x->ne[2] == 1) && (x->ne[3] == 1) &&
(add->ne[2] == 1) && (add->ne[3] == 1) &&
(a->ne[2] == 1) && (a->ne[3] == 1) &&
(inv_b->ne[2] == 1) && (inv_b->ne[3] == 1);
// kernel reads x[idx] and a[c] / inv_b[c] linearly, so every operand is contiguous
const bool contig_ok =
ggml_is_contiguous(x) && ggml_is_contiguous(add) &&
ggml_is_contiguous(a) && ggml_is_contiguous(inv_b);
return types_ok && shape_ok && dim_ok && contig_ok && x_in_add == x;
}
// ---- patterns ------------------------------------------------------------
static const ggml_op ops_norm_mul[] = { GGML_OP_NORM, GGML_OP_MUL };
static const ggml_op ops_norm_mul_add[] = { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD };
static const ggml_op ops_rms_norm_mul[] = { GGML_OP_RMS_NORM, GGML_OP_MUL };
static const ggml_op ops_rms_norm_mul_add[] = { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD };
static const ggml_op ops_add_2[] = { GGML_OP_ADD, GGML_OP_ADD };
static const ggml_op ops_add_3[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD };
static const ggml_op ops_add_4[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD };
static const ggml_op ops_add_5[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD };
static const ggml_op ops_add_6[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD };
static const ggml_op ops_add_7[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD };
static const ggml_op ops_snake[] = { GGML_OP_MUL, GGML_OP_SIN, GGML_OP_SQR, GGML_OP_MUL, GGML_OP_ADD };
static const ggml_op ops_gdn_cache[] = { GGML_OP_GATED_DELTA_NET, GGML_OP_CPY };
static const ggml_metal_fusion ggml_metal_fusions[] = {
{ GGML_METAL_FUSION_NORM_MUL, ops_norm_mul, 2, false, ggml_metal_fusion_check_norm },
{ GGML_METAL_FUSION_NORM_MUL_ADD, ops_norm_mul_add, 3, false, ggml_metal_fusion_check_norm },
{ GGML_METAL_FUSION_NORM_MUL, ops_rms_norm_mul, 2, false, ggml_metal_fusion_check_norm },
{ GGML_METAL_FUSION_NORM_MUL_ADD, ops_rms_norm_mul_add, 3, false, ggml_metal_fusion_check_norm },
{ GGML_METAL_FUSION_ADD_CHAIN, ops_add_2, 2, false, ggml_metal_fusion_check_add_chain },
{ GGML_METAL_FUSION_ADD_CHAIN, ops_add_3, 3, false, ggml_metal_fusion_check_add_chain },
{ GGML_METAL_FUSION_ADD_CHAIN, ops_add_4, 4, false, ggml_metal_fusion_check_add_chain },
{ GGML_METAL_FUSION_ADD_CHAIN, ops_add_5, 5, false, ggml_metal_fusion_check_add_chain },
{ GGML_METAL_FUSION_ADD_CHAIN, ops_add_6, 6, false, ggml_metal_fusion_check_add_chain },
{ GGML_METAL_FUSION_ADD_CHAIN, ops_add_7, 7, false, ggml_metal_fusion_check_add_chain },
{ GGML_METAL_FUSION_SNAKE, ops_snake, 5, false, ggml_metal_fusion_check_snake },
{ GGML_METAL_FUSION_GDN_CACHE, ops_gdn_cache, 2, true, ggml_metal_fusion_check_gdn_cache },
};
const ggml_metal_fusion * ggml_metal_fusion_all(int * n) {
*n = (int) sizeof(ggml_metal_fusions) / sizeof(ggml_metal_fusions[0]);
return ggml_metal_fusions;
}
// ---- shared fusion info ---------------------------------------------------
static std::string ggml_metal_fusion_label(const ggml_metal_fusion * fusion) {
GGML_ASSERT(fusion != nullptr);
std::string label;
for (int j = 0; j < fusion->n_ops; j++) {
if (j > 0) {
label += '+';
}
label += ggml_op_name(fusion->ops[j]);
}
return label;
}
struct ggml_metal_fusion_info {
std::vector<std::string> labels;
std::vector<uint64_t> counts;
bool enabled;
bool stats;
bool labels_set;
int debug;
};
struct ggml_metal_fusion_info * ggml_metal_fusion_info_init(bool enabled, int debug) {
ggml_metal_fusion_info * finfo = new ggml_metal_fusion_info;
finfo->enabled = enabled;
finfo->stats = debug > 0;
finfo->labels_set = false;
finfo->debug = debug;
if (finfo->stats) {
ggml_metal_fusion_info_labels_init(finfo);
}
return finfo;
}
void ggml_metal_fusion_info_free(struct ggml_metal_fusion_info * finfo) {
delete finfo;
}
bool ggml_metal_fusion_info_enabled(const struct ggml_metal_fusion_info * finfo) {
return finfo->enabled;
}
bool ggml_metal_fusion_info_stats(const struct ggml_metal_fusion_info * finfo) {
return finfo->stats;
}
int ggml_metal_fusion_info_debug(const struct ggml_metal_fusion_info * finfo) {
return finfo->debug;
}
int ggml_metal_fusion_info_n_fusions(const struct ggml_metal_fusion_info * finfo) {
return (int) finfo->labels.size();
}
const char * ggml_metal_fusion_info_label(const struct ggml_metal_fusion_info * finfo, int idx) {
GGML_ASSERT(idx >= 0 && idx < (int) finfo->labels.size());
return finfo->labels[idx].c_str();
}
uint64_t ggml_metal_fusion_info_count(const struct ggml_metal_fusion_info * finfo, int idx) {
GGML_ASSERT(idx >= 0 && idx < (int) finfo->counts.size());
return finfo->counts[idx];
}
void ggml_metal_fusion_info_count_fusion(struct ggml_metal_fusion_info * finfo, const struct ggml_metal_fusion * fusion) {
if (!finfo->stats || fusion == nullptr) {
return;
}
int n = 0;
const ggml_metal_fusion * all = ggml_metal_fusion_all(&n);
int idx = -1;
for (int i = 0; i < n; i++) {
if (&all[i] == fusion) {
idx = i;
break;
}
}
if (idx >= 0 && idx < (int) finfo->counts.size()) {
finfo->counts[idx]++;
}
}
void ggml_metal_fusion_info_set_enabled(struct ggml_metal_fusion_info * finfo, bool enabled) {
finfo->enabled = enabled;
}
void ggml_metal_fusion_info_labels_init(struct ggml_metal_fusion_info * finfo) {
if (finfo->labels_set) {
return;
}
int n = 0;
const ggml_metal_fusion * all = ggml_metal_fusion_all(&n);
finfo->labels.clear();
finfo->counts.assign(n, 0);
finfo->labels.reserve(n);
for (int i = 0; i < n; i++) {
finfo->labels.emplace_back(ggml_metal_fusion_label(&all[i]));
}
finfo->labels_set = true;
}
void ggml_metal_fusion_info_stats_init(struct ggml_metal_fusion_info * finfo) {
finfo->stats = true;
ggml_metal_fusion_info_labels_init(finfo);
}
void ggml_metal_fusion_info_stats_reset(struct ggml_metal_fusion_info * finfo) {
std::fill(finfo->counts.begin(), finfo->counts.end(), 0);
}
int ggml_metal_fusion_info_stats_get(const struct ggml_metal_fusion_info * finfo, const char ** labels, uint64_t * counts, int n) {
const int n_fusions = (int) finfo->labels.size();
if (labels == nullptr) {
return n_fusions;
}
const int n_fill = std::min(n, n_fusions);
for (int i = 0; i < n_fill; i++) {
labels[i] = finfo->labels[i].c_str();
if (counts != nullptr) {
counts[i] = finfo->counts[i];
}
}
return n_fill;
}
// ---- queries -------------------------------------------------------------
// find the longest pattern matching the node sequence starting at idx
// (idx is a position in node_idxs, which maps to graph node indices)
const ggml_metal_fusion * ggml_metal_fusion_next(
const ggml_cgraph * gf,
const int * node_idxs,
int n_idxs,
int idx,
ggml_metal_fusion_mode mode,
int * n_out) {
int n = 0;
const ggml_metal_fusion * all = ggml_metal_fusion_all(&n);
const ggml_metal_fusion * res = nullptr;
int best = 1;
for (int i = 0; i < n; i++) {
const ggml_metal_fusion * fusion = &all[i];
// only look for a longer match than the current best
if (fusion->n_ops <= best) {
continue;
}
if (idx + fusion->n_ops > n_idxs) {
continue;
}
const ggml_tensor * nodes[GGML_METAL_FUSION_MAX];
// the op sequence must match exactly
bool ok = true;
for (int j = 0; j < fusion->n_ops; j++) {
nodes[j] = gf->nodes[node_idxs[idx + j]];
if (nodes[j]->op != fusion->ops[j]) {
ok = false;
break;
}
}
if (!ok) {
continue;
}
if (!fusion->unsafe) {
// common element-wise chain constraints: each node reads the previous one,
// and all nodes have the same shape
for (int j = 1; j < fusion->n_ops && ok; j++) {
if (nodes[j]->src[0] != nodes[j - 1] && nodes[j]->src[1] != nodes[j - 1]) {
ok = false;
break;
}
if (!ggml_are_same_shape(nodes[j], nodes[j - 1])) {
ok = false;
break;
}
}
if (!ok) {
continue;
}
// all current fusions are single-output elision chains, so the last node is the only output
// TODO: multi-output fusions: store pattern-relative offsets in the table and translate them here
int outputs_buf[1];
outputs_buf[0] = node_idxs[idx + fusion->n_ops - 1];
// structural subgraph checks (op sequence, elidable uses, view containment)
if (!ggml_can_fuse_subgraph_ext(gf, node_idxs + idx, fusion->n_ops, fusion->ops, outputs_buf, 1)) {
continue;
}
}
// pattern-specific checks (the sole validator for unsafe patterns)
if (fusion->check && !fusion->check(fusion, nodes, mode)) {
continue;
}
best = fusion->n_ops;
res = fusion;
}
*n_out = best;
return res;
}
// optimize phase: maximum number of nodes starting at idx (a raw sequential graph index) that
// could be fused, chaining patterns back-to-back. matching runs on the same filtered (view
// transparent) node sequence that the compute phase uses, so the returned count is the raw index
// span from idx to the last matched node (intermediate views are packed along).
int ggml_metal_fusion_max(const ggml_cgraph * gf, int idx) {
// an empty/view node cannot start a pattern - pack it alone
if (ggml_op_is_empty(gf->nodes[idx]->op) || ggml_is_empty(gf->nodes[idx])) {
return 1;
}
// collect the non-empty node indices starting at idx
int idxs[GGML_METAL_FUSION_MAX];
int n_idxs = 0;
for (int i = idx; i < gf->n_nodes && n_idxs < GGML_METAL_FUSION_MAX; i++) {
if (!ggml_op_is_empty(gf->nodes[i]->op) && !ggml_is_empty(gf->nodes[i])) {
idxs[n_idxs++] = i;
}
}
if (n_idxs == 0) {
return 1;
}
int total = 0;
int i_f = 0;
while (i_f < n_idxs && total < GGML_METAL_FUSION_MAX) {
int len = 1;
const ggml_metal_fusion * fusion = ggml_metal_fusion_next(gf, idxs, n_idxs, i_f, GGML_METAL_FUSION_STRUCTURAL, &len);
if (!fusion || total + len > GGML_METAL_FUSION_MAX) {
break;
}
total += len;
i_f += len;
}
if (i_f == 0) {
return 1;
}
// map the matched non-empty nodes back to the raw index span (views are included)
return std::min(GGML_METAL_FUSION_MAX, idxs[i_f - 1] - idx + 1);
}
ggml/src/ggml-metal/ggml-metal-fusion.h
+104
View File
@@ -0,0 +1,104 @@
// single source of truth for the fusions supported by the Metal backend
//
// every fusable subgraph is declared exactly once as a ggml_metal_fusion entry in
// the table in ggml-metal-fusion.cpp. both the graph optimizer (ggml_metal_fusion_max)
// and the op encoders (ggml_metal_fusion_next) consult this same table, so the two
// phases can never disagree about what can be fused.
#pragma once
#include "ggml-impl.h"
#include <stdbool.h>
#ifdef __cplusplus
extern "C" {
#endif
// the maximum number of nodes that can be fused in a single kernel
// (also the maximum length of a packed fusion group during graph optimization)
#define GGML_METAL_FUSION_MAX 16
typedef enum ggml_metal_fusion_mode {
// structural checks only; used by the graph optimizer, at which point the graph
// tensors are not allocated yet, so buffer placement cannot be verified
GGML_METAL_FUSION_STRUCTURAL = 0,
// full checks, including buffer placement; used by the op encoders
GGML_METAL_FUSION_FULL,
} ggml_metal_fusion_mode;
// identifier of each fusion pattern so the op encoders know which kernel to use
typedef enum ggml_metal_fusion_id {
GGML_METAL_FUSION_NONE = 0,
GGML_METAL_FUSION_NORM_MUL, // NORM/RMS_NORM + MUL
GGML_METAL_FUSION_NORM_MUL_ADD, // NORM/RMS_NORM + MUL + ADD
GGML_METAL_FUSION_ADD_CHAIN, // ADD x N (N in [2, 7])
GGML_METAL_FUSION_SNAKE, // MUL + SIN + SQR + MUL + ADD
GGML_METAL_FUSION_GDN_CACHE, // GATED_DELTA_NET + CPY (write snapshots into the recurrent cache)
} ggml_metal_fusion_id;
struct ggml_metal_fusion {
ggml_metal_fusion_id id;
const enum ggml_op * ops; // op sequence (fixed length)
int n_ops; // number of ops
// if unsafe: the generic chain/shape + ggml_can_fuse_subgraph checks are skipped and the
// check callback below is the sole validator (used for patterns that are not elision chains,
// e.g. the gdn + cache-cpy write-through fusion)
bool unsafe;
// extra backend constraints on top of ggml_can_fuse_subgraph
// nodes[j] is the j-th node of the pattern
bool (*check)(const struct ggml_metal_fusion * fusion,
const struct ggml_tensor * const * nodes,
ggml_metal_fusion_mode mode);
};
typedef struct ggml_metal_fusion ggml_metal_fusion;
// the single table of all fusions supported by the Metal backend
const ggml_metal_fusion * ggml_metal_fusion_all(int * n);
// ---- shared fusion info ---------------------------------------------------
// shared fusion debugging context, owned by the device; newly created backend contexts for that
// device register with it so the fusion counters are race-free and accumulate across contexts.
struct ggml_metal_fusion_info; // defined in ggml-metal-fusion.cpp
struct ggml_metal_fusion_info * ggml_metal_fusion_info_init(bool enabled, int debug);
void ggml_metal_fusion_info_free(struct ggml_metal_fusion_info * finfo);
bool ggml_metal_fusion_info_enabled(const struct ggml_metal_fusion_info * finfo);
bool ggml_metal_fusion_info_stats (const struct ggml_metal_fusion_info * finfo);
int ggml_metal_fusion_info_debug (const struct ggml_metal_fusion_info * finfo);
int ggml_metal_fusion_info_n_fusions(const struct ggml_metal_fusion_info * finfo);
const char * ggml_metal_fusion_info_label (const struct ggml_metal_fusion_info * finfo, int idx);
uint64_t ggml_metal_fusion_info_count (const struct ggml_metal_fusion_info * finfo, int idx);
void ggml_metal_fusion_info_count_fusion(struct ggml_metal_fusion_info * finfo, const struct ggml_metal_fusion * fusion);
void ggml_metal_fusion_info_set_enabled (struct ggml_metal_fusion_info * finfo, bool enabled);
void ggml_metal_fusion_info_stats_init ( struct ggml_metal_fusion_info * finfo);
void ggml_metal_fusion_info_stats_reset( struct ggml_metal_fusion_info * finfo);
int ggml_metal_fusion_info_stats_get (const struct ggml_metal_fusion_info * finfo, const char ** labels, uint64_t * counts, int n);
void ggml_metal_fusion_info_labels_init( struct ggml_metal_fusion_info * finfo);
// compute phase: longest fusion starting at idx (a position in node_idxs) that matches in `mode`.
// returns the matching pattern (nullptr if no fusion) and sets *n_out to the number of nodes consumed.
const ggml_metal_fusion * ggml_metal_fusion_next(
const struct ggml_cgraph * gf,
const int * node_idxs,
int n_idxs,
int idx,
ggml_metal_fusion_mode mode,
int * n_out);
// optimize phase: maximum number of nodes starting at idx (a raw sequential graph index) that
// could be fused, chaining patterns back-to-back. returns at least 1.
int ggml_metal_fusion_max(const struct ggml_cgraph * gf, int idx);
#ifdef __cplusplus
}
#endif
ggml/src/ggml-metal/ggml-metal-impl.h
+7
View File
@@ -62,18 +62,23 @@
#define N_R0_IQ1_S 4
#define N_SG_IQ1_S 2
#define N_R0_IQ1_S_SPLIT 8
#define N_R0_IQ1_M 4
#define N_SG_IQ1_M 2
#define N_R0_IQ1_M_SPLIT 8
#define N_R0_IQ2_XXS 4
#define N_SG_IQ2_XXS 2
#define N_R0_IQ2_XXS_SPLIT 8
#define N_R0_IQ2_XS 4
#define N_SG_IQ2_XS 2
#define N_R0_IQ2_XS_SPLIT 8
#define N_R0_IQ2_S 4
#define N_SG_IQ2_S 2
#define N_R0_IQ2_S_SPLIT 8
#define N_R0_IQ3_XXS 4
#define N_SG_IQ3_XXS 2
@@ -81,6 +86,7 @@
#define N_R0_IQ3_S 4
#define N_SG_IQ3_S 2
#define N_R0_IQ3_S_SPLIT 8
#define N_R0_IQ4_NL 2
#define N_SG_IQ4_NL 2
@@ -979,6 +985,7 @@ typedef struct {
uint64_t nb1;
uint64_t nb2;
uint64_t nb3;
uint64_t nb_out; // 0 => snapshots are appended after the attn scores (unfused)
} ggml_metal_kargs_gated_delta_net;
typedef struct {
ggml/src/ggml-metal/ggml-metal-ops.cpp
+107 -167
View File
@@ -7,6 +7,7 @@
#include "ggml-metal-impl.h"
#include "ggml-metal-common.h"
#include "ggml-metal-device.h"
#include "ggml-metal-fusion.h"
#include "ggml-metal-tuning.h"
#include <cassert>
@@ -31,24 +32,22 @@ struct ggml_metal_op {
ggml_metal_device_t dev,
ggml_metal_cmd_buf_t cmd_buf,
ggml_cgraph * gf,
ggml_metal_fusion_info * finfo,
int idx_start,
int idx_end,
bool use_fusion,
bool use_concurrency,
bool use_capture,
int debug_graph,
int debug_fusion) {
int debug_graph) {
this->dev = dev;
this->lib = ggml_metal_device_get_library(dev);
this->enc = ggml_metal_encoder_init(cmd_buf, use_concurrency);
this->mem_ranges = ggml_mem_ranges_init(debug_graph);
this->finfo = finfo;
this->idx_start = idx_start;
this->idx_end = idx_end;
this->use_fusion = use_fusion;
this->use_concurrency = use_concurrency;
this->use_capture = use_capture;
this->debug_graph = debug_graph;
this->debug_fusion = debug_fusion;
this->gf = gf;
idxs.reserve(gf->n_nodes);
@@ -78,15 +77,24 @@ struct ggml_metal_op {
return ggml_graph_node(gf, idxs[i]);
}
bool can_fuse(int i0, const ggml_op * ops, int n_ops) const {
assert(use_fusion);
// consult the fusion table for the longest pattern starting at i0
// returns the matching pattern (nullptr if no fusion) and sets *n_out to the number of nodes
const ggml_metal_fusion * can_fuse(int i0, enum ggml_metal_fusion_mode mode, int * n_out) const {
assert(use_fusion());
assert(i0 >= 0 && i0 < n_nodes());
if (i0 + n_ops > n_nodes()) {
return false;
}
return ggml_metal_fusion_next(gf, idxs.data(), (int) idxs.size(), i0, mode, n_out);
}
return ggml_can_fuse_ext(gf, idxs.data() + i0, ops, n_ops);
// whether to attempt fusion; the toggle lives in the shared fusion debugging context owned
// by the device (initialized from GGML_METAL_FUSION_DISABLE, overridable by the test)
bool use_fusion() const {
return ggml_metal_fusion_info_enabled(finfo);
}
// record that a fusion fired, indexed by the matching table entry
void count_fusions(const ggml_metal_fusion * fusion) const {
ggml_metal_fusion_info_count_fusion(finfo, fusion);
}
ggml_metal_device_t dev;
@@ -94,12 +102,13 @@ struct ggml_metal_op {
ggml_metal_encoder_t enc;
ggml_mem_ranges_t mem_ranges;
bool use_fusion;
// shared fusion debugging context
ggml_metal_fusion_info * finfo;
bool use_concurrency;
bool use_capture;
int debug_graph;
int debug_fusion;
private:
ggml_cgraph * gf;
@@ -115,24 +124,22 @@ ggml_metal_op_t ggml_metal_op_init(
ggml_metal_device_t dev,
ggml_metal_cmd_buf_t cmd_buf,
ggml_cgraph * gf,
ggml_metal_fusion_info * finfo,
int idx_start,
int idx_end,
bool use_fusion,
bool use_concurrency,
bool use_capture,
int debug_graph,
int debug_fusion) {
int debug_graph) {
ggml_metal_op_t res = new ggml_metal_op(
dev,
cmd_buf,
gf,
finfo,
idx_start,
idx_end,
use_fusion,
use_concurrency,
use_capture,
debug_graph,
debug_fusion);
debug_graph);
return res;
}
@@ -1868,6 +1875,8 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) {
ggml_metal_library_t lib = ctx->lib;
ggml_metal_encoder_t enc = ctx->enc;
const bool use_fusion = ctx->use_fusion();
const int debug_fusion = ggml_metal_fusion_info_debug(ctx->finfo);
GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);
GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);
@@ -1880,6 +1889,31 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) {
auto pipeline = ggml_metal_library_get_pipeline_gated_delta_net(lib, op);
// when fused with the trailing cache cpy, the snapshots are written straight into the
// recurrent cache and the cpy is skipped (see GGML_METAL_FUSION_GDN_CACHE)
ggml_metal_buffer_id bid_out = ggml_metal_get_buffer_id(op);
uint64_t nb_out = 0;
int n_fuse = 1;
if (use_fusion) {
int n = 1;
const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n);
if (fusion && fusion->id == GGML_METAL_FUSION_GDN_CACHE) {
const ggml_tensor * dst_cache = ctx->node(idx + 1)->src[1]; // cache view
bid_out = ggml_metal_get_buffer_id(dst_cache);
nb_out = dst_cache->nb[2]/sizeof(float);
n_fuse = 2;
ctx->count_fusions(fusion);
if (debug_fusion > 1) {
GGML_LOG_DEBUG("%s: fuse: GATED_DELTA_NET + CPY\n", __func__);
}
}
}
int ida = 0;
ggml_metal_kargs_gated_delta_net args = {
@@ -1918,23 +1952,25 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) {
/*.nb1 =*/ nb1,
/*.nb2 =*/ nb2,
/*.nb3 =*/ nb3,
/*.nb_out =*/ nb_out,
};
ggml_metal_encoder_set_pipeline(enc, pipeline);
ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), ida++);
ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), ida++); // args
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), ida++); // q
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), ida++); // k
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), ida++); // v
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[3]), ida++); // gate
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[4]), ida++); // beta
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[5]), ida++); // state
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++); // dst
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++); // dst (attn)
ggml_metal_encoder_set_buffer (enc, bid_out, ida++); // state_out
const int nsg = pipeline.nsg;
ggml_metal_encoder_dispatch_threadgroups(enc, op->src[2]->ne[0]/nsg, op->src[2]->ne[1], op->src[2]->ne[3], 32, nsg, 1);
return 1;
return n_fuse;
}
int ggml_metal_op_solve_tri(ggml_metal_op_t ctx, int idx) {
@@ -3718,56 +3754,20 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
return 1;
}
// Snake activation autofuse: mul -> sin -> sqr -> mul -> add
static bool ggml_metal_op_can_fuse_snake(ggml_metal_op_t ctx, int idx) {
static constexpr ggml_op snake_ops[5] = { GGML_OP_MUL, GGML_OP_SIN, GGML_OP_SQR, GGML_OP_MUL, GGML_OP_ADD };
if (ctx->node(idx)->op != GGML_OP_MUL || !ctx->can_fuse(idx, snake_ops, 5)) {
return false;
}
const ggml_tensor * mul0 = ctx->node(idx + 0);
const ggml_tensor * sin_node = ctx->node(idx + 1);
const ggml_tensor * sqr = ctx->node(idx + 2);
const ggml_tensor * mul1 = ctx->node(idx + 3);
const ggml_tensor * add = ctx->node(idx + 4);
// x carries the full activation shape, a is the broadcast operand
const ggml_tensor * x = ggml_are_same_shape(mul0, mul0->src[0]) ? mul0->src[0] : mul0->src[1];
const ggml_tensor * a = (x == mul0->src[0]) ? mul0->src[1] : mul0->src[0];
// mul1 reads sqr and inv_b in either operand order
const ggml_tensor * inv_b = (mul1->src[0] == sqr) ? mul1->src[1] : mul1->src[0];
// closure check: the trailing add reads the same x as the leading mul
const ggml_tensor * x_in_add = (add->src[0] == mul1) ? add->src[1] : add->src[0];
// x is in the supported whitelist and every chain intermediate shares x's type.
// a and inv_b bind as device const float * in the kernel, so they stay F32.
const bool types_ok =
(x->type == GGML_TYPE_F32 || x->type == GGML_TYPE_F16 || x->type == GGML_TYPE_BF16) &&
(a->type == GGML_TYPE_F32) && (inv_b->type == GGML_TYPE_F32) &&
(mul0->type == x->type) && (sin_node->type == x->type) &&
(sqr->type == x->type) && (mul1->type == x->type) &&
(add->type == x->type);
// a / inv_b collapse to [1, C, 1, 1], x and add stay 2D
const bool shape_ok = ggml_are_same_shape(a, inv_b) && a->ne[0] == 1 && a->ne[1] == x->ne[1];
const bool dim_ok =
(x->ne[2] == 1) && (x->ne[3] == 1) &&
(add->ne[2] == 1) && (add->ne[3] == 1) &&
(a->ne[2] == 1) && (a->ne[3] == 1) &&
(inv_b->ne[2] == 1) && (inv_b->ne[3] == 1);
// kernel reads x[idx] and a[c] / inv_b[c] linearly, so every operand is contiguous
const bool contig_ok =
ggml_is_contiguous(x) && ggml_is_contiguous(add) &&
ggml_is_contiguous(a) && ggml_is_contiguous(inv_b);
return types_ok && shape_ok && dim_ok && contig_ok && x_in_add == x;
}
int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) {
if (ctx->use_fusion && ggml_metal_op_can_fuse_snake(ctx, idx)) {
return ggml_metal_op_snake_fused(ctx, idx);
int n_fuse = 1;
const ggml_metal_fusion * fusion = nullptr;
if (ctx->use_fusion()) {
int n = 1;
fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n);
n_fuse = n;
// snake activation autofuse: mul -> sin -> sqr -> mul -> add
if (fusion && fusion->id == GGML_METAL_FUSION_SNAKE) {
ctx->count_fusions(fusion);
return ggml_metal_op_snake_fused(ctx, idx);
}
}
ggml_tensor * op = ctx->node(idx);
@@ -3775,9 +3775,9 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) {
ggml_metal_library_t lib = ctx->lib;
ggml_metal_encoder_t enc = ctx->enc;
const bool use_fusion = ctx->use_fusion;
const bool use_fusion = ctx->use_fusion();
const int debug_fusion = ctx->debug_fusion;
const int debug_fusion = ggml_metal_fusion_info_debug(ctx->finfo);
GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);
GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);
@@ -3822,57 +3822,19 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) {
/*.o1 =*/ { bid_src1.offs },
};
ggml_op fops[8];
int n_fuse = 1;
// c[0] = add(a, b[0])
// c[1] = add(c[0], b[1])
// c[2] = add(c[1], b[2])
// ...
if (use_fusion) {
fops[0] = GGML_OP_ADD;
fops[1] = GGML_OP_ADD;
fops[2] = GGML_OP_ADD;
fops[3] = GGML_OP_ADD;
fops[4] = GGML_OP_ADD;
fops[5] = GGML_OP_ADD;
fops[6] = GGML_OP_ADD;
fops[7] = GGML_OP_ADD;
// note: in metal, we sometimes encode the graph in parallel so we have to avoid fusing ops
// across splits. idx_end indicates the last node in the current split
for (n_fuse = 0; n_fuse <= 6; ++n_fuse) {
if (!ctx->can_fuse(idx + n_fuse, fops + n_fuse, 2)) {
break;
}
ggml_tensor * f0 = ctx->node(idx + n_fuse);
ggml_tensor * f1 = ctx->node(idx + n_fuse + 1);
if (f0 != f1->src[0]) {
break;
}
// b[0] === b[1] === ...
if (!ggml_are_same_layout(f0->src[1], f1->src[1])) {
break;
}
// only fuse ops if src1 is in the same Metal buffer
ggml_metal_buffer_id bid_fuse = ggml_metal_get_buffer_id(f1->src[1]);
if (bid_fuse.metal != bid_src1.metal) {
break;
}
//ctx->fuse_cnt[ops[n_fuse + 1]->op]++;
args.o1[n_fuse + 1] = bid_fuse.offs;
if (use_fusion && fusion && fusion->id == GGML_METAL_FUSION_ADD_CHAIN) {
// the offsets of the fused addends are relative to the start of the src1 buffer
for (int i = 1; i < n_fuse; i++) {
args.o1[i] = ggml_metal_get_buffer_id(ctx->node(idx + i)->src[1]).offs;
}
++n_fuse;
ctx->count_fusions(fusion);
if (debug_fusion > 1 && n_fuse > 1) {
if (debug_fusion > 1) {
GGML_LOG_DEBUG("%s: fuse: ADD x %d\n", __func__, n_fuse);
}
}
@@ -4080,9 +4042,9 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) {
ggml_metal_library_t lib = ctx->lib;
ggml_metal_encoder_t enc = ctx->enc;
const bool use_fusion = ctx->use_fusion;
const bool use_fusion = ctx->use_fusion();
const int debug_fusion = ctx->debug_fusion;
const int debug_fusion = ggml_metal_fusion_info_debug(ctx->finfo);
GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);
GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);
@@ -4110,8 +4072,6 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) {
/*.nbf3 =*/ { nb03 },
};
ggml_op fops[8];
int n_fuse = 1;
ggml_metal_buffer_id bid_fuse[2] = { bid_src0, bid_src0 };
@@ -4120,55 +4080,35 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) {
// d[1] = mul(d[0], b)
// d[2] = add(d[1], c)
if (use_fusion) {
fops[0] = op->op;
fops[1] = GGML_OP_MUL;
fops[2] = GGML_OP_ADD;
int n = 1;
const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n);
for (n_fuse = 0; n_fuse <= 1; ++n_fuse) {
if (!ctx->can_fuse(idx + n_fuse, fops + n_fuse, 2)) {
break;
if (fusion && (fusion->id == GGML_METAL_FUSION_NORM_MUL || fusion->id == GGML_METAL_FUSION_NORM_MUL_ADD)) {
n_fuse = n;
ctx->count_fusions(fusion);
for (int i = 1; i < n_fuse; i++) {
const ggml_tensor * fn = ctx->node(idx + i);
bid_fuse[i - 1] = ggml_metal_get_buffer_id(fn->src[1]);
args.nef1[i] = fn->src[1]->ne[1];
args.nef2[i] = fn->src[1]->ne[2];
args.nef3[i] = fn->src[1]->ne[3];
args.nbf1[i] = fn->src[1]->nb[1];
args.nbf2[i] = fn->src[1]->nb[2];
args.nbf3[i] = fn->src[1]->nb[3];
}
ggml_tensor * f0 = ctx->node(idx + n_fuse);
ggml_tensor * f1 = ctx->node(idx + n_fuse + 1);
if (f0 != f1->src[0]) {
break;
}
if (f1->src[1]->ne[0] != op->ne[0]) {
break;
}
if (!ggml_is_contiguous_rows(f1->src[1])) {
break;
}
if (f1->type != GGML_TYPE_F32) {
break;
}
//ctx->fuse_cnt[f1->op]++;
bid_fuse[n_fuse] = ggml_metal_get_buffer_id(f1->src[1]);
args.nef1[n_fuse + 1] = f1->src[1]->ne[1];
args.nef2[n_fuse + 1] = f1->src[1]->ne[2];
args.nef3[n_fuse + 1] = f1->src[1]->ne[3];
args.nbf1[n_fuse + 1] = f1->src[1]->nb[1];
args.nbf2[n_fuse + 1] = f1->src[1]->nb[2];
args.nbf3[n_fuse + 1] = f1->src[1]->nb[3];
}
++n_fuse;
if (debug_fusion > 1 && n_fuse > 1) {
if (n_fuse == 2) {
GGML_LOG_DEBUG("%s: fuse: %s + MUL\n", __func__, ggml_op_name(op->op));
}
if (n_fuse == 3) {
GGML_LOG_DEBUG("%s: fuse: %s + MUL + ADD\n", __func__, ggml_op_name(op->op));
if (debug_fusion > 1) {
if (n_fuse == 2) {
GGML_LOG_DEBUG("%s: fuse: %s + MUL\n", __func__, ggml_op_name(op->op));
}
if (n_fuse == 3) {
GGML_LOG_DEBUG("%s: fuse: %s + MUL + ADD\n", __func__, ggml_op_name(op->op));
}
}
}
}
ggml/src/ggml-metal/ggml-metal-ops.h
+4 -3
View File
@@ -8,17 +8,18 @@ extern "C" {
typedef struct ggml_metal_op * ggml_metal_op_t;
struct ggml_metal_fusion; // forward decl (ggml-metal-device.h)
ggml_metal_op_t ggml_metal_op_init(
ggml_metal_device_t dev,
ggml_metal_cmd_buf_t cmd_buf,
struct ggml_cgraph * gf,
struct ggml_metal_fusion_info * finfo,
int idx_start,
int idx_end,
bool use_fusion,
bool use_concurrency,
bool use_capture,
int debug_graph,
int debug_fusion);
int debug_graph);
void ggml_metal_op_free(ggml_metal_op_t ctx);
ggml/src/ggml-metal/ggml-metal.cpp
+42
View File
@@ -4,6 +4,7 @@
#include "ggml-backend-impl.h"
#include "ggml-metal-device.h"
#include "ggml-metal-fusion.h"
#include "ggml-metal-context.h"
#include "ggml-metal-ops.h"
#include "ggml-metal-tuning.h"
@@ -906,6 +907,30 @@ static const char * ggml_backend_metal_tuning_device_token(ggml_backend_dev_t de
return ggml_metal_device_id_token(ggml_metal_device_get_props(ctx_dev)->device_id);
}
// generic fusion debugging API (ad-hoc proc-address mechanism): the test resolves the device
// fusion context once and passes that opaque handle to the rest of the functions
typedef void * ggml_backend_fusion_t;
static ggml_backend_fusion_t ggml_backend_metal_fusion_get(ggml_backend_dev_t dev) {
return ggml_metal_device_get_fusion_info((ggml_metal_device_t)dev->context);
}
static void ggml_backend_metal_fusion_stats_init(ggml_backend_fusion_t finfo) {
ggml_metal_fusion_info_stats_init((struct ggml_metal_fusion_info *) finfo);
}
static void ggml_backend_metal_fusion_stats_reset(ggml_backend_fusion_t finfo) {
ggml_metal_fusion_info_stats_reset((struct ggml_metal_fusion_info *) finfo);
}
static int ggml_backend_metal_fusion_stats_get(ggml_backend_fusion_t finfo, const char ** labels, uint64_t * counts, int n) {
return ggml_metal_fusion_info_stats_get((struct ggml_metal_fusion_info *) finfo, labels, counts, n);
}
static void ggml_backend_metal_fusion_set_enabled(ggml_backend_fusion_t finfo, bool enabled) {
ggml_metal_fusion_info_set_enabled((struct ggml_metal_fusion_info *) finfo, enabled);
}
static void * ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, const char * name) {
if (strcmp(name, "ggml_backend_get_features") == 0) {
return (void *)ggml_backend_metal_get_features;
@@ -928,6 +953,23 @@ static void * ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, const
if (strcmp(name, "ggml_backend_metal_tuning_device_token") == 0) {
return (void *)ggml_backend_metal_tuning_device_token;
}
// generic fusion debugging API (ad-hoc proc-address mechanism, not part of the official
// ggml backend interface yet; a backend that adopts it exports these exact names)
if (strcmp(name, "ggml_backend_fusion_get") == 0) {
return (void *)ggml_backend_metal_fusion_get;
}
if (strcmp(name, "ggml_backend_fusion_stats_init") == 0) {
return (void *)ggml_backend_metal_fusion_stats_init;
}
if (strcmp(name, "ggml_backend_fusion_stats_reset") == 0) {
return (void *)ggml_backend_metal_fusion_stats_reset;
}
if (strcmp(name, "ggml_backend_fusion_stats_get") == 0) {
return (void *)ggml_backend_metal_fusion_stats_get;
}
if (strcmp(name, "ggml_backend_fusion_set_enabled") == 0) {
return (void *)ggml_backend_metal_fusion_set_enabled;
}
return NULL;
ggml/src/ggml-metal/kernels/gated_delta_net.metal
+17 -3
View File
@@ -15,6 +15,7 @@ kernel void kernel_gated_delta_net_impl(
device const char * b,
device const char * s,
device char * dst,
device char * dst_fuse,
uint3 tgpig[[threadgroup_position_in_grid]],
uint3 tpitg[[thread_position_in_threadgroup]],
uint3 ntg[[threads_per_threadgroup]]) {
@@ -65,6 +66,12 @@ kernel void kernel_gated_delta_net_impl(
// per-(seq,head) offset within a slot
const uint state_out_base = (i23*args.ne21 + i21)*S_v*S_v + i20*S_v;
// when fused with the cache cpy, write the snapshots straight into the cache buffer using
// the slot stride; otherwise append them after the attn scores (nb_out == 0)
const bool fused = args.nb_out > 0;
const device float * state_out = fused ? (device float *)dst_fuse : (device float *)dst + attn_size;
const uint slot_stride = fused ? (uint)args.nb_out : state_size_per_snap;
for (short t = 0; t < args.ne22; t++) {
float s_k = 0.0f;
@@ -116,7 +123,7 @@ kernel void kernel_gated_delta_net_impl(
if (K > 1) {
const int target_slot = (int)args.ne22 - 1 - (int)t;
if (target_slot >= 0 && target_slot < (int)K) {
device float * dst_state = (device float *) (dst) + attn_size + (uint)target_slot * state_size_per_snap + state_out_base;
device float * dst_state = (device float *)state_out + (uint)target_slot * slot_stride + state_out_base;
FOR_UNROLL (short j = 0; j < NSG; j++) {
const short is = tx*NSG + j;
dst_state[is] = ls[j];
@@ -126,7 +133,7 @@ kernel void kernel_gated_delta_net_impl(
}
if (K == 1) {
device float * dst_state = (device float *) (dst) + attn_size + state_out_base;
device float * dst_state = (device float *)state_out + state_out_base;
FOR_UNROLL (short j = 0; j < NSG; j++) {
const short is = tx*NSG + j;
dst_state[is] = ls[j];
@@ -158,6 +165,7 @@ kernel void kernel_gated_delta_net_impl(
device const char * b,
device const char * s,
device char * dst,
device char * dst_fuse,
uint3 tgpig[[threadgroup_position_in_grid]],
uint3 tpitg[[thread_position_in_threadgroup]],
uint3 ntg[[threads_per_threadgroup]]) {
@@ -230,7 +238,13 @@ kernel void kernel_gated_delta_net_impl(
dst_attn += args.ne21*S_v;
}
device float * dst_state = (device float *) (dst) + args.ne23*args.ne22*args.ne21*S_v + (i23*args.ne21 + i21)*S_v*S_v + i20;
// when fused with the cache cpy, write the snapshots straight into the cache buffer using
// the slot stride; otherwise append them after the attn scores (nb_out == 0)
const bool fused = args.nb_out > 0;
const device float * state_out = fused ? (device float *)dst_fuse : (device float *)dst + args.ne23*args.ne22*args.ne21*S_v;
const uint slot_stride = fused ? (uint)args.nb_out : S_v*S_v;
device float * dst_state = (device float *)state_out + (i23*args.ne21 + i21)*slot_stride + i20;
device T * dstt_state = (device T *) (dst_state);
FOR_UNROLL (short j = 0; j < NSG; j++) {
ggml/src/ggml-metal/kernels/mul_mm.metal
+64 -35
View File
@@ -496,6 +496,13 @@ kernel void kernel_mul_mm_id(
+ args.nb11*i11
+ args.nb10*iy);
// skip the upper half of the token tile when the expert did not fill it
constexpr short NR1H = NR1/2;
const bool has_hi = nr1 > NR1H;
const short lb1 = (short) tiitg/NL1; // 0 .. NR1-1, this thread's row of the B tile
#ifndef GGML_METAL_HAS_TENSOR
S0_8x8 ma[4];
S1_8x8 mb[2];
@@ -505,15 +512,22 @@ kernel void kernel_mul_mm_id(
for (short i = 0; i < 8; i++){
mc[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
}
// simdgroups 2,3 own rows NR1H..NR1-1
const bool sg_active = has_hi || sgitg < 2;
#else
auto tA = tensor<threadgroup S0, dextents<int32_t, 2>, tensor_inline>(sa, dextents<int32_t, 2>(NK, NR0));
auto tB = tensor<threadgroup S1, dextents<int32_t, 2>, tensor_inline>(sb, dextents<int32_t, 2>(NR1, NK ));
auto tA = tensor<threadgroup S0, dextents<int32_t, 2>, tensor_inline>(sa, dextents<int32_t, 2>(NK, NR0));
// sb is [NR1][NK] row-major
auto tB0 = tensor<threadgroup S1, dextents<int32_t, 2>, tensor_inline>(sb, dextents<int32_t, 2>(NK, NR1H));
auto tB1 = tensor<threadgroup S1, dextents<int32_t, 2>, tensor_inline>(sb + NR1H*NK, dextents<int32_t, 2>(NK, NR1H));
mpp::tensor_ops::matmul2d<
mpp::tensor_ops::matmul2d_descriptor(NR1, NR0, NK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate),
mpp::tensor_ops::matmul2d_descriptor(NR1H, NR0, NK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate),
execution_simdgroups<4>> mm;
auto cT = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB), float>();
auto cT0 = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB0), float>();
auto cT1 = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB1), float>();
#endif
for (int loop_k = 0; loop_k < args.ne00; loop_k += NK) {
@@ -656,37 +670,45 @@ kernel void kernel_mul_mm_id(
threadgroup_barrier(mem_flags::mem_threadgroup);
#ifndef GGML_METAL_HAS_TENSOR
// load matrices from threadgroup memory and conduct outer products
threadgroup const S0 * lsma = (sa + 4*64*(sgitg%2));
threadgroup const S1 * lsmb = (sb + 2*64*(sgitg/2));
if (sg_active) {
// load matrices from threadgroup memory and conduct outer products
threadgroup const S0 * lsma = (sa + 4*64*(sgitg%2));
threadgroup const S1 * lsmb = (sb + 2*64*(sgitg/2));
FOR_UNROLL (short ik = 0; ik < NK/8; ik++) {
simdgroup_barrier(mem_flags::mem_none);
FOR_UNROLL (short ik = 0; ik < NK/8; ik++) {
simdgroup_barrier(mem_flags::mem_none);
FOR_UNROLL (short i = 0; i < 4; i++) {
simdgroup_load(ma[i], lsma + 64*i, 8, 0, false);
FOR_UNROLL (short i = 0; i < 4; i++) {
simdgroup_load(ma[i], lsma + 64*i, 8, 0, false);
}
simdgroup_barrier(mem_flags::mem_none);
FOR_UNROLL (short i = 0; i < 2; i++) {
simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false);
}
simdgroup_barrier(mem_flags::mem_none);
FOR_UNROLL (short i = 0; i < 8; i++){
simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]);
}
lsma += 8*64;
lsmb += 4*64;
}
simdgroup_barrier(mem_flags::mem_none);
FOR_UNROLL (short i = 0; i < 2; i++) {
simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false);
}
simdgroup_barrier(mem_flags::mem_none);
FOR_UNROLL (short i = 0; i < 8; i++){
simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]);
}
lsma += 8*64;
lsmb += 4*64;
}
#else
auto sA = tA.slice(0, 0);
auto sB = tB.slice(0, 0);
auto sA = tA.slice(0, 0);
auto sB0 = tB0.slice(0, 0);
mm.run(sB, sA, cT);
mm.run(sB0, sA, cT0);
if (has_hi) {
auto sB1 = tB1.slice(0, 0);
mm.run(sB1, sA, cT1);
}
#endif
}
@@ -694,13 +716,20 @@ kernel void kernel_mul_mm_id(
threadgroup_barrier(mem_flags::mem_threadgroup);
#ifdef GGML_METAL_HAS_TENSOR
auto tC = tensor<threadgroup float, dextents<int32_t, 2>, tensor_inline>(sc, dextents<int32_t, 2>(NR0, NR1));
cT.store(tC);
#else
threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0;
auto tC0 = tensor<threadgroup float, dextents<int32_t, 2>, tensor_inline>(sc, dextents<int32_t, 2>(NR0, NR1H));
cT0.store(tC0);
for (short i = 0; i < 8; i++) {
simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false);
if (has_hi) {
auto tC1 = tensor<threadgroup float, dextents<int32_t, 2>, tensor_inline>(sc + NR1H*NR0, dextents<int32_t, 2>(NR0, NR1H));
cT1.store(tC1);
}
#else
if (sg_active) {
threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0;
for (short i = 0; i < 8; i++) {
simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false);
}
}
#endif
ggml/src/ggml-metal/kernels/mul_mv.metal
+226 -82
View File
@@ -1889,8 +1889,19 @@ void kernel_mul_mv_iq2_xxs_f32_impl(
const uint i12 = im%FC_mul_mv_ne12;
const uint i13 = im/FC_mul_mv_ne12;
const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
const int nb32 = nb * (QK_K / 32);
const short ntx = FC_mul_mv_split ? nb32 : 32;
const short nrep = 32 / ntx;
const short ix = tiisg % ntx;
const short irep = tiisg / ntx;
const short row0 = (nr0 * irep ) / nrep;
const short row1 = (nr0 * (irep + 1)) / nrep;
const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
device const block_iq2_xxs * x = (device const block_iq2_xxs *) (src0 + offset0);
device const float * y = (device const float *) (src1 + offset1);
@@ -1898,8 +1909,6 @@ void kernel_mul_mv_iq2_xxs_f32_impl(
float yl[32];
float sumf[nr0]={0.f};
const int nb32 = nb * (QK_K / 32);
threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem);
threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 256);
{
@@ -1912,11 +1921,9 @@ void kernel_mul_mv_iq2_xxs_f32_impl(
threadgroup_barrier(mem_flags::mem_threadgroup);
}
const int ix = tiisg;
device const float * y4 = y + 32 * ix;
for (int ib32 = ix; ib32 < nb32; ib32 += 32) {
for (int ib32 = ix; ib32 < nb32; ib32 += ntx) {
for (short i = 0; i < 32; ++i) {
yl[i] = y4[i];
}
@@ -1928,7 +1935,7 @@ void kernel_mul_mv_iq2_xxs_f32_impl(
device const uint16_t * q2 = xr->qs + 4 * ib;
device const half * dh = &xr->d;
for (short row = 0; row < nr0; row++) {
for (short row = row0; row < row1; row++) {
const float db = dh[0];
device const uint8_t * aux8 = (device const uint8_t *)q2;
const uint32_t aux32 = q2[2] | (q2[3] << 16);
@@ -1948,7 +1955,7 @@ void kernel_mul_mv_iq2_xxs_f32_impl(
q2 += args.nb01/2;
}
y4 += 32 * 32;
y4 += 32 * ntx;
}
device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
@@ -1961,6 +1968,23 @@ void kernel_mul_mv_iq2_xxs_f32_impl(
}
}
template<typename args_t>
void kernel_mul_mv_iq2_xxs_f32_disp(
args_t args,
device const char * src0,
device const char * src1,
device char * dst,
threadgroup char * shmem,
uint3 tgpig,
ushort tiisg,
ushort sgitg) {
if (FC_mul_mv_split) {
kernel_mul_mv_iq2_xxs_f32_impl<N_R0_IQ2_XXS_SPLIT, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
} else {
kernel_mul_mv_iq2_xxs_f32_impl<N_R0_IQ2_XXS, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
}
}
[[host_name("kernel_mul_mv_iq2_xxs_f32")]]
kernel void kernel_mul_mv_iq2_xxs_f32(
constant ggml_metal_kargs_mul_mv & args,
@@ -1971,7 +1995,7 @@ kernel void kernel_mul_mv_iq2_xxs_f32(
uint3 tgpig[[threadgroup_position_in_grid]],
ushort tiisg[[thread_index_in_simdgroup]],
ushort sgitg[[simdgroup_index_in_threadgroup]]) {
kernel_mul_mv_iq2_xxs_f32_impl<N_R0_IQ2_XXS, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
kernel_mul_mv_iq2_xxs_f32_disp<constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
}
template<int nr0, typename args_t>
@@ -1997,8 +2021,19 @@ void kernel_mul_mv_iq2_xs_f32_impl(
const uint i12 = im%FC_mul_mv_ne12;
const uint i13 = im/FC_mul_mv_ne12;
const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
const int nb32 = nb * (QK_K / 32);
const short ntx = FC_mul_mv_split ? nb32 : 32;
const short nrep = 32 / ntx;
const short ix = tiisg % ntx;
const short irep = tiisg / ntx;
const short row0 = (nr0 * irep ) / nrep;
const short row1 = (nr0 * (irep + 1)) / nrep;
const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
device const block_iq2_xs * x = (device const block_iq2_xs *) (src0 + offset0);
device const float * y = (device const float *) (src1 + offset1);
@@ -2006,8 +2041,6 @@ void kernel_mul_mv_iq2_xs_f32_impl(
float yl[32];
float sumf[nr0]={0.f};
const int nb32 = nb * (QK_K / 32);
threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem);
threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 512);
{
@@ -2020,11 +2053,9 @@ void kernel_mul_mv_iq2_xs_f32_impl(
threadgroup_barrier(mem_flags::mem_threadgroup);
}
const int ix = tiisg;
device const float * y4 = y + 32 * ix;
for (int ib32 = ix; ib32 < nb32; ib32 += 32) {
for (int ib32 = ix; ib32 < nb32; ib32 += ntx) {
for (short i = 0; i < 32; ++i) {
yl[i] = y4[i];
}
@@ -2037,7 +2068,7 @@ void kernel_mul_mv_iq2_xs_f32_impl(
device const uint8_t * sc = xr->scales + ib;
device const half * dh = &xr->d;
for (short row = 0; row < nr0; row++) {
for (short row = row0; row < row1; row++) {
const float db = dh[0];
const uint8_t ls1 = sc[0] & 0xf;
const uint8_t ls2 = sc[0] >> 4;
@@ -2066,7 +2097,7 @@ void kernel_mul_mv_iq2_xs_f32_impl(
sc += args.nb01;
}
y4 += 32 * 32;
y4 += 32 * ntx;
}
device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
@@ -2079,6 +2110,23 @@ void kernel_mul_mv_iq2_xs_f32_impl(
}
}
template<typename args_t>
void kernel_mul_mv_iq2_xs_f32_disp(
args_t args,
device const char * src0,
device const char * src1,
device char * dst,
threadgroup char * shmem,
uint3 tgpig,
ushort tiisg,
ushort sgitg) {
if (FC_mul_mv_split) {
kernel_mul_mv_iq2_xs_f32_impl<N_R0_IQ2_XS_SPLIT, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
} else {
kernel_mul_mv_iq2_xs_f32_impl<N_R0_IQ2_XS, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
}
}
[[host_name("kernel_mul_mv_iq2_xs_f32")]]
kernel void kernel_mul_mv_iq2_xs_f32(
constant ggml_metal_kargs_mul_mv & args,
@@ -2090,7 +2138,7 @@ kernel void kernel_mul_mv_iq2_xs_f32(
ushort tiisg[[thread_index_in_simdgroup]],
ushort sgitg[[simdgroup_index_in_threadgroup]]) {
kernel_mul_mv_iq2_xs_f32_impl<N_R0_IQ2_XS, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
kernel_mul_mv_iq2_xs_f32_disp<constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
}
// FC_mul_mv_split: for nb32 < 32 (nb32 divides 32), 32/nb32 threads share each chunk and each takes a slice of the rows
@@ -2117,8 +2165,19 @@ void kernel_mul_mv_iq3_xxs_f32_impl(
const uint i12 = im%FC_mul_mv_ne12;
const uint i13 = im/FC_mul_mv_ne12;
const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
const int nb32 = nb * (QK_K / 32);
const short ntx = FC_mul_mv_split ? nb32 : 32;
const short nrep = 32 / ntx;
const short ix = tiisg % ntx;
const short irep = tiisg / ntx;
const short row0 = (nr0 * irep ) / nrep;
const short row1 = (nr0 * (irep + 1)) / nrep;
const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
device const block_iq3_xxs * x = (device const block_iq3_xxs *) (src0 + offset0);
device const float * y = (device const float *) (src1 + offset1);
@@ -2126,8 +2185,6 @@ void kernel_mul_mv_iq3_xxs_f32_impl(
float yl[32];
float sumf[nr0]={0.f};
const int nb32 = nb * (QK_K / 32);
threadgroup uint32_t * svalues = (threadgroup uint32_t *)(shmem);
threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 256);
{
@@ -2140,15 +2197,6 @@ void kernel_mul_mv_iq3_xxs_f32_impl(
threadgroup_barrier(mem_flags::mem_threadgroup);
}
const short ntx = FC_mul_mv_split ? nb32 : 32;
const short nrep = 32 / ntx;
const short ix = tiisg % ntx;
const short irep = tiisg / ntx;
const short row0 = (nr0 * irep ) / nrep;
const short row1 = (nr0 * (irep + 1)) / nrep;
device const float * y4 = y + 32 * ix;
for (int ib32 = ix; ib32 < nb32; ib32 += ntx) {
@@ -2160,9 +2208,9 @@ void kernel_mul_mv_iq3_xxs_f32_impl(
const int ib = ib32 % (QK_K / 32);
device const block_iq3_xxs * xr = x + ibl;
device const uint8_t * q3 = xr->qs + 8 * ib + (uint64_t) row0*args.nb01;
device const uint16_t * gas = (device const uint16_t *)(xr->qs + QK_K/4) + 2 * ib + (uint64_t) row0*args.nb01/2;
device const half * dh = &xr->d + (uint64_t) row0*args.nb01/2;
device const uint8_t * q3 = xr->qs + 8 * ib;
device const uint16_t * gas = (device const uint16_t *)(xr->qs + QK_K/4) + 2 * ib;
device const half * dh = &xr->d;
for (short row = row0; row < row1; row++) {
const float db = dh[0];
@@ -2253,8 +2301,19 @@ void kernel_mul_mv_iq3_s_f32_impl(
const uint i12 = im%FC_mul_mv_ne12;
const uint i13 = im/FC_mul_mv_ne12;
const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
const int nb32 = nb * (QK_K / 32);
const short ntx = FC_mul_mv_split ? nb32 : 32;
const short nrep = 32 / ntx;
const short ix = tiisg % ntx;
const short irep = tiisg / ntx;
const short row0 = (nr0 * irep ) / nrep;
const short row1 = (nr0 * (irep + 1)) / nrep;
const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
device const block_iq3_s * x = (device const block_iq3_s *) (src0 + offset0);
device const float * y = (device const float *) (src1 + offset1);
@@ -2262,8 +2321,6 @@ void kernel_mul_mv_iq3_s_f32_impl(
float yl[32];
float sumf[nr0]={0.f};
const int nb32 = nb * (QK_K / 32);
threadgroup uint32_t * svalues = (threadgroup uint32_t *) shmem;
{
int nval = 8;
@@ -2272,11 +2329,9 @@ void kernel_mul_mv_iq3_s_f32_impl(
threadgroup_barrier(mem_flags::mem_threadgroup);
}
const int ix = tiisg;
device const float * y4 = y + 32 * ix;
for (int ib32 = ix; ib32 < nb32; ib32 += 32) {
for (int ib32 = ix; ib32 < nb32; ib32 += ntx) {
for (short i = 0; i < 32; ++i) {
yl[i] = y4[i];
}
@@ -2291,7 +2346,7 @@ void kernel_mul_mv_iq3_s_f32_impl(
device const uint8_t * signs = xr->signs + 4 * ib;
device const half * dh = &xr->d;
for (short row = 0; row < nr0; row++) {
for (short row = row0; row < row1; row++) {
const float db = dh[0];
const float d = db * (1 + 2*((sc[0] >> 4*(ib%2)) & 0xf));
@@ -2315,7 +2370,7 @@ void kernel_mul_mv_iq3_s_f32_impl(
signs += args.nb01;
}
y4 += 32 * 32;
y4 += 32 * ntx;
}
device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
@@ -2328,6 +2383,23 @@ void kernel_mul_mv_iq3_s_f32_impl(
}
}
template<typename args_t>
void kernel_mul_mv_iq3_s_f32_disp(
args_t args,
device const char * src0,
device const char * src1,
device char * dst,
threadgroup char * shmem,
uint3 tgpig,
ushort tiisg,
ushort sgitg) {
if (FC_mul_mv_split) {
kernel_mul_mv_iq3_s_f32_impl<N_R0_IQ3_S_SPLIT, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
} else {
kernel_mul_mv_iq3_s_f32_impl<N_R0_IQ3_S, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
}
}
[[host_name("kernel_mul_mv_iq3_s_f32")]]
kernel void kernel_mul_mv_iq3_s_f32(
constant ggml_metal_kargs_mul_mv & args,
@@ -2339,7 +2411,7 @@ kernel void kernel_mul_mv_iq3_s_f32(
ushort tiisg[[thread_index_in_simdgroup]],
ushort sgitg[[simdgroup_index_in_threadgroup]]) {
kernel_mul_mv_iq3_s_f32_impl<N_R0_IQ3_S, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
kernel_mul_mv_iq3_s_f32_disp<constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
}
template<int nr0, typename args_t>
@@ -2365,8 +2437,19 @@ void kernel_mul_mv_iq2_s_f32_impl(
const uint i12 = im%FC_mul_mv_ne12;
const uint i13 = im/FC_mul_mv_ne12;
const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
const int nb32 = nb * (QK_K / 32);
const short ntx = FC_mul_mv_split ? nb32 : 32;
const short nrep = 32 / ntx;
const short ix = tiisg % ntx;
const short irep = tiisg / ntx;
const short row0 = (nr0 * irep ) / nrep;
const short row1 = (nr0 * (irep + 1)) / nrep;
const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
device const block_iq2_s * x = (device const block_iq2_s *) (src0 + offset0);
device const float * y = (device const float *) (src1 + offset1);
@@ -2374,8 +2457,6 @@ void kernel_mul_mv_iq2_s_f32_impl(
float yl[32];
float sumf[nr0]={0.f};
const int nb32 = nb * (QK_K / 32);
//threadgroup uint64_t * svalues = (threadgroup uint64_t *) shmem;
//{
// int nval = 32;
@@ -2384,11 +2465,9 @@ void kernel_mul_mv_iq2_s_f32_impl(
// threadgroup_barrier(mem_flags::mem_threadgroup);
//}
const short ix = tiisg;
device const float * y4 = y + 32 * ix;
for (int ib32 = ix; ib32 < nb32; ib32 += 32) {
for (int ib32 = ix; ib32 < nb32; ib32 += ntx) {
for (short i = 0; i < 32; ++i) {
yl[i] = y4[i];
}
@@ -2403,7 +2482,7 @@ void kernel_mul_mv_iq2_s_f32_impl(
device const uint8_t * signs = qs + QK_K/8;
device const half * dh = &xr->d;
for (short row = 0; row < nr0; row++) {
for (short row = row0; row < row1; row++) {
const float db = dh[0];
const float d1 = db * (0.5f + (sc[0] & 0xf));
const float d2 = db * (0.5f + (sc[0] >> 4));
@@ -2428,7 +2507,7 @@ void kernel_mul_mv_iq2_s_f32_impl(
signs += args.nb01;
}
y4 += 32 * 32;
y4 += 32 * ntx;
}
device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
@@ -2441,6 +2520,23 @@ void kernel_mul_mv_iq2_s_f32_impl(
}
}
template<typename args_t>
void kernel_mul_mv_iq2_s_f32_disp(
args_t args,
device const char * src0,
device const char * src1,
device char * dst,
threadgroup char * shmem,
uint3 tgpig,
ushort tiisg,
ushort sgitg) {
if (FC_mul_mv_split) {
kernel_mul_mv_iq2_s_f32_impl<N_R0_IQ2_S_SPLIT, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
} else {
kernel_mul_mv_iq2_s_f32_impl<N_R0_IQ2_S, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
}
}
[[host_name("kernel_mul_mv_iq2_s_f32")]]
kernel void kernel_mul_mv_iq2_s_f32(
constant ggml_metal_kargs_mul_mv & args,
@@ -2452,7 +2548,7 @@ kernel void kernel_mul_mv_iq2_s_f32(
ushort tiisg[[thread_index_in_simdgroup]],
ushort sgitg[[simdgroup_index_in_threadgroup]]) {
kernel_mul_mv_iq2_s_f32_impl<N_R0_IQ2_S, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
kernel_mul_mv_iq2_s_f32_disp<constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
}
template<int nr0, typename args_t>
@@ -2478,8 +2574,19 @@ void kernel_mul_mv_iq1_s_f32_impl(
const uint i12 = im%FC_mul_mv_ne12;
const uint i13 = im/FC_mul_mv_ne12;
const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
const int nb32 = nb * (QK_K / 32);
const short ntx = FC_mul_mv_split ? nb32 : 32;
const short nrep = 32 / ntx;
const short ix = tiisg % ntx;
const short irep = tiisg / ntx;
const short row0 = (nr0 * irep ) / nrep;
const short row1 = (nr0 * (irep + 1)) / nrep;
const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
device const block_iq1_s * x = (device const block_iq1_s *) (src0 + offset0);
device const float * y = (device const float *) (src1 + offset1);
@@ -2487,13 +2594,9 @@ void kernel_mul_mv_iq1_s_f32_impl(
float yl[32];
float sumf[nr0]={0.f};
const int nb32 = nb * (QK_K / 32);
const short ix = tiisg;
device const float * y4 = y + 32 * ix;
for (int ib32 = ix; ib32 < nb32; ib32 += 32) {
for (int ib32 = ix; ib32 < nb32; ib32 += ntx) {
float sumy = 0;
for (short i = 0; i < 32; ++i) {
yl[i] = y4[i];
@@ -2508,7 +2611,7 @@ void kernel_mul_mv_iq1_s_f32_impl(
device const uint16_t * qh = xr->qh + ib;
device const half * dh = &xr->d;
for (short row = 0; row < nr0; row++) {
for (short row = row0; row < row1; row++) {
constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700)));
constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 5) & 0x700)));
constant uint8_t * grid3 = (constant uint8_t *)(iq1s_grid_gpu + (qs[2] | ((qh[0] << 2) & 0x700)));
@@ -2528,7 +2631,7 @@ void kernel_mul_mv_iq1_s_f32_impl(
qh += args.nb01/2;
}
y4 += 32 * 32;
y4 += 32 * ntx;
}
device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
@@ -2541,6 +2644,23 @@ void kernel_mul_mv_iq1_s_f32_impl(
}
}
template<typename args_t>
void kernel_mul_mv_iq1_s_f32_disp(
args_t args,
device const char * src0,
device const char * src1,
device char * dst,
threadgroup char * shmem,
uint3 tgpig,
ushort tiisg,
ushort sgitg) {
if (FC_mul_mv_split) {
kernel_mul_mv_iq1_s_f32_impl<N_R0_IQ1_S_SPLIT, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
} else {
kernel_mul_mv_iq1_s_f32_impl<N_R0_IQ1_S, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
}
}
[[host_name("kernel_mul_mv_iq1_s_f32")]]
kernel void kernel_mul_mv_iq1_s_f32(
constant ggml_metal_kargs_mul_mv & args,
@@ -2551,7 +2671,7 @@ kernel void kernel_mul_mv_iq1_s_f32(
ushort tiisg[[thread_index_in_simdgroup]],
ushort sgitg[[simdgroup_index_in_threadgroup]]) {
kernel_mul_mv_iq1_s_f32_impl<N_R0_IQ1_S, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);
kernel_mul_mv_iq1_s_f32_disp<constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);
}
template<int nr0, typename args_t>
@@ -2577,8 +2697,19 @@ void kernel_mul_mv_iq1_m_f32_impl(
const uint i12 = im%FC_mul_mv_ne12;
const uint i13 = im/FC_mul_mv_ne12;
const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
const int nb32 = nb * (QK_K / 32);
const short ntx = FC_mul_mv_split ? nb32 : 32;
const short nrep = 32 / ntx;
const short ix = tiisg % ntx;
const short irep = tiisg / ntx;
const short row0 = (nr0 * irep ) / nrep;
const short row1 = (nr0 * (irep + 1)) / nrep;
const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
device const block_iq1_m * x = (device const block_iq1_m *) (src0 + offset0);
device const float * y = (device const float *) (src1 + offset1);
@@ -2586,15 +2717,11 @@ void kernel_mul_mv_iq1_m_f32_impl(
float yl[32];
float sumf[nr0]={0.f};
const int nb32 = nb * (QK_K / 32);
const short ix = tiisg;
device const float * y4 = y + 32 * ix;
iq1m_scale_t scale;
for (int ib32 = ix; ib32 < nb32; ib32 += 32) {
for (int ib32 = ix; ib32 < nb32; ib32 += ntx) {
float4 sumy = {0.f};
for (short i = 0; i < 8; ++i) {
yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0];
@@ -2611,7 +2738,7 @@ void kernel_mul_mv_iq1_m_f32_impl(
device const uint8_t * qh = xr->qh + 2 * ib;
device const uint16_t * sc = (device const uint16_t *)xr->scales;
for (short row = 0; row < nr0; row++) {
for (short row = row0; row < row1; row++) {
scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);
constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700)));
@@ -2637,7 +2764,7 @@ void kernel_mul_mv_iq1_m_f32_impl(
qh += args.nb01;
}
y4 += 32 * 32;
y4 += 32 * ntx;
}
device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
@@ -2650,6 +2777,23 @@ void kernel_mul_mv_iq1_m_f32_impl(
}
}
template<typename args_t>
void kernel_mul_mv_iq1_m_f32_disp(
args_t args,
device const char * src0,
device const char * src1,
device char * dst,
threadgroup char * shmem,
uint3 tgpig,
ushort tiisg,
ushort sgitg) {
if (FC_mul_mv_split) {
kernel_mul_mv_iq1_m_f32_impl<N_R0_IQ1_M_SPLIT, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
} else {
kernel_mul_mv_iq1_m_f32_impl<N_R0_IQ1_M, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
}
}
[[host_name("kernel_mul_mv_iq1_m_f32")]]
kernel void kernel_mul_mv_iq1_m_f32(
constant ggml_metal_kargs_mul_mv & args,
@@ -2660,7 +2804,7 @@ kernel void kernel_mul_mv_iq1_m_f32(
ushort tiisg[[thread_index_in_simdgroup]],
ushort sgitg[[simdgroup_index_in_threadgroup]]) {
kernel_mul_mv_iq1_m_f32_impl<N_R0_IQ1_M, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);
kernel_mul_mv_iq1_m_f32_disp<constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);
}
template<int NR0, typename args_t>
@@ -3239,13 +3383,13 @@ template [[host_name("kernel_mul_mv_id_q3_K_f32")]] kernel kernel_mul_mv_id_t
template [[host_name("kernel_mul_mv_id_q4_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_q4_K_f32_impl <N_R0_Q4_K>>>;
template [[host_name("kernel_mul_mv_id_q5_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_q5_K_f32_impl <N_R0_Q5_K>>>;
template [[host_name("kernel_mul_mv_id_q6_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_q6_K_f32_impl <N_R0_Q6_K>>>;
template [[host_name("kernel_mul_mv_id_iq1_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq1_s_f32_impl <N_R0_IQ1_S>>>;
template [[host_name("kernel_mul_mv_id_iq1_m_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq1_m_f32_impl <N_R0_IQ1_M>>>;
template [[host_name("kernel_mul_mv_id_iq2_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq2_xxs_f32_impl<N_R0_IQ2_XXS>>>;
template [[host_name("kernel_mul_mv_id_iq2_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq2_xs_f32_impl <N_R0_IQ2_XS>>>;
template [[host_name("kernel_mul_mv_id_iq1_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq1_s_f32_disp<ggml_metal_kargs_mul_mv>>>;
template [[host_name("kernel_mul_mv_id_iq1_m_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq1_m_f32_disp<ggml_metal_kargs_mul_mv>>>;
template [[host_name("kernel_mul_mv_id_iq2_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq2_xxs_f32_disp<ggml_metal_kargs_mul_mv>>>;
template [[host_name("kernel_mul_mv_id_iq2_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq2_xs_f32_disp<ggml_metal_kargs_mul_mv>>>;
template [[host_name("kernel_mul_mv_id_iq3_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq3_xxs_f32_disp<ggml_metal_kargs_mul_mv>>>;
template [[host_name("kernel_mul_mv_id_iq3_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq3_s_f32_impl <N_R0_IQ3_S>>>;
template [[host_name("kernel_mul_mv_id_iq2_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq2_s_f32_impl <N_R0_IQ2_S>>>;
template [[host_name("kernel_mul_mv_id_iq3_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq3_s_f32_disp<ggml_metal_kargs_mul_mv>>>;
template [[host_name("kernel_mul_mv_id_iq2_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq2_s_f32_disp<ggml_metal_kargs_mul_mv>>>;
template [[host_name("kernel_mul_mv_id_iq4_nl_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq4_nl_f32_impl <N_R0_IQ4_NL>>>;
template [[host_name("kernel_mul_mv_id_iq4_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq4_xs_f32_impl <N_R0_IQ4_XS>>>;
template [[host_name("kernel_mul_mv_id_tq2_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_tq2_0_f32_impl <N_R0_TQ2_0>>>;
ggml/src/ggml-vulkan/ggml-vulkan.cpp
+95 -46
View File
@@ -5430,8 +5430,9 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
bool prefer_large = tiles_m > shader_core_count || tiles_l > shader_core_count ||
(tiles_l <= shader_core_count / 3 && tiles_m > shader_core_count / 2);
if (n > crossover_large && prefer_large) return last;
uint32_t crossover_medium = configs[0].unaligned->wg_denoms[1];
if (n > crossover_medium) return 1;
uint32_t crossover_medium_m = configs[0].unaligned->wg_denoms[0];
uint32_t crossover_medium_n = configs[0].unaligned->wg_denoms[1];
if (m > crossover_medium_m && n > crossover_medium_n) return 1;
return 0;
};
device->matmul_id_tile_selector = [](uint32_t /*m*/, uint32_t n, uint32_t /*k*/, uint32_t /*shader_core_count*/,
@@ -9061,7 +9062,7 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m,
}
uint32_t split_k = 1;
if (ctx->device->shader_core_count != 0 && m >= pipeline->wg_denoms[0] && n >= pipeline->wg_denoms[1]) {
if (ctx->device->shader_core_count != 0 && n >= pipeline->wg_denoms[1]) {
// If k is 'large' and the SMs will fill less than halfway, use split_k.
uint32_t m_tiles = CEIL_DIV(m, pipeline->wg_denoms[0]);
uint32_t n_tiles = CEIL_DIV(n, pipeline->wg_denoms[1]);
@@ -9814,10 +9815,10 @@ static bool ggml_vk_should_use_mmvq(const vk_device& device, uint32_t m, uint32_
GGML_UNUSED(m);
}
static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {
static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx, bool swap_inputs = false) {
ggml_tensor * dst = cgraph->nodes[node_idx];
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
const ggml_tensor * src0 = dst->src[swap_inputs ? 1 : 0];
const ggml_tensor * src1 = dst->src[swap_inputs ? 0 : 1];
VK_LOG_DEBUG("ggml_vk_mul_mat_vec_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3];
std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3];
@@ -9836,8 +9837,8 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context&
const uint64_t ne12 = src1->ne[2];
const uint64_t ne13 = src1->ne[3];
const uint64_t ne20 = dst->ne[0];
const uint64_t ne21 = dst->ne[1];
const uint64_t ne20 = dst->ne[swap_inputs ? 1 : 0];
const uint64_t ne21 = dst->ne[swap_inputs ? 0 : 1];
// const uint64_t ne22 = dst->ne[2];
// const uint64_t ne23 = dst->ne[3];
@@ -10451,6 +10452,16 @@ static void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context& subctx, c
src0->ne[1] <= ctx->device->properties.limits.maxComputeWorkGroupCount[1] &&
src1->ne[2] <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]) {
ggml_vk_mul_mat_vec_nc_f16_f32(ctx, subctx, cgraph, node_idx);
// With one output row, B^T*A has the same flat output as A^T*B.
} else if (ctx->num_additional_fused_ops == 0 &&
(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) &&
(src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_BF16 || ggml_is_quantized(src1->type)) &&
dst->ne[0] == 1 && dst->ne[1] > mul_mat_vec_max_cols &&
src0->ne[2] == 1 && src0->ne[3] == 1 &&
src1->ne[2] == 1 && src1->ne[3] == 1 &&
ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst) &&
get_misalign_bytes(ctx, src0) == 0 && get_misalign_bytes(ctx, src1) == 0 && get_misalign_bytes(ctx, dst) == 0) {
ggml_vk_mul_mat_vec_q_f16(ctx, subctx, cgraph, node_idx, true);
// mul_mat_vec supports batching ne12*ne13 when ne11==1, or treating ne11 as the batch size (up to four)
// when ne12 and ne13 are one.
} else if ((dst->ne[1] == 1 || (dst->ne[1] <= mul_mat_vec_max_cols && src1->ne[2] * src1->ne[3] == 1)) &&
@@ -17158,6 +17169,22 @@ static bool ggml_backend_vk_cpy_tensor_async(ggml_backend_t backend_src, ggml_ba
return false;
}
// If the backend is idle, use a CPU copy to avoid GPU synchronization overhead.
static constexpr size_t max_cpu_copy_size = 128 * 1024;
const bool src_backend_synchronous = backend_src->iface.synchronize == nullptr;
const bool transfer_idle = !ctx->device->async_use_transfer_queue ||
ctx->transfer_semaphore_last_submitted == ctx->transfer_semaphore.value;
const bool backend_idle = ctx->compute_ctx.expired() && ctx->transfer_ctx.expired() &&
!ctx->submit_pending && !ctx->almost_ready_fence_pending && transfer_idle;
const bool dst_host_coherent =
(dst_buf->memory_property_flags & (vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent)) ==
(vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent);
if ((backend_src == backend_dst || src_backend_synchronous) && backend_idle && dst_host_coherent && ggml_nbytes(src) <= max_cpu_copy_size) {
ggml_vk_buffer_write(dst_buf, vk_tensor_offset(dst) + dst->view_offs, src->data, ggml_nbytes(src));
return true;
}
vk_context cpy_ctx;
if (ctx->device->async_use_transfer_queue) {
cpy_ctx = ggml_vk_get_transfer_ctx(ctx);
@@ -17170,7 +17197,6 @@ static bool ggml_backend_vk_cpy_tensor_async(ggml_backend_t backend_src, ggml_ba
src->data, ggml_nbytes(src));
}
GGML_UNUSED(backend_src);
return false;
}
@@ -18351,38 +18377,30 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
bool need_disable = false;
// topk_moe often overwrites the source, but for a given row all the src values are
// loaded before anything is stored. If there's only one row, this is safe, so treat
// this as a special case.
bool is_topk_moe_single_row = ctx->fused_topk_moe_mode != TOPK_MOE_COUNT &&
ggml_nrows(cgraph->nodes[i]->src[0]) == 1;
if (!is_topk_moe_single_row) {
for (int j = 0; j < 2; ++j) {
ggml_tensor *dst = output_nodes[j];
if (!dst) {
continue;
}
// Loop over all srcs of all nodes in the fusion. If the src overlaps
// the destination and the src is not an intermediate node that's being
// elided, then disable fusion.
for (int k = 0; k <= ctx->num_additional_fused_ops; ++k) {
for (uint32_t s = 0; s < GGML_MAX_SRC; ++s) {
ggml_tensor *src = cgraph->nodes[i + k]->src[s];
if (!src || src->op == GGML_OP_NONE) {
continue;
for (int j = 0; j < 2; ++j) {
ggml_tensor *dst = output_nodes[j];
if (!dst) {
continue;
}
// Loop over all srcs of all nodes in the fusion. If the src overlaps
// the destination and the src is not an intermediate node that's being
// elided, then disable fusion.
for (int k = 0; k <= ctx->num_additional_fused_ops; ++k) {
for (uint32_t s = 0; s < GGML_MAX_SRC; ++s) {
ggml_tensor *src = cgraph->nodes[i + k]->src[s];
if (!src || src->op == GGML_OP_NONE) {
continue;
}
if (ggml_vk_tensors_overlap(src, dst, op_srcs_fused_elementwise[k])) {
bool found = false;
for (int n = 0; n < k; ++n) {
if (cgraph->nodes[i + n] == src) {
found = true;
break;
}
}
if (ggml_vk_tensors_overlap(src, dst, op_srcs_fused_elementwise[k])) {
bool found = false;
for (int n = 0; n < k; ++n) {
if (cgraph->nodes[i + n] == src) {
found = true;
break;
}
}
if (!found) {
need_disable = true;
}
if (!found) {
need_disable = true;
}
}
}
@@ -18395,6 +18413,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
ctx->fused_topk_moe_scale = false;
ctx->fused_topk_qsa = false;
ctx->fused_rms_norm_mode = RMS_NORM_COUNT;
fusion_string = nullptr;
}
}
@@ -18497,7 +18516,6 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
// Sort the graph for improved parallelism.
static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, struct ggml_backend_graph_optimize_params * params)
{
GGML_UNUSED(params);
VK_LOG_DEBUG("ggml_vk_graph_optimize(" << graph->n_nodes << " nodes)");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
@@ -18583,19 +18601,50 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph *
return false;
};
if (keep_pattern(topk_moe_early_softmax_norm)) {
auto const &add_pattern_alloc_deps = [&](const std::initializer_list<ggml_op> &pattern, int last_node) {
// Keep external inputs alive through the fused output.
std::set<ggml_tensor *> seen;
for (size_t j = 0; j < pattern.size(); ++j) {
ggml_tensor * node = graph->nodes[first_unused + j];
for (uint32_t s = 0; s < GGML_MAX_SRC; ++s) {
ggml_tensor * src = node->src[s];
if (src && seen.insert(src).second) {
params->add_alloc_dep(params->user_data, src, graph->nodes[last_node]);
}
}
seen.insert(node);
}
};
auto const &keep_topk_moe_pattern = [&](const std::initializer_list<ggml_op> &pattern) -> bool {
if (!match_pattern(pattern, first_unused)) {
return false;
}
int last_node = first_unused + (int) pattern.size() - 1;
// Some TOPK_MOE variants fuse a trailing scale.
if (last_node + 1 < graph->n_nodes && graph->nodes[last_node + 1]->op == GGML_OP_SCALE) {
last_node++;
}
add_pattern_alloc_deps(pattern, last_node);
return keep_pattern(pattern);
};
if (keep_topk_moe_pattern(topk_moe_early_softmax_norm)) {
continue;
}
if (keep_pattern(topk_moe_sigmoid_norm_bias)) {
if (keep_topk_moe_pattern(topk_moe_sigmoid_norm_bias)) {
continue;
}
if (keep_pattern(topk_moe_sqrt_softplus_norm_bias)) {
if (keep_topk_moe_pattern(topk_moe_sqrt_softplus_norm_bias)) {
continue;
}
if (keep_pattern(topk_moe_early_softmax)) {
if (keep_topk_moe_pattern(topk_moe_early_softmax)) {
continue;
}
if (keep_pattern(topk_moe_late_softmax)) {
if (keep_topk_moe_pattern(topk_moe_late_softmax)) {
continue;
}
if (keep_pattern(snake_pattern)) {
ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp
+17 -11
View File
@@ -33,7 +33,11 @@ void argsort(bool needs_bounds_check, const uint row) {
const uint row_offset = row * p.ncols;
// initialize indices
dst_row[col] = ivec2(col, floatBitsToInt(data_a[row_offset + col]));
ivec2 value = ivec2(col, 0);
if (!needs_bounds_check || col < p.ncols) {
value.y = floatBitsToInt(data_a[row_offset + col]);
}
dst_row[col] = value;
barrier();
uint num_outer_loop_iters = NCOLS_PADDED_LOG2;
@@ -42,18 +46,20 @@ void argsort(bool needs_bounds_check, const uint row) {
[[unroll]] for (uint j = k / 2, inner_idx = 0; inner_idx < num_inner_loop_iters; j /= 2, inner_idx++) {
const int ixj = int(col ^ j);
int idx_0 = (col & k) == 0 ? col : ixj;
int idx_1 = (col & k) == 0 ? ixj : col;
if (ixj > col) {
int idx_0 = (col & k) == 0 ? col : ixj;
int idx_1 = (col & k) == 0 ? ixj : col;
ivec2 sh_idx_0 = dst_row[idx_0];
ivec2 sh_idx_1 = dst_row[idx_1];
bool idx_0_oob = needs_bounds_check ? sh_idx_0.x >= p.ncols : false;
bool idx_1_oob = needs_bounds_check ? sh_idx_1.x >= p.ncols : false;
ivec2 sh_idx_0 = dst_row[idx_0];
ivec2 sh_idx_1 = dst_row[idx_1];
bool idx_0_oob = needs_bounds_check ? sh_idx_0.x >= p.ncols : false;
bool idx_1_oob = needs_bounds_check ? sh_idx_1.x >= p.ncols : false;
if ((idx_0_oob ||
(!idx_1_oob && intBitsToFloat(sh_idx_0.y) > intBitsToFloat(sh_idx_1.y))) && (ixj > col)) {
dst_row[idx_0] = sh_idx_1;
dst_row[idx_1] = sh_idx_0;
if (idx_0_oob ||
(!idx_1_oob && intBitsToFloat(sh_idx_0.y) > intBitsToFloat(sh_idx_1.y))) {
dst_row[idx_0] = sh_idx_1;
dst_row[idx_1] = sh_idx_0;
}
}
barrier();
ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp
+4 -1
View File
@@ -42,7 +42,10 @@ void argsort(bool needs_bounds_check, const uint row) {
[[unroll]] for (int u = 0; u < WG_UNROLL_FACTOR; ++u) {
uint c = u*BLOCK_SIZE + col;
if (c < p.ncols_padded) {
ivec2 v = ivec2(c, floatBitsToInt(data_a[row_offset + c]));
ivec2 v = ivec2(c, 0);
if (!needs_bounds_check || c < p.ncols) {
v.y = floatBitsToInt(data_a[row_offset + c]);
}
tmp_idx[idx_offset + c] = v;
}
}
gpttype_adapter.cpp
+1 -1
View File
@@ -1114,7 +1114,7 @@ static speculative_draft_result speculative_decoding_eval_chunk(llama_context *
auto & dp = common_speculative_get_draft_params(draft_spec, 0);
dp.drafting = true;
dp.n_max = n_draft_max;
dp.n_past = n_past;
dp.pos0 = n_past;
dp.id_last = embd[0];
dp.prompt = &prompt_tokens;
dp.result = &drafted_ids;
src/llama-context.cpp
+2
View File
@@ -675,7 +675,9 @@ void llama_context::sched_reserve() {
// need to implement a more robust mechanism that tries a few different inputs and analyzes the results
ggml_cgraph * gf = nullptr;
switch (model.arch) {
case LLM_ARCH_KIMI_LINEAR:
case LLM_ARCH_MINIMAX_01:
// [TAG_RESERVE_DIAG_DECAY]
// the `inp_diag_decay` tensor size scales with `n_seq_tokens^2` which
// makes `n_seqs == 1` use more memory for the compute graph compared to `n_seqs > 1`
gf = graph_reserve(n_tokens, 1, n_outputs_pp, mctx.get(), model.hparams.no_alloc);
src/llama-model.cpp
+3 -3
View File
@@ -2798,9 +2798,9 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
filter = [&](uint32_t il) { return il >= hparams.n_layer(); };
}
if ((arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_HY_V3 || arch == LLM_ARCH_GLM_DSA ||
arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_DEEPSEEK32) &&
hparams.n_layer_nextn > 0) {
// don't filter when n_layer_nextn is repurposed for a router layer the trunk attends
// or when a model is entirely n_layer_nextn layers and has no trunk
if (hparams.n_layer_nextn > 0 && hparams.n_layer() > 0 && hparams.router_layer < 0) {
if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) {
filter = [&](uint32_t il) { return il >= hparams.n_layer(); };
} else {
src/models/gemma3n.cpp
+10 -10
View File
@@ -82,7 +82,7 @@ std::unique_ptr<llm_graph_context> llama_model_gemma3n::build_arch_graph(const l
}
// get 2D slice view from a 3D tensor, the idx corresponds to the 3rd dim
static ggml_tensor * ggml_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) {
static ggml_tensor * gemma3n_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) {
GGML_ASSERT(idx < (int) x->ne[2]);
return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], ggml_row_size(x->type, x->ne[0]),
idx * x->ne[0] * x->ne[1] * ggml_element_size(x));
@@ -139,7 +139,7 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par
ggml_tensor * predictions = altup_predict(cur, il); // [n_embd, n_tokens, n_altup]
// predicted value will go through self-attention and laurel
ggml_tensor * active_prediction = ggml_view_2d_slice(ctx0, predictions, i_altup_act); // [n_embd, n_tokens]
ggml_tensor * active_prediction = gemma3n_view_2d_slice(ctx0, predictions, i_altup_act); // [n_embd, n_tokens]
cur = active_prediction;
cb(cur, "active_prediction", il);
@@ -236,13 +236,13 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par
ggml_tensor * first_prediction; // [n_embd, n_tokens]
{
first_prediction = ggml_view_2d_slice(ctx0, corrected, i_altup_act); // [n_embd, n_tokens]
first_prediction = gemma3n_view_2d_slice(ctx0, corrected, i_altup_act); // [n_embd, n_tokens]
first_prediction = ggml_mul(ctx0, first_prediction, model.layers[il].altup_correct_scale);
first_prediction = build_lora_mm(model.layers[il].per_layer_inp_gate, first_prediction);
first_prediction = ggml_gelu(ctx0, first_prediction); // [n_embd_altup, n_tokens]
cb(first_prediction, "first_prediction_gated", il);
ggml_tensor * inp_this_layer = ggml_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_altup, n_tokens]
ggml_tensor * inp_this_layer = gemma3n_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_altup, n_tokens]
first_prediction = ggml_mul(ctx0, first_prediction, inp_this_layer); // [n_embd_altup, n_tokens]
cb(first_prediction, "first_prediction_scaled", il);
@@ -253,7 +253,7 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par
}
// equivalent to python code: corrected_predictions[1:] += first_prediction
{
ggml_tensor * slice_first = ggml_view_2d_slice(ctx0, corrected, 0);
ggml_tensor * slice_first = gemma3n_view_2d_slice(ctx0, corrected, 0);
ggml_tensor * slice_rest = ggml_view_3d(
ctx0, corrected, n_embd, n_tokens, n_altup - 1, ggml_row_size(corrected->type, n_embd),
ggml_row_size(corrected->type, n_embd * n_tokens), n_embd * n_tokens * ggml_element_size(corrected));
@@ -271,7 +271,7 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par
// cur now has multiple altup(s), we want to merge them back to 1 altup
{
ggml_tensor * target_magnitude = calc_magnitude(ggml_view_2d_slice(ctx0, cur, i_altup_act)); // [n_embd, n_tokens]
ggml_tensor * target_magnitude = calc_magnitude(gemma3n_view_2d_slice(ctx0, cur, i_altup_act)); // [n_embd, n_tokens]
// do a view to skip the first slice (active altup)
ggml_tensor * alt_slice =
ggml_view_3d(ctx0, cur, n_embd, n_tokens, n_altup - 1, ggml_row_size(cur->type, n_embd),
@@ -283,9 +283,9 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par
cb(altup_unembd, "altup_unembd", -1);
// equivalent to torch.mean(hidden_states, dim=0)
cur = ggml_view_2d_slice(ctx0, cur, 0); // [n_embd, n_tokens]
cur = gemma3n_view_2d_slice(ctx0, cur, 0); // [n_embd, n_tokens]
for (int i = 0; i < n_altup - 1; ++i) {
cur = ggml_add(ctx0, cur, ggml_view_2d_slice(ctx0, altup_unembd, i));
cur = ggml_add(ctx0, cur, gemma3n_view_2d_slice(ctx0, altup_unembd, i));
}
cur = ggml_scale(ctx0, cur, 1.0f / float(n_altup)); // [n_embd, n_tokens]
cb(cur, "unembd_merged", -1);
@@ -419,7 +419,7 @@ ggml_tensor * llama_model_gemma3n::graph::altup_compute_router_modalities(ggml_t
// input cur shape: [n_embd, n_tokens, n_altup]
// output shape: [n_embd, n_tokens, n_altup]
ggml_tensor * llama_model_gemma3n::graph::altup_predict(ggml_tensor * cur, int il) {
ggml_tensor * activated = ggml_view_2d_slice(ctx0, cur, i_altup_act); // [n_embd, n_tokens]
ggml_tensor * activated = gemma3n_view_2d_slice(ctx0, cur, i_altup_act); // [n_embd, n_tokens]
ggml_tensor * modalities = altup_compute_router_modalities(activated, il); // [n_altup, n_tokens]
cb(modalities, "modalities", il);
@@ -447,7 +447,7 @@ ggml_tensor * llama_model_gemma3n::graph::altup_correct(ggml_tensor * prediction
ggml_tensor * modalities = altup_compute_router_modalities(activated, il); // [n_altup, n_tokens]
cb(modalities, "modalities", il);
ggml_tensor * active_prediction = ggml_view_2d_slice(ctx0, predictions, i_altup_act);
ggml_tensor * active_prediction = gemma3n_view_2d_slice(ctx0, predictions, i_altup_act);
ggml_tensor * innovation = ggml_sub(ctx0, activated, active_prediction); // [n_embd, n_tokens]
cb(innovation, "innovation", il);
src/models/gemma4.cpp
+6 -6
View File
@@ -145,11 +145,11 @@ std::unique_ptr<llm_graph_context> llama_model_gemma4::build_arch_graph(const ll
}
// get 2D slice view from a 3D tensor, the idx corresponds to the 3rd dim
// static ggml_tensor * ggml_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) {
// GGML_ASSERT(idx < (int) x->ne[2]);
// return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], ggml_row_size(x->type, x->ne[0]),
// idx * x->ne[0] * x->ne[1] * ggml_element_size(x));
// }
static ggml_tensor * gemma4_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) {
GGML_ASSERT(idx < (int) x->ne[2]);
return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], ggml_row_size(x->type, x->ne[0]),
idx * x->ne[0] * x->ne[1] * ggml_element_size(x));
}
llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_params & params) :
llm_graph_context(params),
@@ -372,7 +372,7 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para
cur = build_lora_mm(model.layers[il].per_layer_inp_gate, cur); // [n_embd_per_layer, n_tokens]
cur = ggml_gelu(ctx0, cur);
ggml_tensor * inp_this_layer = ggml_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_per_layer, n_tokens]
ggml_tensor * inp_this_layer = gemma4_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_per_layer, n_tokens]
// TODO @ngxson : improve this
if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) {
src/models/minimax-01.cpp
+1
View File
@@ -229,6 +229,7 @@ llama_model_minimax_01::graph::graph(const llama_model & model, const llm_graph_
ggml_set_input(inp->inp_k_decay);
cb(inp->inp_k_decay, "k_decay_exp", -1);
// [TAG_RESERVE_DIAG_DECAY]
inp->inp_diag_decay = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_seq_tokens, n_seq_tokens, n_head, n_seqs);
ggml_set_input(inp->inp_diag_decay);
cb(inp->inp_diag_decay, "diag_decay_exp", -1);
src/models/plamo2.cpp
+5 -5
View File
@@ -142,6 +142,11 @@ llama_model_plamo2::graph::graph(const llama_model & model, const llm_graph_para
cur = build_plamo2_attn_layer(inp_hybrid->get_attn(), inp_pos, cur, model, il);
}
if (il == n_layer - 1 && inp_out_ids) {
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
residual = ggml_get_rows(ctx0, residual, inp_out_ids);
}
// post_mixer_norm
cur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il);
cb(cur, "attn_post_norm", il);
@@ -167,11 +172,6 @@ llama_model_plamo2::graph::graph(const llama_model & model, const llm_graph_para
cur = build_norm(cur, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, il);
cb(cur, "ffn_post_norm", il);
if (il == n_layer - 1 && inp_out_ids) {
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
residual = ggml_get_rows(ctx0, residual, inp_out_ids);
}
// residual connection
cur = ggml_add(ctx0, cur, residual);
cb(cur, "ffn_residual", il);
src/models/qwen3vl.cpp
+1
View File
@@ -18,6 +18,7 @@ void llama_model_qwen3vl::load_arch_tensors(llama_model_loader &) {
int64_t n_vocab_out = n_vocab;
if (arch == LLM_ARCH_QWEN3TTS) {
// [TAG_LLAMA_N_VOCAB_OUT]
n_vocab_out = 3072;
}
tools/server/server-common.cpp
+147
View File
@@ -15,6 +15,23 @@
#include <limits>
#include <cstring>
#include <type_traits>
#include <chrono>
#include <thread>
#ifdef _WIN32
// windows.h defines min and max as macros, which breaks std::min and std::max
#define WIN32_LEAN_AND_MEAN
#ifndef NOMINMAX
# define NOMINMAX
#endif
#include <windows.h>
#include <io.h>
#else
#include <errno.h>
#include <fcntl.h>
#include <poll.h>
#include <unistd.h>
#endif
json format_error_response(const std::string & message, const enum error_type type) {
std::string type_str;
@@ -1832,3 +1849,133 @@ server_tokens format_prompt_rerank(
return result;
}
//
// server_subproc
//
bool server_subproc::has_output() {
if (out_handle >= 0) {
return true;
}
FILE * f = sproc.stdout_file(); // combined stdout/stderr
if (!f) {
return false;
}
#ifdef _WIN32
HANDLE h = (HANDLE) _get_osfhandle(_fileno(f));
if (h != INVALID_HANDLE_VALUE) {
out_handle = (intptr_t) h;
}
#else
int fd = fileno(f);
if (fd >= 0) {
fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
out_handle = fd;
}
#endif
return out_handle >= 0;
}
int server_subproc::read_output(char * buf, size_t len) {
if (!has_output()) {
return -1;
}
#ifdef _WIN32
HANDLE h = (HANDLE) out_handle;
DWORD avail = 0;
if (!PeekNamedPipe(h, NULL, 0, NULL, &avail, NULL)) {
return -1; // pipe broken, child gone
}
if (avail == 0) {
return 0;
}
DWORD to_read = avail < (DWORD) len ? avail : (DWORD) len;
DWORD got = 0;
if (!ReadFile(h, buf, to_read, &got, NULL) || got == 0) {
return -1;
}
return (int) got;
#else
while (true) {
ssize_t r = read((int) out_handle, buf, len);
if (r > 0) {
return (int) r;
}
if (r == 0) {
return -1; // EOF
}
if (errno == EINTR) {
continue;
}
if (errno == EAGAIN || errno == EWOULDBLOCK) {
return 0;
}
return -1;
}
#endif
}
server_subproc::waiter::waiter() {
#ifndef _WIN32
int fds[2];
GGML_ASSERT(pipe(fds) == 0);
for (int fd : fds) {
fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
}
wake_fd[0] = fds[0];
wake_fd[1] = fds[1];
#endif
}
server_subproc::waiter::~waiter() {
#ifndef _WIN32
close((int) wake_fd[0]);
close((int) wake_fd[1]);
#endif
}
void server_subproc::waiter::wake() {
#ifndef _WIN32
char c = 1;
(void) !write((int) wake_fd[1], &c, 1);
#endif
}
void server_subproc::waiter::wait(const std::vector<server_subproc *> & procs, std::vector<bool> & ready, int64_t timeout_ms) {
ready.assign(procs.size(), false);
#ifdef _WIN32
// no waitable wait exists for anonymous pipes, so poll them in 50 ms steps
bool any = false;
for (size_t i = 0; i < procs.size(); i++) {
DWORD avail = 0;
if (!procs[i]->has_output() || !PeekNamedPipe((HANDLE) procs[i]->out_handle, NULL, 0, NULL, &avail, NULL) || avail > 0) {
ready[i] = true; // data or broken pipe, read_output() tells which
any = true;
}
}
if (!any) {
int64_t step = timeout_ms < 0 ? 50 : std::min<int64_t>(timeout_ms, 50);
std::this_thread::sleep_for(std::chrono::milliseconds(step));
}
#else
std::vector<pollfd> pfds;
pfds.reserve(procs.size() + 1);
pfds.push_back({ (int) wake_fd[0], POLLIN, 0 });
for (auto * p : procs) {
pfds.push_back({ p->has_output() ? (int) p->out_handle : -1, POLLIN, 0 }); // poll() skips negative fds
}
int timeout = timeout_ms < 0 ? -1 : (int) std::min<int64_t>(timeout_ms, std::numeric_limits<int>::max());
int r = poll(pfds.data(), pfds.size(), timeout);
if (r < 0 && errno != EINTR) {
LOG_ERR("%s: poll() failed: %s\n", __func__, strerror(errno));
}
if (pfds[0].revents) {
char buf[64];
while (read((int) wake_fd[0], buf, sizeof(buf)) > 0) {}
}
for (size_t i = 0; i < procs.size(); i++) {
ready[i] = pfds[i + 1].fd < 0 || pfds[i + 1].revents != 0;
}
#endif
}
tools/server/server-common.h
+38
View File
@@ -6,6 +6,7 @@
#include "chat.h"
#include "mtmd.h"
#include "mtmd-helper.h"
#include "subproc.h"
#include "json.h"
@@ -13,6 +14,7 @@
#include <chrono>
#include <condition_variable>
#include <cinttypes>
#include <cstdio>
#include <functional>
#include <mutex>
#include <queue>
@@ -611,3 +613,39 @@ struct server_pipe {
return true;
}
};
// wrapper around common_subproc to manage a child server process
// mainly used by router mode
struct server_subproc {
common_subproc sproc;
std::atomic<bool> stopped{false}; // set by the monitor once the process exited and was reaped
bool is_alive() { return sproc.alive(); }
void terminate() { sproc.terminate(); }
int join() { return sproc.join(); }
// true if the child's combined stdout/stderr pipe is available (call after create())
bool has_output();
// non-blocking read
// returns the number of bytes read, 0 when nothing is available, -1 when the pipe is closed or broken
int read_output(char * buf, size_t len);
// wait until one of a set of children has output, wake() is called, or a timeout passes
struct waiter {
waiter();
~waiter();
// thread-safe; on Windows this is a no-op, wait() returns within 50 ms anyway
void wake();
// timeout_ms < 0 waits until data or wake(); ready[i] is set for each child with data (or a broken pipe)
void wait(const std::vector<server_subproc *> & procs, std::vector<bool> & ready, int64_t timeout_ms);
private:
intptr_t wake_fd[2] = { -1, -1 }; // POSIX self-pipe
};
private:
intptr_t out_handle = -1; // fd on POSIX, HANDLE on Windows; taken lazily from sproc
};
tools/server/server-context.cpp
+1 -1
View File
@@ -3028,7 +3028,7 @@ private:
common_speculative_get_draft_params(spec.get(), slot.id) = {
/* .drafting = */ true,
/* .n_max = */ n_draft_max,
/* .n_past = */ slot.prompt.n_tokens(),
/* .pos0 = */ slot.prompt.tokens.pos_next(),
/* .id_last = */ slot.sampled,
/* .prompt = */ &slot.spec_prompt,
/* .result = */ &slot.spec_draft,
tools/server/server-models.cpp
+278 -201
View File
@@ -44,30 +44,215 @@ extern char **environ;
#define CMD_ROUTER_TO_CHILD_EXIT "cmd_router_to_child:exit"
#define CMD_CHILD_TO_ROUTER_STATE "cmd_child_to_router:state:" // followed by json string
// note: SIGPIPE is ignored by the server
static void request_child_exit(server_subproc & proc) {
FILE * stdin_file = proc.sproc.stdin_file();
if (stdin_file) {
fprintf(stdin_file, "%s\n", CMD_ROUTER_TO_CHILD_EXIT);
fflush(stdin_file);
}
}
// address for child process, this is needed because router may run on 0.0.0.0
// ref: https://github.com/ggml-org/llama.cpp/issues/17862
#define CHILD_ADDR "127.0.0.1"
struct server_subproc {
common_subproc sproc; // not yet spawned while in DOWNLOADING state
std::atomic<bool> stopped{false}; // set to cancel a download or signal child process exit
bool is_alive() {
return sproc.alive();
// single-threaded, watching all child processes at once
struct server_monitor {
server_monitor(server_models & models) : models(models) {
th = std::thread([this]() { run(); });
}
void request_exit() {
FILE * stdin_file = sproc.stdin_file();
if (stdin_file) {
fprintf(stdin_file, "%s\n", CMD_ROUTER_TO_CHILD_EXIT);
fflush(stdin_file);
~server_monitor() {
push({ cmd_t::QUIT, {}, "", 0, false });
th.join();
}
// thread-safe
void watch(const std::string & name, std::shared_ptr<server_subproc> proc, server_child_mode mode, int port) {
child_t c;
c.name = name;
c.proc = std::move(proc);
c.mode = mode;
c.port = port;
if (!c.proc->has_output()) {
SRV_ERR("failed to get stdout/stderr of child process for name=%s\n", name.c_str());
c.eof = true;
}
stopped.store(true, std::memory_order_relaxed);
push({ cmd_t::WATCH, std::move(c), "", 0, false });
}
void terminate() {
sproc.terminate();
// thread-safe
void stop(const std::string & name, int stop_timeout, bool send_exit) {
push({ cmd_t::STOP, {}, name, stop_timeout, send_exit });
}
private:
struct child_t {
std::string name;
std::shared_ptr<server_subproc> proc;
server_child_mode mode = SERVER_CHILD_MODE_NORMAL;
int port = 0;
std::string buf; // partial line
bool eof = false; // output closed, waiting for the process to be reaped
int64_t deadline = 0; // force-kill time in ms, 0 when no stop is pending
};
struct cmd_t {
enum { WATCH, STOP, QUIT } type;
child_t child;
std::string name;
int stop_timeout;
bool send_exit;
};
void push(cmd_t && cmd) {
{
std::lock_guard<std::mutex> lk(mu);
cmds.push_back(std::move(cmd));
}
waiter.wake();
}
// returns true if the loop should exit
bool handle_commands() {
std::deque<cmd_t> batch;
{
std::lock_guard<std::mutex> lk(mu);
batch.swap(cmds);
}
for (auto & cmd : batch) {
switch (cmd.type) {
case cmd_t::WATCH:
children.push_back(std::move(cmd.child));
break;
case cmd_t::STOP:
// the newest child with this name is the one the registry knows
for (auto it = children.rbegin(); it != children.rend(); ++it) {
if (it->name != cmd.name) {
continue;
}
if (cmd.send_exit && !it->eof) {
request_child_exit(*it->proc);
}
it->deadline = ggml_time_ms() + (int64_t) cmd.stop_timeout * 1000;
break;
}
break;
case cmd_t::QUIT:
return true;
}
}
return false;
}
// read what the child wrote, forward complete lines
void read_output(child_t & c) {
char chunk[4096];
while (!c.eof) {
int n = c.proc->read_output(chunk, sizeof(chunk));
if (n < 0) {
c.eof = true;
break;
}
if (n == 0) {
break;
}
c.buf.append(chunk, (size_t) n);
size_t start = 0;
while (true) {
size_t nl = c.buf.find('\n', start);
if (nl == std::string::npos) {
break;
}
std::string line = c.buf.substr(start, nl + 1 - start);
start = nl + 1;
on_line(c, line);
}
c.buf.erase(0, start);
if (c.buf.size() > max_line) {
c.buf.clear(); // a child that never writes a newline must not grow this without bound
}
}
if (c.eof && !c.buf.empty()) {
on_line(c, c.buf);
c.buf.clear();
}
}
void on_line(child_t & c, const std::string & line) {
if (string_starts_with(line, CMD_CHILD_TO_ROUTER_STATE)) {
LOG_DBG("[%5d] %s", c.port, line.c_str()); // prevent spamming the log
models.handle_child_state(c.name, line);
} else {
LOG("[%5d] %s", c.port, line.c_str()); // forward log
}
}
void run() {
while (true) {
if (handle_commands()) {
return;
}
// wait for output, a wakeup, or the next deadline;
// a child whose output closed is polled for its exit every 50 ms
int64_t now = ggml_time_ms();
int64_t timeout = -1;
for (const auto & c : children) {
if (c.eof) {
timeout = timeout < 0 ? 50 : std::min<int64_t>(timeout, 50);
}
if (c.deadline) {
int64_t d = std::max<int64_t>(0, c.deadline - now);
timeout = timeout < 0 ? d : std::min(timeout, d);
}
}
std::vector<server_subproc *> procs;
std::vector<child_t *> owners;
for (auto & c : children) {
if (!c.eof) {
procs.push_back(c.proc.get());
owners.push_back(&c);
}
}
std::vector<bool> ready;
waiter.wait(procs, ready, timeout);
for (size_t i = 0; i < owners.size(); i++) {
if (ready[i]) {
read_output(*owners[i]);
}
}
// deadlines and exits
now = ggml_time_ms();
for (auto it = children.begin(); it != children.end();) {
if (it->deadline && now >= it->deadline && !it->proc->stopped.load(std::memory_order_acquire)) {
SRV_WRN("force-killing model instance name=%s after timeout\n", it->name.c_str());
it->proc->terminate();
it->deadline = 0;
}
if (it->eof && !it->proc->is_alive()) {
int exit_code = it->proc->join();
it->proc->stopped.store(true, std::memory_order_release);
models.on_child_exit(it->name, it->proc, it->mode, exit_code);
SRV_INF("instance name=%s exited with status %d\n", it->name.c_str(), exit_code);
it = children.erase(it);
} else {
++it;
}
}
}
}
static constexpr size_t max_line = 1024 * 1024;
server_models & models;
std::mutex mu;
std::deque<cmd_t> cmds;
std::vector<child_t> children; // monitor thread only
server_subproc::waiter waiter;
std::thread th;
};
struct server_lru_sched {
@@ -395,7 +580,8 @@ server_models::server_models(
base_params(params),
base_env(get_environment()),
base_preset(ctx_preset.load_from_args(argc, argv)),
sched(std::make_unique<server_lru_sched>(*this)) {
sched(std::make_unique<server_lru_sched>(*this)),
monitor(std::make_unique<server_monitor>(*this)) {
// clean up base preset
unset_reserved_args(base_preset, true);
// set binary path
@@ -412,6 +598,10 @@ server_models::server_models(
server_models::~server_models() = default;
void server_models::instance_t::request_exit() const {
request_child_exit(*subproc);
}
void server_models::add_model(server_model_meta && meta) {
if (mapping.find(meta.name) != mapping.end()) {
throw std::runtime_error(string_format("model '%s' appears multiple times", meta.name.c_str()));
@@ -466,7 +656,6 @@ void server_models::add_model(server_model_meta && meta) {
std::string name = meta.name;
mapping[name] = instance_t{
/* subproc */ std::make_shared<server_subproc>(),
/* th */ std::thread(),
/* meta */ std::move(meta)
};
}
@@ -621,9 +810,7 @@ void server_models::load_models() {
};
// Phase 2: acquire the lock once for all mapping mutations.
// We temporarily release it only when calling functions that acquire it internally
// (unload, load) or when joining threads (the monitoring thread calls update_status
// which locks the mutex, so joining while holding it would deadlock).
// We temporarily release it only when calling functions that acquire it internally (unload)
std::unique_lock<std::mutex> lk(mutex);
need_reload = false;
@@ -708,49 +895,15 @@ void server_models::load_models() {
return true;
});
// collect all threads to join in one pass while the lock is held:
// - monitoring threads from just-unloaded models (to_unload)
// - threads of finished downloads (DOWNLOADED), they acquire the mutex on exit
// - threads of already-UNLOADED models that are being removed from source
std::vector<std::thread> threads_to_join;
for (const auto & name : to_unload) {
auto it = mapping.find(name);
if (it != mapping.end() && it->second.th.joinable()) {
threads_to_join.push_back(std::move(it->second.th));
}
}
for (auto & [name, inst] : mapping) {
if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {
continue; // downloading models are not from config sources, leave them alone
}
if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADED) {
// joining this thread under the lock deadlocks: it locks the mutex on its way out
if (inst.th.joinable()) {
threads_to_join.push_back(std::move(inst.th));
}
continue;
}
if (final_presets.find(name) == final_presets.end() && !inst.meta.is_running() && inst.th.joinable()) {
threads_to_join.push_back(std::move(inst.th));
}
}
// join outside the lock - monitoring thread calls update_status (needs lock)
lk.unlock();
for (auto & th : threads_to_join) th.join();
lk.lock();
// erase models no longer in any source
for (auto it = mapping.begin(); it != mapping.end(); ) {
if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {
++it; // download thread is still busy, skip
} else if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADED) {
// download finished, thread is joined above, safe to erase
GGML_ASSERT(!it->second.th.joinable());
// download finished, safe to erase
it = mapping.erase(it);
} else if (final_presets.find(it->first) == final_presets.end()) {
SRV_INF("(reload) removing model name=%s (no longer in source)\n", it->first.c_str());
GGML_ASSERT(!it->second.th.joinable()); // must have been joined above
it = mapping.erase(it);
} else {
++it;
@@ -976,7 +1129,8 @@ void server_models::load(const std::string & name, const load_options & opts) {
// exceeding models_max. Without this, the window between unload_lru()
// releasing its lock and this lock_guard acquiring allows multiple
// threads to each observe capacity and all proceed to load.
if (base_params.models_max > 0) {
// Download workers do not use models_max slots.
if (opts.mode == SERVER_CHILD_MODE_NORMAL && base_params.models_max > 0) {
size_t count_active = 0;
for (const auto & m : mapping) {
if (m.second.meta.is_running()) {
@@ -1030,117 +1184,12 @@ void server_models::load(const std::string & name, const load_options & opts) {
}
}
// start a thread to manage the child process
// captured variables are guaranteed to be destroyed only after the thread is joined
inst.th = std::thread([
this, name,
child_proc = inst.subproc,
port = inst.meta.port,
stop_timeout = inst.meta.stop_timeout,
child_mode = opts.mode
]() {
FILE * stdin_file = child_proc->sproc.stdin_file();
FILE * stdout_file = child_proc->sproc.stdout_file(); // combined stdout/stderr
std::thread log_thread([&]() {
// read stdout/stderr and forward to main server log
// also handle status report from child process
std::vector<char> vec_buf(128 * 1024); // large buffer for storing info
char * buffer = vec_buf.data();
if (stdout_file) {
while (fgets(buffer, vec_buf.size(), stdout_file) != nullptr) {
std::string str(buffer);
if (string_starts_with(buffer, CMD_CHILD_TO_ROUTER_STATE)) {
LOG_DBG("[%5d] %s", port, buffer); // prevent spamming the log
this->handle_child_state(name, str);
} else {
// forward log
LOG("[%5d] %s", port, buffer);
}
}
} else {
SRV_ERR("failed to get stdout/stderr of child process for name=%s\n", name.c_str());
}
});
std::thread stopping_thread([&]() {
// thread to monitor explicit stop requests; child crash is signalled via child_proc->stopped
auto is_stopping = [this, &name]() {
return this->stopping_models.find(name) != this->stopping_models.end();
};
{
std::unique_lock<std::mutex> lk(this->mutex);
this->cv_stop.wait(lk, [&]() {
return is_stopping() || child_proc->stopped.load(std::memory_order_acquire);
});
}
// child crashed or finished on its own, skip graceful shutdown sequence
if (child_proc->stopped.load(std::memory_order_acquire)) {
return;
}
SRV_INF("stopping model instance name=%s\n", name.c_str());
fprintf(stdin_file, "%s\n", CMD_ROUTER_TO_CHILD_EXIT);
fflush(stdin_file);
int64_t start_time = ggml_time_ms();
while (true) {
std::unique_lock<std::mutex> lk(this->mutex);
if (!is_stopping() || child_proc->stopped.load(std::memory_order_acquire)) {
return;
}
int64_t elapsed = ggml_time_ms() - start_time;
if (elapsed >= stop_timeout * 1000) {
lk.unlock();
SRV_WRN("force-killing model instance name=%s after %d seconds timeout\n", name.c_str(), stop_timeout);
child_proc->terminate();
return;
}
this->cv_stop.wait_for(lk, std::chrono::seconds(1), [&]() {
return !is_stopping() || child_proc->stopped.load(std::memory_order_acquire);
});
}
});
// we reach here when the child process exits (stdout EOF)
// note: we cannot join() prior to this point because it will close stdin_file
if (log_thread.joinable()) {
log_thread.join();
}
child_proc->stopped.store(true, std::memory_order_release);
{
std::lock_guard<std::mutex> lk(this->mutex);
stopping_models.erase(name);
cv_stop.notify_all();
}
if (stopping_thread.joinable()) {
stopping_thread.join();
}
// get the exit code
int exit_code = child_proc->sproc.join();
// update status and exit code
if (child_mode == SERVER_CHILD_MODE_DOWNLOAD) {
// instance will be cleaned up on next load_models() call
} else {
this->update_status(name, {
SERVER_MODEL_STATUS_UNLOADED,
exit_code
});
}
SRV_INF("instance name=%s exited with status %d\n", name.c_str(), exit_code);
});
// clean up old process/thread if exists
// old process should have exited already, but just in case, we clean it up here
{
auto & old_instance = mapping[name];
// old process should have exited already, but just in case, we clean it up here
if (old_instance.subproc && old_instance.subproc->is_alive()) {
auto it = mapping.find(name);
if (it != mapping.end() && it->second.subproc && it->second.subproc->is_alive()) {
SRV_WRN("old process for model name=%s is still alive, this is unexpected\n", name.c_str());
old_instance.subproc->terminate(); // force kill
}
if (old_instance.th.joinable()) {
old_instance.th.join();
it->second.subproc->terminate(); // force kill
}
}
@@ -1148,13 +1197,41 @@ void server_models::load(const std::string & name, const load_options & opts) {
{"status", server_model_status_to_string(inst.meta.status)},
});
auto proc = inst.subproc;
int port = inst.meta.port;
mapping[name] = std::move(inst);
monitor->watch(name, proc, opts.mode, port);
cv.notify_all();
}
void server_models::request_stop(const std::string & name) {
void server_models::request_stop(const std::string & name, bool send_exit) {
auto it = mapping.find(name);
if (it == mapping.end() || stopping_models.count(name)) {
return;
}
stopping_models.insert(name);
cv_stop.notify_all();
monitor->stop(name, it->second.meta.stop_timeout, send_exit);
}
void server_models::on_child_exit(const std::string & name, const std::shared_ptr<server_subproc> & proc, server_child_mode mode, int exit_code) {
{
std::lock_guard<std::mutex> lk(mutex);
stopping_models.erase(name);
auto it = mapping.find(name);
if (it == mapping.end() || it->second.subproc != proc) {
return; // entry erased, or a newer instance took the name
}
}
if (mode == SERVER_CHILD_MODE_DOWNLOAD) {
// instance will be cleaned up on next load_models() call
std::lock_guard<std::mutex> lk(mutex);
cv.notify_all();
} else {
update_status(name, {
SERVER_MODEL_STATUS_UNLOADED,
exit_code
});
}
}
void server_models::unload(const std::string & name) {
@@ -1163,20 +1240,21 @@ void server_models::unload(const std::string & name) {
if (it != mapping.end()) {
if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {
SRV_INF("cancelling download for model name=%s\n", name.c_str());
it->second.subproc->request_exit();
it->second.request_exit();
// for convenience, we wait the status change here
wait(lk, name, [](const server_model_meta & new_meta) {
return new_meta.status != SERVER_MODEL_STATUS_DOWNLOADING;
});
} else if (it->second.meta.is_running()) {
SRV_INF("stopping model instance name=%s\n", name.c_str());
if (it->second.meta.status == SERVER_MODEL_STATUS_LOADING) {
bool loading = it->second.meta.status == SERVER_MODEL_STATUS_LOADING;
if (loading) {
// special case: if model is in loading state, unloading means force-killing it
SRV_WRN("model name=%s is still loading, force-killing\n", name.c_str());
it->second.subproc->terminate();
}
request_stop(name);
// status change will be handled by the managing thread
request_stop(name, !loading);
// status change will be handled by the monitor
} else {
SRV_WRN("model instance name=%s is not running\n", name.c_str());
}
@@ -1184,27 +1262,29 @@ void server_models::unload(const std::string & name) {
}
void server_models::unload_all() {
std::vector<std::thread> to_join;
{
std::lock_guard<std::mutex> lk(mutex);
for (auto & [name, inst] : mapping) {
if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {
SRV_INF("cancelling download for model name=%s\n", name.c_str());
inst.subproc->stopped.store(true, std::memory_order_relaxed);
} else if (inst.meta.is_running()) {
SRV_INF("stopping model instance name=%s\n", name.c_str());
request_stop(name);
// status change will be handled by the managing thread
std::unique_lock<std::mutex> lk(mutex);
for (auto & [name, inst] : mapping) {
if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {
SRV_INF("cancelling download for model name=%s\n", name.c_str());
inst.request_exit();
} else if (inst.meta.is_running()) {
SRV_INF("stopping model instance name=%s\n", name.c_str());
bool loading = inst.meta.status == SERVER_MODEL_STATUS_LOADING;
if (loading) {
inst.subproc->terminate();
}
// moving the thread to join list to avoid deadlock
to_join.push_back(std::move(inst.th));
request_stop(name, !loading);
}
}
for (auto & th : to_join) {
if (th.joinable()) {
th.join();
// wait for every child to exit, the monitor force-kills the ones that ignore the exit command
cv.wait(lk, [this]() {
for (const auto & [name, inst] : mapping) {
if (inst.meta.is_running() || inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {
return false;
}
}
}
return true;
});
}
void server_models::update_status(const std::string & name, const update_status_args & args) {
@@ -1291,18 +1371,18 @@ bool server_models::remove(const std::string & name) {
if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {
// cancel in-flight download
SRV_INF("cancelling download for model name=%s\n", name.c_str());
it->second.subproc->request_exit();
it->second.request_exit();
} else if (it->second.meta.is_running()) {
// stop running instance
SRV_INF("stopping model instance name=%s\n", name.c_str());
stopping_models.insert(name);
if (it->second.meta.status == SERVER_MODEL_STATUS_LOADING) {
bool loading = it->second.meta.status == SERVER_MODEL_STATUS_LOADING;
if (loading) {
it->second.subproc->terminate();
}
cv_stop.notify_all();
request_stop(name, !loading);
}
// wait until the monitoring thread finishes
// wait until the child is gone
wait(lk, name, [](const server_model_meta & meta) {
return meta.status == SERVER_MODEL_STATUS_UNLOADED
|| meta.status == SERVER_MODEL_STATUS_DOWNLOADED;
@@ -1311,8 +1391,7 @@ bool server_models::remove(const std::string & name) {
// re-find after wait - load_models() may have erased the entry during the wait
it = mapping.find(name);
if (it == mapping.end()) {
// load_models() already joined the thread and erased the entry;
// we just need to clean up the cached files on disk
// load_models() already erased the entry; we just need to clean up the cached files on disk
lk.unlock();
bool ok = common_download_remove(name);
SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "partial");
@@ -1320,11 +1399,6 @@ bool server_models::remove(const std::string & name) {
return true;
}
// join before erasing - thread no longer acquires this mutex
if (it->second.th.joinable()) {
it->second.th.join();
}
// remove from disk (best-effort: cancelled downloads may have no cached files)
bool ok = common_download_remove(name);
mapping.erase(name);
@@ -1539,7 +1613,7 @@ void server_models::handle_child_state(const std::string & name, const std::stri
std::lock_guard<std::mutex> lk(mutex);
auto it = mapping.find(name);
if (it != mapping.end()) {
return it->second.subproc->request_exit();
return it->second.request_exit();
}
};
if (result == "download_finished") {
@@ -1713,7 +1787,10 @@ void server_child::notify_to_router(const std::string & state, const json & payl
std::lock_guard<std::mutex> lk(mtx_stdout);
common_log_pause(common_log_main());
fflush(stdout);
fprintf(stdout, "%s%s\n", CMD_CHILD_TO_ROUTER_STATE, safe_json_to_str(data).c_str());
// the router matches the command on a line prefix, so the leading newline
// closes whatever the logger left open on the shared pipe, down to the
// trailing color reset that carries no newline of its own
fprintf(stdout, "\n%s%s\n", CMD_CHILD_TO_ROUTER_STATE, safe_json_to_str(data).c_str());
fflush(stdout);
common_log_resume(common_log_main());
}
tools/server/server-models.h
+18 -8
View File
@@ -9,6 +9,7 @@
#include <mutex>
#include <condition_variable>
#include <thread>
#include <functional>
#include <memory>
#include <optional>
@@ -107,27 +108,29 @@ struct server_model_meta {
};
struct server_models_routes;
struct server_subproc; // defined in server-models.cpp
struct server_lru_sched; // defined in server-models.cpp
struct server_monitor; // defined in server-models.cpp
struct server_models {
friend struct server_models_routes;
friend struct server_lru_sched;
friend struct server_monitor;
private:
struct instance_t {
std::shared_ptr<server_subproc> subproc; // shared between main thread and monitoring thread
std::thread th;
std::shared_ptr<server_subproc> subproc; // shared with the monitor thread
server_model_meta meta;
int req_count = 0; // number of active proxy requests
// ask the child to exit (it handles the command on its stdin, see server_child::setup)
void request_exit() const;
};
std::mutex mutex;
std::condition_variable cv;
std::map<std::string, instance_t> mapping;
// for stopping models
std::condition_variable cv_stop;
// models asked to stop, still counted as running until the monitor records their exit
std::set<std::string> stopping_models;
// set to true while load_models() is executing a reload; load() will wait until clear
@@ -216,9 +219,12 @@ private:
// not thread-safe, caller must hold mutex
void add_model(server_model_meta && meta);
// ask the monitoring thread to stop a running instance
// ask the monitor to stop a running instance; send_exit is false for a child that was already force-killed
// not thread-safe, caller must hold mutex
void request_stop(const std::string & name);
void request_stop(const std::string & name, bool send_exit = true);
// called by the monitor once a child exited and was reaped
void on_child_exit(const std::string & name, const std::shared_ptr<server_subproc> & proc, server_child_mode mode, int exit_code);
// notify SSE clients
void notify_sse(const std::string & event, const std::string & model_id, const json & data = nullptr);
@@ -297,12 +303,16 @@ public:
// handle message sent from server_child::notify_to_router()
// raw input must starts with CMD_CHILD_TO_ROUTER_STATE, followed by a JSON string
// this function is not thread-safe, must be called from instance's monitoring thread
// called from the monitor thread
// payload per state:
// state = loading -> payload = {} (TODO: add progress info)
// state = ready -> payload = model_info (json), or {} if wakeup from sleeping
// state = sleeping -> payload = {}
void handle_child_state(const std::string & name, const std::string & raw_input);
private:
// one thread watching every child; keep last, the destructor joins the thread
std::unique_ptr<server_monitor> monitor;
};
struct server_child {
tools/server/tests/unit/test_completion.py
+6 -1
View File
@@ -394,7 +394,12 @@ def test_completion_unified(n_ctx, n_slots, n_predict_vals, expected_success):
results = parallel_function_calls(tasks)
for res, n_predict, expect_ok in zip(results, n_predict_vals, expected_success):
if expect_ok:
assert res.status_code == 200
# the pool is aborted as a whole, so a request that fits on its own
# is still dropped when the slots overlap, and it says so explicitly
assert res.status_code == 200 or (
res.status_code == 500
and "context size has been exceeded" in res.body["error"]["message"].lower()
)
# note: https://github.com/ggml-org/llama.cpp/pull/18700#issuecomment-3728695581
if res.status_code == 200:
tools/server/tests/unit/test_router.py
+6 -1
View File
@@ -540,13 +540,17 @@ def _wait_for_sse_event(collected: list, event_type: str, model: str, timeout: i
def test_router_download_model():
"""Case 1: download a model, verify SSE events and GET /models."""
"""Case 1: download a model at the model limit, verify SSE events and GET /models."""
global server
server.models_max = 1
server.start()
# Ensure the model is not present before we start
server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}")
# A download worker must not consume or evict a model slot
_load_model_and_wait(MODEL_B, timeout=120)
sse_events: list = []
stop = threading.Event()
sse_ready = threading.Event()
@@ -580,6 +584,7 @@ def test_router_download_model():
# Model should now appear in GET /models
ids = _get_model_ids(is_reload=False)
assert MODEL_DOWNLOAD_ID in ids, f"{MODEL_DOWNLOAD_ID} not found in /models after download"
assert _get_model_status(MODEL_B) == "loaded"
def test_router_delete_model():
vendor/cpp-httplib/httplib.cpp
Vendored
+217 -79
View File
@@ -912,17 +912,42 @@ bool write_websocket_frame(Stream &strm, ws::Opcode opcode,
namespace ws {
namespace impl {
bool read_websocket_frame(Stream &strm, Opcode &opcode,
std::string &payload, bool &fin,
bool expect_masked, size_t max_len) {
// Read first 2 bytes
// Read exactly `size` bytes. Stream::read may return less than asked for -- it
// hands back whatever its buffer already holds -- so every multi-byte field has
// to loop. Reading a 2-byte header with a single read() fails whenever the
// header straddles the read buffer's boundary.
//
// Timeout is reported only when nothing at all was consumed. Once a byte has
// been taken the stream sits mid-field and cannot be resumed, so a timeout
// there is a failure like any other. (When read() fails it always records why,
// so the error belongs to this call and not to an earlier one.)
FrameRead read_exact(Stream &strm, void *buf, size_t size) {
auto p = static_cast<char *>(buf);
size_t total = 0;
while (total < size) {
auto n = strm.read(p + total, size - total);
if (n <= 0) {
auto timed_out = total == 0 && strm.get_error() == Error::Timeout;
return timed_out ? FrameRead::Timeout : FrameRead::Fail;
}
total += static_cast<size_t>(n);
}
return FrameRead::Ok;
}
FrameRead read_websocket_frame(Stream &strm, Opcode &opcode,
std::string &payload, bool &fin,
bool expect_masked, size_t max_len) {
// Read first 2 bytes. This is the only read that may report a timeout: it
// sits on a frame boundary, where nothing has been consumed yet.
uint8_t header[2];
if (strm.read(reinterpret_cast<char *>(header), 2) != 2) { return false; }
FrameRead first = read_exact(strm, header, 2);
if (first != FrameRead::Ok) { return first; }
fin = (header[0] & 0x80) != 0;
// RSV1, RSV2, RSV3 must be 0 when no extension is negotiated
if (header[0] & 0x70) { return false; }
if (header[0] & 0x70) { return FrameRead::Fail; }
opcode = static_cast<Opcode>(header[0] & 0x0F);
bool masked = (header[1] & 0x80) != 0;
@@ -932,46 +957,44 @@ bool read_websocket_frame(Stream &strm, Opcode &opcode,
// MUST have a payload length of 125 bytes or less
bool is_control = (static_cast<uint8_t>(opcode) & 0x08) != 0;
if (is_control) {
if (!fin) { return false; }
if (payload_len > 125) { return false; }
if (!fin) { return FrameRead::Fail; }
if (payload_len > 125) { return FrameRead::Fail; }
}
if (masked != expect_masked) { return false; }
if (masked != expect_masked) { return FrameRead::Fail; }
// Extended payload length
if (payload_len == 126) {
uint8_t ext[2];
if (strm.read(reinterpret_cast<char *>(ext), 2) != 2) { return false; }
if (read_exact(strm, ext, 2) != FrameRead::Ok) { return FrameRead::Fail; }
payload_len = (static_cast<uint64_t>(ext[0]) << 8) | ext[1];
} else if (payload_len == 127) {
uint8_t ext[8];
if (strm.read(reinterpret_cast<char *>(ext), 8) != 8) { return false; }
if (read_exact(strm, ext, 8) != FrameRead::Ok) { return FrameRead::Fail; }
// RFC 6455 Section 5.2: the most significant bit MUST be 0
if (ext[0] & 0x80) { return false; }
if (ext[0] & 0x80) { return FrameRead::Fail; }
payload_len = 0;
for (int i = 0; i < 8; i++) {
payload_len = (payload_len << 8) | ext[i];
}
}
if (payload_len > max_len) { return false; }
if (payload_len > max_len) { return FrameRead::Fail; }
// Read mask key if present
uint8_t mask_key[4] = {0};
if (masked) {
if (strm.read(reinterpret_cast<char *>(mask_key), 4) != 4) { return false; }
if (read_exact(strm, mask_key, 4) != FrameRead::Ok) {
return FrameRead::Fail;
}
}
// Read payload
payload.resize(static_cast<size_t>(payload_len));
if (payload_len > 0) {
size_t total_read = 0;
while (total_read < payload_len) {
auto n = strm.read(&payload[total_read],
static_cast<size_t>(payload_len - total_read));
if (n <= 0) { return false; }
total_read += static_cast<size_t>(n);
}
if (payload_len > 0 &&
read_exact(strm, &payload[0], static_cast<size_t>(payload_len)) !=
FrameRead::Ok) {
return FrameRead::Fail;
}
// Unmask if needed
@@ -981,7 +1004,7 @@ bool read_websocket_frame(Stream &strm, Opcode &opcode,
}
}
return true;
return FrameRead::Ok;
}
} // namespace impl
@@ -1728,7 +1751,9 @@ ssize_t select_impl(socket_t sock, short events, time_t sec,
pfd.events = events;
pfd.revents = 0;
auto timeout = static_cast<int>(sec * 1000 + usec / 1000);
// A negative timeout waits forever, poll's own convention. 0 keeps meaning
// "return immediately", which callers here rely on to probe a socket.
auto timeout = sec < 0 ? -1 : static_cast<int>(sec * 1000 + usec / 1000);
return handle_EINTR([&]() { return poll_wrapper(&pfd, 1, timeout); });
}
@@ -1810,8 +1835,11 @@ private:
bool ensure_readable();
socket_t sock_;
time_t read_timeout_sec_;
time_t read_timeout_usec_;
// Atomic because ws::WebSocket::set_read_timeout() reaches this from another
// thread while a read is in flight -- that is the point of it, for a caller
// holding one connection and wanting control back to send on it.
std::atomic<time_t> read_timeout_sec_;
std::atomic<time_t> read_timeout_usec_;
time_t write_timeout_sec_;
time_t write_timeout_usec_;
time_t max_timeout_msec_;
@@ -2204,12 +2232,10 @@ int getaddrinfo_with_timeout(const char *node, const char *service,
// actually finish before letting the stack frame go. The trade-off is that
// a wedged DNS server can hold this thread for the system resolver timeout
// (~30s by default) past the caller's connection timeout.
struct gaicb request {};
struct gaicb request{};
struct gaicb *requests[1] = {&request};
struct sigevent sevp {};
struct timespec timeout {
timeout_sec, 0
};
struct sigevent sevp{};
struct timespec timeout{timeout_sec, 0};
request.ar_name = node;
request.ar_service = service;
@@ -2948,8 +2974,21 @@ EncodingType encoding_type(const Request &req,
return best;
}
// `content_type` is taken separately because a file-backed response has not
// been given one yet when its coding has to be decided.
EncodingType encoding_type(const Request &req, const Response &res,
const std::string &content_type) {
// The response already names a content coding of its own: a handler serving
// a body it encoded itself (pre-compressed static assets, say), or a mount
// point whose headers name the coding its files are stored in. Applying one
// on top of that would double-encode the body and append a second
// `Content-Encoding` field line.
if (res.has_header("Content-Encoding")) { return EncodingType::None; }
return encoding_type(req, content_type);
}
EncodingType encoding_type(const Request &req, const Response &res) {
return encoding_type(req, res.get_header_value("Content-Type"));
return encoding_type(req, res, res.get_header_value("Content-Type"));
}
std::unique_ptr<compressor> make_compressor(EncodingType type) {
@@ -3677,6 +3716,17 @@ bool is_chunked_transfer_encoding(const Headers &headers) {
return case_ignore::equal(last_coding, "chunked");
}
bool has_conflicting_content_length(const Headers &headers) {
// RFC 9112 §6.3: a message carrying both Transfer-Encoding and a non-zero
// Content-Length is framed ambiguously. The body readers here delimit it by
// the transfer coding and drop Content-Length, while an intermediary may do
// the reverse, so the two disagree on where the body ends and a reused
// connection is desynchronised (request/response smuggling). Content-Length:
// 0 is tolerated for compatibility with existing peers.
return has_header(headers, "Transfer-Encoding") &&
get_header_value_u64(headers, "Content-Length", 0, 0) > 0;
}
template <typename T, typename U>
bool prepare_content_receiver(T &x, int &status,
ContentReceiverWithProgress receiver,
@@ -4035,7 +4085,7 @@ void set_file_content_provider(Response &res,
return true;
});
res.file_content_encoding_ = encoding;
res.content_coding_ = encoding;
}
template <typename T, typename U>
@@ -4361,13 +4411,20 @@ bool parse_range_header(const std::string &s, Ranges &ranges) try {
ssize_t first = -1;
if (!lhs.empty()) {
ssize_t v;
auto res = detail::from_chars(lhs.data(), lhs.data() + lhs.size(), v);
if (res.ec == std::errc{}) { first = v; }
// Reject an overflowing first-byte-pos; treating it as absent (-1)
// would turn the range into a suffix range.
auto res =
detail::from_chars(lhs.data(), lhs.data() + lhs.size(), first);
if (res.ec != std::errc{}) {
all_valid_ranges = false;
return;
}
}
ssize_t last = -1;
if (!rhs.empty()) {
// An overflowing last-byte-pos is past any content length, so keeping
// -1 ("remainder", RFC 9110 14.1.2) is correct here.
ssize_t v;
auto res = detail::from_chars(rhs.data(), rhs.data() + rhs.size(), v);
if (res.ec == std::errc{}) { last = v; }
@@ -6902,7 +6959,7 @@ void Response::set_content(const char *s, size_t n,
auto rng = headers.equal_range("Content-Type");
headers.erase(rng.first, rng.second);
set_header("Content-Type", content_type);
file_content_encoding_ = detail::EncodingType::None;
content_coding_ = detail::EncodingType::None;
}
void Response::set_content(const std::string &s,
@@ -6917,7 +6974,7 @@ void Response::set_content(std::string &&s,
auto rng = headers.equal_range("Content-Type");
headers.erase(rng.first, rng.second);
set_header("Content-Type", content_type);
file_content_encoding_ = detail::EncodingType::None;
content_coding_ = detail::EncodingType::None;
}
void Response::set_content_provider(
@@ -6928,7 +6985,7 @@ void Response::set_content_provider(
if (in_length > 0) { content_provider_ = std::move(provider); }
content_provider_resource_releaser_ = std::move(resource_releaser);
is_chunked_content_provider_ = false;
file_content_encoding_ = detail::EncodingType::None;
content_coding_ = detail::EncodingType::None;
}
void Response::set_content_provider(
@@ -6939,7 +6996,7 @@ void Response::set_content_provider(
content_provider_ = detail::ContentProviderAdapter(std::move(provider));
content_provider_resource_releaser_ = std::move(resource_releaser);
is_chunked_content_provider_ = false;
file_content_encoding_ = detail::EncodingType::None;
content_coding_ = detail::EncodingType::None;
}
void Response::set_chunked_content_provider(
@@ -6950,7 +7007,7 @@ void Response::set_chunked_content_provider(
content_provider_ = detail::ContentProviderAdapter(std::move(provider));
content_provider_resource_releaser_ = std::move(resource_releaser);
is_chunked_content_provider_ = true;
file_content_encoding_ = detail::EncodingType::None;
content_coding_ = detail::EncodingType::None;
}
void Response::set_file_content(const std::string &path,
@@ -7991,12 +8048,19 @@ ssize_t WebSocketSSLStream::read(char *ptr, size_t size) {
needs_readable || (err.code == tls::ErrorCode::SyscallError &&
WSAGetLastError() == WSAETIMEDOUT);
#endif
if (!needs_readable && err.code != tls::ErrorCode::WantWrite) { return -1; }
if (!needs_readable && err.code != tls::ErrorCode::WantWrite) {
error_ = Error::Read;
return -1;
}
if (!(needs_readable ? wait_readable() : wait_writable())) {
error_ = Error::Timeout;
return -1;
}
}
// Out of retries. Recording a reason matters: a caller that reads get_error()
// to tell a timeout from a close would otherwise see whatever the previous
// failure left behind (error_ is never cleared on success).
error_ = Error::Read;
return -1;
}
@@ -8653,9 +8717,10 @@ Server::write_content_with_provider(Stream &strm, const Request &req,
}
} else {
if (res.is_chunked_content_provider_) {
auto type = detail::encoding_type(req, res);
auto compressor = detail::make_compressor(type);
// Use the coding `apply_ranges()` chose when it wrote the headers;
// re-negotiating here would disagree with them, e.g. once a handler's
// own Content-Encoding header suppresses the negotiation.
auto compressor = detail::make_compressor(res.content_coding_);
if (!compressor) {
compressor = detail::make_unique<detail::nocompressor>();
}
@@ -8881,7 +8946,8 @@ bool Server::handle_file_request(Request &req, Response &res) {
auto encoding = detail::EncodingType::None;
if (static_file_compression_) {
content_type = content_type_of();
encoding = static_file_encoding(req, content_type, stat.size());
encoding =
static_file_encoding(req, res, content_type, stat.size());
}
// The ETag names the representation actually sent, so a client that
@@ -9296,8 +9362,10 @@ bool Server::dispatch_request(Request &req, Response &res,
// the ETag, which has to name the representation actually sent, and
// `apply_static_file_compression()` go through this, so the two cannot drift
// apart.
detail::EncodingType Server::static_file_encoding(
const Request &req, const std::string &content_type, size_t length) const {
detail::EncodingType
Server::static_file_encoding(const Request &req, const Response &res,
const std::string &content_type,
size_t length) const {
if (!static_file_compression_) { return detail::EncodingType::None; }
// Nothing to compress, and an empty file already answers with
@@ -9322,14 +9390,14 @@ detail::EncodingType Server::static_file_encoding(
return detail::EncodingType::None;
}
return detail::encoding_type(req, content_type);
return detail::encoding_type(req, res, content_type);
}
// Compresses a file-backed content provider into `res.body` and takes over the
// framing headers. Returns false when the response is left untouched.
bool Server::apply_static_file_compression(const Request &req,
Response &res) const {
auto type = res.file_content_encoding_;
auto type = res.content_coding_;
if (type == detail::EncodingType::None || !res.content_provider_) {
return false;
}
@@ -9353,7 +9421,7 @@ bool Server::apply_static_file_compression(const Request &req,
res.content_provider_success_ = true;
res.content_provider_ = nullptr;
res.content_length_ = 0;
res.file_content_encoding_ = detail::EncodingType::None;
res.content_coding_ = detail::EncodingType::None;
res.set_header("Content-Encoding", detail::encoding_name(type));
res.set_header("Vary", "Accept-Encoding");
@@ -9412,6 +9480,7 @@ void Server::apply_ranges(const Request &req, Response &res,
if (res.content_provider_) {
if (res.is_chunked_content_provider_) {
res.set_header("Transfer-Encoding", "chunked");
res.content_coding_ = type;
if (type != detail::EncodingType::None) {
res.set_header("Content-Encoding", detail::encoding_name(type));
res.set_header("Vary", "Accept-Encoding");
@@ -9568,8 +9637,8 @@ Server::process_request(Stream &strm, const std::string &remote_addr,
// coding is not chunked, which leaves the body length undeterminable. The
// latter must not fall through to the "no body" path, or the body bytes are
// parsed as the next request on a persistent connection.
if (req.has_header("Transfer-Encoding") &&
(req.get_header_value_u64("Content-Length") > 0 ||
if (detail::has_conflicting_content_length(req.headers) ||
(req.has_header("Transfer-Encoding") &&
!detail::is_chunked_transfer_encoding(req.headers))) {
connection_closed = true;
res.status = StatusCode::BadRequest_400;
@@ -9734,7 +9803,7 @@ Server::process_request(Stream &strm, const std::string &remote_addr,
auto ws_strm =
std::unique_ptr<Stream>(new detail::WebSocketSSLStream(
strm.socket(), const_cast<tls::session_t>(req.ssl),
CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND, 0,
CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND, 0,
write_timeout_sec_, write_timeout_usec_));
ws::WebSocket ws(std::move(ws_strm), req, true,
websocket_ping_interval_sec_,
@@ -9744,7 +9813,8 @@ Server::process_request(Stream &strm, const std::string &remote_addr,
}
#endif
// Use WebSocket-specific read timeout instead of HTTP timeout
strm.set_read_timeout(CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND, 0);
strm.set_read_timeout(CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND,
0);
ws::WebSocket ws(strm, req, true, websocket_ping_interval_sec_,
websocket_max_missed_pongs_);
entry.handler(req, ws);
@@ -9808,7 +9878,7 @@ Server::process_request(Stream &strm, const std::string &remote_addr,
detail::set_file_content_provider(
res, mm, content_type,
static_file_encoding(req, content_type, mm->size()));
static_file_encoding(req, res, content_type, mm->size()));
}
}
@@ -10228,8 +10298,12 @@ Result ClientImpl::send_(Request &&req) {
void ClientImpl::prepare_default_headers(Request &r, bool for_stream,
const std::string &ct) {
(void)for_stream;
for (const auto &header : default_headers_) {
if (!r.has_header(header.first)) { r.headers.insert(header); }
// Default headers are meant for the origin and may carry its credentials, so
// keep them off the CONNECT request the proxy reads.
if (r.method != "CONNECT") {
for (const auto &header : default_headers_) {
if (!r.has_header(header.first)) { r.headers.insert(header); }
}
}
// RFC 9110 5.3 recommends sending control data such as Host first, so
@@ -10379,6 +10453,17 @@ ClientImpl::open_stream(const std::string &method, const std::string &path,
return handle;
}
// Same framing check as ClientImpl::process_request(). A HEAD or bodyless
// (204/304) response legitimately carries framing headers with no body.
if (method != "HEAD" &&
handle.response->status != StatusCode::NoContent_204 &&
handle.response->status != StatusCode::NotModified_304 &&
detail::has_conflicting_content_length(handle.response->headers)) {
handle.error = Error::Read;
handle.response.reset();
return handle;
}
handle.body_reader_.stream = handle.stream_;
handle.body_reader_.payload_max_length = payload_max_length_;
@@ -10910,24 +10995,24 @@ bool ClientImpl::write_request(Stream &strm, Request &req,
}
}
if (!basic_auth_password_.empty() || !basic_auth_username_.empty()) {
if (!req.has_header("Authorization")) {
// A CONNECT request is read by the proxy; everything sent through the tunnel
// it opens is read by the origin. Each credential goes only to its own hop.
auto is_connect = req.method == "CONNECT";
if (!is_connect && !req.has_header("Authorization")) {
if (!basic_auth_password_.empty() || !basic_auth_username_.empty()) {
req.headers.insert(make_basic_authentication_header(
basic_auth_username_, basic_auth_password_, false));
}
}
if (!bearer_token_auth_token_.empty()) {
if (!req.has_header("Authorization")) {
} else if (!bearer_token_auth_token_.empty()) {
req.headers.insert(make_bearer_token_authentication_header(
bearer_token_auth_token_, false));
}
}
// Proxy-Authorization is only sent when the proxy is actually used for
// this target — otherwise NO_PROXY-matched requests would leak proxy
// credentials directly to the destination server.
if (is_proxy_enabled_for_host(host_)) {
// Proxy-Authorization is only sent when the proxy reads this message —
// otherwise NO_PROXY-matched requests, and requests inside a TLS tunnel,
// would leak proxy credentials to the destination server.
if (is_proxy_enabled_for_host(host_) && (!is_ssl() || is_connect)) {
if (!proxy_basic_auth_username_.empty() &&
!proxy_basic_auth_password_.empty() &&
!req.has_header("Proxy-Authorization")) {
@@ -11323,6 +11408,17 @@ bool ClientImpl::process_request(Stream &strm, Request &req,
// Body
if ((res.status != StatusCode::NoContent_204) && req.method != "HEAD" &&
req.method != "CONNECT") {
// Reject ambiguous framing (RFC 9112 §6.3). Unlike a request, a response
// whose final transfer coding is not chunked is not ambiguous: its body
// runs until the server closes the connection, so it is not rejected.
// HEAD/204 are excluded above and a 304 carries no body.
if (res.status != StatusCode::NotModified_304 &&
detail::has_conflicting_content_length(res.headers)) {
error = Error::Read;
output_error_log(error, &req);
return false;
}
auto redirect = 300 < res.status && res.status < 400 &&
res.status != StatusCode::NotModified_304 &&
follow_location_;
@@ -17562,8 +17658,16 @@ ReadResult WebSocket::read(std::string &msg) {
std::string payload;
bool fin;
if (!impl::read_websocket_frame(strm_, opcode, payload, fin, is_server_,
CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH)) {
impl::FrameRead r =
impl::read_websocket_frame(strm_, opcode, payload, fin, is_server_,
CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH);
// A timeout landed on a frame boundary: the connection is untouched and
// still usable, so hand control back without closing it. That is only
// useful to a caller who asked for the timeout; the compile-time default
// is a backstop against a peer gone quiet, and elapsing it closes the
// connection so a plain `while (ws.read(msg))` loop ends.
if (r == impl::FrameRead::Timeout && read_timeout_set_) { return Timeout; }
if (r != impl::FrameRead::Ok) {
closed_ = true;
return Fail;
}
@@ -17600,9 +17704,14 @@ ReadResult WebSocket::read(std::string &msg) {
Opcode cont_opcode;
std::string cont_payload;
bool cont_fin;
if (!impl::read_websocket_frame(
// A timeout is not reportable here: half of a fragmented message is
// already in `msg` and read() has no way to resume it, so it is a
// failure like any other. Timeouts are only ever seen on a message
// boundary.
if (impl::read_websocket_frame(
strm_, cont_opcode, cont_payload, cont_fin, is_server_,
CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH)) {
CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH) !=
impl::FrameRead::Ok) {
closed_ = true;
return Fail;
}
@@ -17696,7 +17805,8 @@ void WebSocket::close(CloseStatus status, const std::string &reason) {
Opcode op;
std::string resp;
bool fin;
while (impl::read_websocket_frame(strm_, op, resp, fin, is_server_, 125)) {
while (impl::read_websocket_frame(strm_, op, resp, fin, is_server_, 125) ==
impl::FrameRead::Ok) {
if (op == Opcode::Close) { break; }
}
}
@@ -17741,6 +17851,15 @@ const Request &WebSocket::request() const { return req_; }
bool WebSocket::is_open() const { return !closed_; }
void WebSocket::set_read_timeout(time_t sec, time_t usec) {
// 0 waits forever here, as it does for SO_RCVTIMEO. The stream waits with
// poll(), where 0 would instead mean "return immediately", so hand it the
// negative poll uses for an unbounded wait.
if (sec == 0 && usec == 0) { sec = -1; }
strm_.set_read_timeout(sec, usec);
read_timeout_set_ = true;
}
// WebSocketClient implementation
WebSocketClient::WebSocketClient(
const std::string &scheme_host_port_path, const Headers &headers)
@@ -17843,6 +17962,16 @@ void WebSocketClient::shutdown_and_close() {
bool WebSocketClient::create_stream(std::unique_ptr<Stream> &strm,
Error &error, int &ssl_error,
uint64_t &ssl_backend_error) {
// A read timeout of 0 means "wait forever", the way SO_RCVTIMEO reads it.
// The streams wait with poll(), where 0 instead means "return immediately",
// so they are given the negative poll uses for an unbounded wait.
auto unbounded = read_timeout_sec_ == 0 && read_timeout_usec_ == 0;
time_t strm_read_sec = unbounded ? -1 : read_timeout_sec_;
time_t strm_read_usec = unbounded ? 0 : read_timeout_usec_;
// The handshake belongs to establishing the connection, so an unset read
// timeout leaves it bounded by the connection timeout instead of forever.
time_t hs_sec = unbounded ? connection_timeout_sec_ : read_timeout_sec_;
time_t hs_usec = unbounded ? connection_timeout_usec_ : read_timeout_usec_;
#ifdef CPPHTTPLIB_SSL_ENABLED
if (is_ssl_) {
// A plain flag rather than SSLClient::load_certs()'s call_once: connect()
@@ -17862,8 +17991,8 @@ bool WebSocketClient::create_stream(std::unique_ptr<Stream> &strm,
detail::ClientTlsSessionError tls_error;
if (!detail::setup_client_tls_session(host_, tls_ctx_, tls_session_, sock_,
server_certificate_verification_,
read_timeout_sec_, read_timeout_usec_,
&tls_error, options)) {
hs_sec, hs_usec, &tls_error,
options)) {
error = tls_error.error;
ssl_error = tls_error.ssl_error;
ssl_backend_error = tls_error.backend_error;
@@ -17871,17 +18000,19 @@ bool WebSocketClient::create_stream(std::unique_ptr<Stream> &strm,
}
strm = std::unique_ptr<Stream>(new detail::WebSocketSSLStream(
sock_, tls_session_, read_timeout_sec_, read_timeout_usec_,
write_timeout_sec_, write_timeout_usec_));
sock_, tls_session_, strm_read_sec, strm_read_usec, write_timeout_sec_,
write_timeout_usec_));
return true;
}
#else
(void)error;
(void)ssl_error;
(void)ssl_backend_error;
(void)hs_sec;
(void)hs_usec;
#endif
strm = std::unique_ptr<Stream>(
new detail::SocketStream(sock_, read_timeout_sec_, read_timeout_usec_,
new detail::SocketStream(sock_, strm_read_sec, strm_read_usec,
write_timeout_sec_, write_timeout_usec_));
return true;
}
@@ -17951,6 +18082,9 @@ Result WebSocketClient::connect() {
ws_ = std::unique_ptr<WebSocket>(new WebSocket(std::move(strm), req, false,
websocket_ping_interval_sec_,
websocket_max_missed_pongs_));
// The stream was created with the timeout already; tell the WebSocket
// whether it came from the caller, so read() knows to report it as Timeout.
ws_->read_timeout_set_ = read_timeout_set_;
return Result{Error::Success, upgrade.status, std::move(upgrade.headers)};
}
@@ -17983,6 +18117,10 @@ const std::string &WebSocketClient::subprotocol() const {
void WebSocketClient::set_read_timeout(time_t sec, time_t usec) {
read_timeout_sec_ = sec;
read_timeout_usec_ = usec;
read_timeout_set_ = true;
// The members above only seed the next connect(); read() consults the
// stream, so an already-open connection has to be told directly.
if (ws_) { ws_->set_read_timeout(sec, usec); }
}
void WebSocketClient::set_write_timeout(time_t sec, time_t usec) {
vendor/cpp-httplib/httplib.h
Vendored
+80 -12
View File
@@ -8,8 +8,8 @@
#ifndef CPPHTTPLIB_HTTPLIB_H
#define CPPHTTPLIB_HTTPLIB_H
#define CPPHTTPLIB_VERSION "0.54.1"
#define CPPHTTPLIB_VERSION_NUM "0x003601"
#define CPPHTTPLIB_VERSION "0.56.0"
#define CPPHTTPLIB_VERSION_NUM "0x003800"
#ifdef _WIN32
#if defined(_WIN32_WINNT) && _WIN32_WINNT < 0x0A00
@@ -215,8 +215,36 @@
#define CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH 16777216
#endif
#ifndef CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND
#define CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND 300
// One macro used to set the read timeout for both sides. They want different
// defaults: a client's read timeout is the caller's own tool (it waits forever
// until asked not to), while a server keeps a ceiling that reclaims a worker
// from a peer that has gone quiet. The old name still works and sets both.
#ifdef CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND
#pragma message( \
"CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND is deprecated; define " \
"CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND and/or " \
"CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND instead")
#ifndef CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND
#define CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND \
CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND
#endif
#ifndef CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND
#define CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND \
CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND
#endif
#endif
// 0 waits forever. A read timeout is how a caller gets control back to send on
// the same connection; it is not a liveness check (that is ping/pong). Only a
// timeout set at runtime through set_read_timeout() is reported as
// ws::Timeout; when one of these compile-time defaults elapses, read() returns
// ws::Fail and closes the connection.
#ifndef CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND
#define CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND 0
#endif
#ifndef CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND
#define CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND 300
#endif
#ifndef CPPHTTPLIB_WEBSOCKET_CLOSE_TIMEOUT_SECOND
@@ -1817,10 +1845,12 @@ struct Response {
std::string file_content_path_;
std::string file_content_content_type_;
// Content coding chosen for a file-backed content provider, decided once
// where the file is opened so that the ETag and the body cannot disagree.
// `EncodingType::None` for every other kind of response.
detail::EncodingType file_content_encoding_ = detail::EncodingType::None;
// Content coding chosen for the response body, decided once so that the
// headers and the body cannot disagree: where the file is opened for a
// file-backed content provider (keeping the ETag honest), and in
// `apply_ranges()` for a chunked content provider. `EncodingType::None`
// for every other kind of response.
detail::EncodingType content_coding_ = detail::EncodingType::None;
};
enum class Error {
@@ -2359,6 +2389,7 @@ private:
bool parse_request_line(const char *s, Request &req) const;
detail::EncodingType static_file_encoding(const Request &req,
const Response &res,
const std::string &content_type,
size_t length) const;
bool apply_static_file_compression(const Request &req, Response &res) const;
@@ -3663,6 +3694,9 @@ ssize_t read_socket(socket_t sock, void *ptr, size_t size, int flags);
EncodingType encoding_type(const Request &req, const std::string &content_type);
EncodingType encoding_type(const Request &req, const Response &res,
const std::string &content_type);
EncodingType encoding_type(const Request &req, const Response &res);
class BufferStream final : public Stream {
@@ -4345,7 +4379,11 @@ enum class CloseStatus : uint16_t {
InternalError = 1011,
};
enum ReadResult : int { Fail = 0, Text = 1, Binary = 2 };
// Timeout is returned only when a read timeout was set and it elapsed before
// any byte of a frame arrived: nothing was consumed and the connection is
// still open, so the caller can send on it and read again. `msg` is left
// untouched, so a `while (ws.read(msg))` loop must not treat it as a message.
enum ReadResult : int { Fail = 0, Text = 1, Binary = 2, Timeout = 3 };
// Result of WebSocketClient::connect(). Truthy only when the WebSocket
// upgrade handshake fully succeeded. On failure error() identifies the
@@ -4405,6 +4443,18 @@ public:
const Request &request() const;
bool is_open() const;
// Bound how long read() waits before returning Timeout. 0 waits forever.
// A server handler owns its connection's timeout this way; a client sets it
// through WebSocketClient. Safe to call while another thread is in read().
//
// Only a timeout set here is reported as Timeout. The compile-time default
// (CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND) is a backstop rather
// than a request for control, so when it elapses read() returns Fail and
// closes the connection, and `while (ws.read(msg))` ends as it always has.
void set_read_timeout(time_t sec, time_t usec = 0);
template <class Rep, class Period>
void set_read_timeout(const std::chrono::duration<Rep, Period> &duration);
private:
friend class httplib::Server;
friend class WebSocketClient;
@@ -4440,6 +4490,10 @@ private:
int max_missed_pongs_;
int unacked_pings_ = 0;
std::atomic<bool> closed_{false};
// Set once the caller has bounded read() through set_read_timeout(). Until
// then the timeout in effect is the compile-time default, and elapsing it
// is a failure that closes the connection, not a Timeout.
std::atomic<bool> read_timeout_set_{false};
std::mutex write_mutex_;
// Owned by whichever thread is parsing frames off strm_. Only one thread
// may do so: read_websocket_frame() reads a payload until it has the whole
@@ -4527,8 +4581,9 @@ private:
bool is_valid_ = false;
socket_t sock_ = INVALID_SOCKET;
std::unique_ptr<WebSocket> ws_;
time_t read_timeout_sec_ = CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND;
time_t read_timeout_sec_ = CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND;
time_t read_timeout_usec_ = 0;
bool read_timeout_set_ = false; // see WebSocket::read_timeout_set_
time_t write_timeout_sec_ = CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_SECOND;
time_t write_timeout_usec_ = CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_USECOND;
time_t websocket_ping_interval_sec_ =
@@ -4560,6 +4615,13 @@ private:
#endif
};
template <class Rep, class Period>
inline void WebSocket::set_read_timeout(
const std::chrono::duration<Rep, Period> &duration) {
detail::duration_to_sec_and_usec(
duration, [&](time_t sec, time_t usec) { set_read_timeout(sec, usec); });
}
template <class Rep, class Period>
inline void WebSocketClient::set_read_timeout(
const std::chrono::duration<Rep, Period> &duration) {
@@ -4586,8 +4648,14 @@ namespace impl {
bool is_valid_utf8(const std::string &s);
bool read_websocket_frame(Stream &strm, Opcode &opcode, std::string &payload,
bool &fin, bool expect_masked, size_t max_len);
// Three states, because a failure that consumed bytes and one that consumed
// none are not the same thing: the first has left the stream in the middle of
// a frame and the connection cannot be reused, the second can just be retried.
enum class FrameRead { Ok, Fail, Timeout };
FrameRead read_websocket_frame(Stream &strm, Opcode &opcode,
std::string &payload, bool &fin,
bool expect_masked, size_t max_len);
} // namespace impl