mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-09-11 07:19:10 +02:00
Merge branch 'upstream' into concedo_experimental
# Conflicts: # .github/workflows/build-android.yml # .github/workflows/build-apple.yml # .github/workflows/build-cpu.yml # .github/workflows/build-cuda-ubuntu.yml # .github/workflows/build-cuda-windows.yml # .github/workflows/build-msys.yml # .github/workflows/build-opencl.yml # .github/workflows/build-openvino.yml # .github/workflows/build-riscv.yml # .github/workflows/build-sanitize.yml # .github/workflows/build-sycl.yml # .github/workflows/build-vulkan.yml # .github/workflows/build-wasm.yml # .github/workflows/build-webgpu.yml # .github/workflows/copilot-setup-steps.yml # .github/workflows/hip-quality-check.yml # .github/workflows/release.yml # .github/workflows/server.yml # CODEOWNERS # build-xcframework.sh # docs/android.md # docs/backend/SYCL.md # docs/build.md # ggml/CMakeLists.txt # ggml/src/ggml-cpu/CMakeLists.txt # ggml/src/ggml-cpu/kleidiai/kleidiai.cpp # ggml/src/ggml-hexagon/ggml-hexagon.cpp # ggml/src/ggml-hexagon/htp-opnode.h # ggml/src/ggml-hexagon/htp/cpy-ops.c # ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c # ggml/src/ggml-hexagon/htp/get-rows-ops.c # ggml/src/ggml-hexagon/htp/htp-ctx.h # ggml/src/ggml-hexagon/htp/htp-ops.h # ggml/src/ggml-hexagon/htp/main.c # ggml/src/ggml-hexagon/htp/matmul-ops.c # ggml/src/ggml-hexagon/htp/matmul-ops.h # ggml/src/ggml-hexagon/htp/set-rows-ops.c # ggml/src/ggml-metal/CMakeLists.txt # ggml/src/ggml-opencl/ggml-opencl.cpp # ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl # ggml/src/ggml-sycl/ggml-sycl.cpp # tests/CMakeLists.txt # tests/test-backend-ops.cpp # tests/test-llama-archs.cpp # tests/test-save-load-state.cpp # tools/mtmd/CMakeLists.txt
This commit is contained in:
Concedo
@@ -413,6 +413,8 @@ add_library(ggml
|
||||
ggml/src/ggml-cpu/kcpp-repackmapper.cpp
|
||||
ggml/src/ggml-cpu/repack.cpp
|
||||
ggml/src/ggml-cpu/repack.h
|
||||
ggml/src/ggml-cpu/iqp.cpp
|
||||
ggml/src/ggml-cpu/iqp.h
|
||||
ggml/src/ggml-cpu/quants.c
|
||||
ggml/src/ggml-cpu/quants.h
|
||||
ggml/src/ggml-cpu/common.h
|
||||
|
||||
@@ -533,6 +533,8 @@ ggml-vec-noavx2.o: ggml/src/ggml-cpu/vec.cpp ggml/src/ggml-cpu/vec.h
|
||||
$(CXX) $(FASTCXXFLAGS) $(SIMPLECFLAGS) $(FAILSAFE_FLAGS) -c $< -o $@
|
||||
ggml-vec-failsafe.o: ggml/src/ggml-cpu/vec.cpp ggml/src/ggml-cpu/vec.h
|
||||
$(CXX) $(FASTCXXFLAGS) $(NONECFLAGS) $(FAILSAFE_FLAGS) -c $< -o $@
|
||||
ggml-iqp.o: ggml/src/ggml-cpu/iqp.cpp ggml/src/ggml-cpu/iqp.h
|
||||
$(CXX) $(CXXFLAGS) -c $< -o $@
|
||||
|
||||
#quants
|
||||
ggml-quants.o: ggml/src/ggml-quants.c ggml/include/ggml.h ggml/src/ggml-quants.h ggml/src/ggml-common.h
|
||||
@@ -771,35 +773,35 @@ clean:
|
||||
rm -vf otherarch/sdcpp/*.o otherarch/sdcpp/*/*.o otherarch/sdcpp/*/*/*.o otherarch/sdcpp/*/*/*/*.o
|
||||
|
||||
# useful tools
|
||||
main: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
main: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-iqp.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
mainvk: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
mainvk: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-iqp.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) $(VULKAN_LIB) -o $@ $(LDFLAGS)
|
||||
fitparams: tools/fit-params/main.cpp tools/fit-params/fit-params.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
fitparams: tools/fit-params/main.cpp tools/fit-params/fit-params.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-iqp.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) $(VULKAN_LIB) -o $@ $(LDFLAGS)
|
||||
sdmain: $(OBJS_SDCOMMON) $(OBJS_SDMAIN) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
sdmain: $(OBJS_SDCOMMON) $(OBJS_SDMAIN) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-iqp.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp kcpp_backend.h kcpp_backend_default.o build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp kcpp_backend.h kcpp_backend_default.o build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-iqp.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
ttsmain: tools/tts/tts.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
ttsmain: tools/tts/tts.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-iqp.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
gguf-split: tools/gguf-split/gguf-split.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o build-info.h clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
gguf-split: tools/gguf-split/gguf-split.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-iqp.o ggml-unops.o llama.o chat.o llama-model.o build-info.h clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
mtmd-cli: tools/mtmd/mtmd-cli.cpp tools/mtmd/clip.cpp common/debug.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
mtmd-cli: tools/mtmd/mtmd-cli.cpp tools/mtmd/clip.cpp common/debug.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-iqp.o ggml-unops.o llama.o chat.o llama-model.o console.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
embedding: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
embedding: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-iqp.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
embeddingvk: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
embeddingvk: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-iqp.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) $(VULKAN_LIB) -o $@ $(LDFLAGS)
|
||||
ttscppmain: otherarch/ttscpp/cli/cli.cpp otherarch/ttscpp/cli/playback.cpp otherarch/ttscpp/cli/playback.h otherarch/ttscpp/cli/write_file.cpp otherarch/ttscpp/cli/write_file.h otherarch/ttscpp/cli/vad.cpp otherarch/ttscpp/cli/vad.h otherarch/ttscpp/src/ttscpp.cpp otherarch/ttscpp/src/ttstokenizer.cpp otherarch/ttscpp/src/ttssampler.cpp otherarch/ttscpp/src/parler_model.cpp otherarch/ttscpp/src/dac_model.cpp otherarch/ttscpp/src/ttsutil.cpp otherarch/ttscpp/src/ttsargs.cpp otherarch/ttscpp/src/ttst5_encoder_model.cpp otherarch/ttscpp/src/phonemizer.cpp otherarch/ttscpp/src/tts_model.cpp otherarch/ttscpp/src/kokoro_model.cpp otherarch/ttscpp/src/dia_model.cpp otherarch/ttscpp/src/orpheus_model.cpp otherarch/ttscpp/src/snac_model.cpp otherarch/ttscpp/src/general_neural_audio_codec.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
ttscppmain: otherarch/ttscpp/cli/cli.cpp otherarch/ttscpp/cli/playback.cpp otherarch/ttscpp/cli/playback.h otherarch/ttscpp/cli/write_file.cpp otherarch/ttscpp/cli/write_file.h otherarch/ttscpp/cli/vad.cpp otherarch/ttscpp/cli/vad.h otherarch/ttscpp/src/ttscpp.cpp otherarch/ttscpp/src/ttstokenizer.cpp otherarch/ttscpp/src/ttssampler.cpp otherarch/ttscpp/src/parler_model.cpp otherarch/ttscpp/src/dac_model.cpp otherarch/ttscpp/src/ttsutil.cpp otherarch/ttscpp/src/ttsargs.cpp otherarch/ttscpp/src/ttst5_encoder_model.cpp otherarch/ttscpp/src/phonemizer.cpp otherarch/ttscpp/src/tts_model.cpp otherarch/ttscpp/src/kokoro_model.cpp otherarch/ttscpp/src/dia_model.cpp otherarch/ttscpp/src/orpheus_model.cpp otherarch/ttscpp/src/snac_model.cpp otherarch/ttscpp/src/general_neural_audio_codec.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-iqp.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
qwen3tts: otherarch/qwen3tts/q3ttsmain.cpp otherarch/qwen3tts/qwen3_tts.cpp otherarch/qwen3tts/text_tokenizer.cpp otherarch/qwen3tts/gguf_loader.cpp otherarch/qwen3tts/tts_transformer.cpp otherarch/qwen3tts/audio_tokenizer_decoder.cpp otherarch/qwen3tts/audio_tokenizer_encoder.cpp otherarch/qwen3tts/coreml_code_predictor_stub.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
qwen3tts: otherarch/qwen3tts/q3ttsmain.cpp otherarch/qwen3tts/qwen3_tts.cpp otherarch/qwen3tts/text_tokenizer.cpp otherarch/qwen3tts/gguf_loader.cpp otherarch/qwen3tts/tts_transformer.cpp otherarch/qwen3tts/audio_tokenizer_decoder.cpp otherarch/qwen3tts/audio_tokenizer_encoder.cpp otherarch/qwen3tts/coreml_code_predictor_stub.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-iqp.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
rpcserver: tools/rpc/rpc-server.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
rpcserver: tools/rpc/rpc-server.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-iqp.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) $(VULKAN_LIB) -o $@ $(LDFLAGS)
|
||||
llamaserver: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
llamaserver: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-iqp.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(LLAMASERVER_CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
llamaservervk: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
llamaservervk: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-iqp.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(LLAMASERVER_CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) $(VULKAN_LIB) -o $@ $(LDFLAGS)
|
||||
|
||||
ggml/src/ggml-vulkan-shaders.cpp: ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
|
||||
@@ -900,7 +902,7 @@ else
|
||||
endif
|
||||
|
||||
# common object files for all libraries
|
||||
KOBOLDCPP_COMMON_OBJS = gpttype_adapter_default.o whispercpp_default.o clip_default.o expose.o chat.o ggml-binops.o ggml-unops.o ggml-backend.o ggml-backend-meta.o ggml-repack.o llama.o llama-model.o embeddings_default.o music_default.o tts_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o $(OBJS) $(OBJS_SDTYPE)
|
||||
KOBOLDCPP_COMMON_OBJS = gpttype_adapter_default.o whispercpp_default.o clip_default.o expose.o chat.o ggml-binops.o ggml-iqp.o ggml-unops.o ggml-backend.o ggml-backend-meta.o ggml-repack.o llama.o llama-model.o embeddings_default.o music_default.o tts_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o $(OBJS) $(OBJS_SDTYPE)
|
||||
|
||||
#generated libraries
|
||||
koboldcpp_default: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml_v3.o ggml_v2.o ggml_v1.o kcpp_backend_default.o ggml-backend-reg_default.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_FULL)
|
||||
@@ -965,17 +967,17 @@ koboldcpp_vulkan_failsafe:
|
||||
endif
|
||||
|
||||
# tools
|
||||
quantize_gguf: tools/quantize/main.cpp tools/quantize/quantize.cpp common/imatrix-loader.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
quantize_gguf: tools/quantize/main.cpp tools/quantize/quantize.cpp common/imatrix-loader.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-iqp.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
|
||||
quantize_gptj: otherarch/tools/gptj_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
quantize_gptj: otherarch/tools/gptj_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-iqp.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
|
||||
quantize_gpt2: otherarch/tools/gpt2_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
quantize_gpt2: otherarch/tools/gpt2_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-iqp.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
|
||||
quantize_neox: otherarch/tools/neox_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
quantize_neox: otherarch/tools/neox_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-iqp.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
|
||||
quantize_mpt: otherarch/tools/mpt_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
quantize_mpt: otherarch/tools/mpt_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-iqp.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
|
||||
quantize_ace: otherarch/acestep/quantize-acestep.cpp tools/mtmd/clip.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
quantize_ace: otherarch/acestep/quantize-acestep.cpp tools/mtmd/clip.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-iqp.o ggml-unops.o llama.o chat.o llama-model.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
|
||||
|
||||
|
||||
|
||||
+2
-2
@@ -438,7 +438,7 @@ void common_log_flush(struct common_log * log) {
|
||||
log->resume();
|
||||
}
|
||||
|
||||
static int common_get_verbosity(enum ggml_log_level level) {
|
||||
int common_log_get_verbosity(enum ggml_log_level level) {
|
||||
switch (level) {
|
||||
case GGML_LOG_LEVEL_DEBUG: return LOG_LEVEL_DEBUG;
|
||||
case GGML_LOG_LEVEL_INFO: return LOG_LEVEL_TRACE;
|
||||
@@ -452,7 +452,7 @@ static int common_get_verbosity(enum ggml_log_level level) {
|
||||
}
|
||||
|
||||
void common_log_default_callback(enum ggml_log_level level, const char * text, void * /*user_data*/) {
|
||||
auto verbosity = common_get_verbosity(level);
|
||||
auto verbosity = common_log_get_verbosity(level);
|
||||
if (verbosity <= common_log_verbosity_thold) {
|
||||
common_log_add(common_log_main(), level, "%s", text);
|
||||
}
|
||||
|
||||
@@ -43,6 +43,8 @@ int common_log_get_verbosity_thold(void);
|
||||
|
||||
void common_log_set_verbosity_thold(int verbosity); // not thread-safe
|
||||
|
||||
int common_log_get_verbosity(enum ggml_log_level level);
|
||||
|
||||
void common_log_default_callback(enum ggml_log_level level, const char * text, void * user_data);
|
||||
|
||||
// the common_log uses an internal worker thread to print/write log messages
|
||||
|
||||
@@ -286,6 +286,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"CogVLMForCausalLM": "cogvlm",
|
||||
"DeepseekOCR2ForCausalLM": "deepseek",
|
||||
"DeepseekOCRForCausalLM": "deepseek",
|
||||
"DeepseekV4ForCausalLM": "deepseek",
|
||||
"Dots3NoteForCausalLM": "dots3",
|
||||
"Dots3NoteForConditionalGeneration": "dots3",
|
||||
"DotsOCRForCausalLM": "dotsocr",
|
||||
|
||||
@@ -578,6 +578,9 @@ class DeepseekV4Model(TextModel):
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
if (name.startswith(("aligner.", "image_"))
|
||||
or name.endswith(".ffn.gate.bias_vl")):
|
||||
return None
|
||||
if name.startswith("mtp."):
|
||||
if not cls.mtp_only:
|
||||
cls._skipped_mtp_tensors += 1
|
||||
@@ -1018,3 +1021,73 @@ class DeepseekV4DSparkModel(DeepseekV4Model):
|
||||
|
||||
self.gguf_writer.add_block_size(self.hparams["dspark_block_size"])
|
||||
self.gguf_writer.add_target_layers([layer + 1 for layer in self.hparams["dspark_target_layer_ids"]])
|
||||
|
||||
|
||||
@ModelBase.register("DeepseekV4ForCausalLM")
|
||||
@ModelBase.example("deepseek-ai/DeepSeek-V4-Flash-Vision-Exp")
|
||||
class DeepseekV4FlashVisionModel(MmprojModel):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
assert self.hparams_vision is not None
|
||||
# no preprocessor_config.json in the repo; normalization is (x/255 - 0.5) / 0.5
|
||||
# ref: inference/image_processor.py (load_image)
|
||||
self.preprocessor_config = {
|
||||
"image_mean": [0.5, 0.5, 0.5],
|
||||
"image_std": [0.5, 0.5, 0.5],
|
||||
**self.preprocessor_config,
|
||||
}
|
||||
|
||||
def get_vision_config(self) -> dict[str, Any] | None:
|
||||
cfg = self.global_config
|
||||
if cfg.get("vision_n_layers", 0) == 0:
|
||||
raise ValueError("DeepseekV4FlashVisionModel requires vision_n_layers > 0 in the model config")
|
||||
return {
|
||||
"num_hidden_layers": cfg["vision_n_layers"],
|
||||
"hidden_size": cfg["vision_dim"],
|
||||
"num_attention_heads": cfg["vision_n_heads"],
|
||||
"intermediate_size": cfg["vision_inter_dim"],
|
||||
"patch_size": cfg["vision_patch_size"],
|
||||
# dynamic resolution; only used for compat / warmup
|
||||
"image_size": cfg["vision_patch_size"] * cfg["vision_downsample_ratio"] * 16,
|
||||
"rope_theta": cfg.get("vision_rope_theta", 10000.0),
|
||||
"downsample_ratio": cfg["vision_downsample_ratio"],
|
||||
"min_pixels": cfg["vision_min_pixels"],
|
||||
}
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
assert self.hparams_vision is not None
|
||||
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.DEEPSEEK4V)
|
||||
# vision RMSNorm eps is the pytorch default, NOT the LLM's rms_norm_eps (1e-20)
|
||||
# ref: inference/vision.py (RMSNorm)
|
||||
self.gguf_writer.add_vision_attention_layernorm_eps(1e-6)
|
||||
self.gguf_writer.add_vision_use_silu(True) # SwiGLU MLP
|
||||
self.gguf_writer.add_vision_projector_scale_factor(self.hparams_vision["downsample_ratio"])
|
||||
self.gguf_writer.add_vision_min_pixels(self.hparams_vision["min_pixels"])
|
||||
# hardcoded on the C++ side (see PROJECTOR_TYPE_DEEPSEEK4V in clip.cpp)
|
||||
# if future models use different values, add GGUF keys for those
|
||||
assert self.global_config["vision_max_n_token"] == 384
|
||||
assert self.global_config["vision_max_wh_ratio"] == 8
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
if not (name.startswith(("vision.", "aligner.", "image_"))):
|
||||
return None
|
||||
return super().filter_tensors(item)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
assert self.hparams_vision is not None
|
||||
if name == "vision.patch_embed.proj.weight":
|
||||
# nn.Linear over flattened (3, p, p) patches == conv2d weight
|
||||
p = self.hparams_vision["patch_size"]
|
||||
data_torch = data_torch.reshape(data_torch.shape[0], 3, p, p)
|
||||
|
||||
if ".mlp.w1." in name:
|
||||
# fused SwiGLU gate+up
|
||||
gate, up = data_torch.chunk(2, dim=0)
|
||||
yield from super().modify_tensors(gate, name.replace("w1", "w1_gate"), bid)
|
||||
yield from super().modify_tensors(up, name.replace("w1", "w1_up"), bid)
|
||||
return
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
@@ -276,6 +276,10 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
|
||||
# ConvTranspose1d kernels: only F16/F32 are implemented, no BF16
|
||||
if new_name.endswith(".conv.weight") and (".up.blk." in new_name or ".dac.blk." in new_name):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
# the code predictor FFN intermediate peaks around 1.5e5, above the F16 range, and mul_mat
|
||||
# casts its input to the weight type
|
||||
if new_name.startswith("a.gen.code.blk.") and new_name.endswith(".ffn_down.weight"):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -2471,6 +2471,12 @@ extern "C" {
|
||||
GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec(
|
||||
const struct ggml_tensor * a);
|
||||
|
||||
// Use finite mask entries as a sparse K/V set. Set 0 to disable.
|
||||
// n_kv_max must bound the number of finite entries in every mask row.
|
||||
GGML_API void ggml_flash_attn_ext_set_n_kv_max(
|
||||
struct ggml_tensor * a,
|
||||
int32_t n_kv_max);
|
||||
|
||||
GGML_API void ggml_flash_attn_ext_add_sinks(
|
||||
struct ggml_tensor * a,
|
||||
struct ggml_tensor * sinks);
|
||||
|
||||
@@ -1131,7 +1131,7 @@ GGML_TABLE_END()
|
||||
#define NGRID_IQ1S 2048
|
||||
#define IQ1S_DELTA 0.125f
|
||||
#define IQ1M_DELTA 0.125f
|
||||
#if defined(GGML_COMMON_IMPL_C)
|
||||
#if defined(GGML_COMMON_IMPL_C) || defined(GGML_COMMON_IMPL_CPP)
|
||||
GGML_TABLE_BEGIN(uint64_t, iq1s_grid, NGRID_IQ1S)
|
||||
0xffffffffffffffff, 0xffffffffffffff01, 0xffffffffffff0000, 0xffffffffffff01ff,
|
||||
0xffffffffffff0101, 0xffffffffff00ff00, 0xffffffffff000000, 0xffffffffff01ffff,
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
#include "ggml-backend-impl.h"
|
||||
#include "ggml-backend.h"
|
||||
#include "traits.h"
|
||||
#include "iqp.h"
|
||||
#include "ggml-cpu-impl.h"
|
||||
#include "ggml-impl.h"
|
||||
#include "quants.h"
|
||||
@@ -1365,6 +1366,13 @@ UseGgmlGemm1:;
|
||||
|
||||
ggml_barrier(params->threadpool);
|
||||
|
||||
// IQ panel gemm (see iqp.h) - must come after the barrier above, it consumes the q8_K rows
|
||||
// of src1 from the work buffer
|
||||
if (ggml_cpu_iqp_supports_mul_mat(dst) && !params->use_ref) {
|
||||
ggml_compute_forward_mul_mat_iqp(params, dst);
|
||||
return;
|
||||
}
|
||||
|
||||
#if GGML_USE_LLAMAFILE
|
||||
if (src1->type != vec_dot_type) {
|
||||
const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;
|
||||
@@ -1582,6 +1590,16 @@ static void ggml_compute_forward_mul_mat_id(
|
||||
char (*atomic_current_chunk)[CACHE_LINE_SIZE] = // [n_as]
|
||||
incr_ptr_aligned(&wdata_cur, CACHE_LINE_SIZE * n_as, CACHE_LINE_SIZE);
|
||||
|
||||
// IQ panel gemm (see iqp.h); per expert eligibility is decided below, but the work buffer is
|
||||
// reserved for the whole node (ggml_graph_plan sizes it without params, use_ref only skips the dispatch)
|
||||
const bool iqp = ggml_cpu_iqp_supports_mul_mat_id(dst) && !params->use_ref;
|
||||
|
||||
char * iqp_panels = NULL;
|
||||
|
||||
if (iqp) {
|
||||
iqp_panels = incr_ptr_aligned(&wdata_cur, nth * ggml_cpu_iqp_scratch_size(dst), 64);
|
||||
}
|
||||
|
||||
GGML_ASSERT(params->wsize >= (size_t)((char *) wdata_cur - (char *) params->wdata));
|
||||
|
||||
if (src1->type != vec_dot_type) {
|
||||
@@ -1653,6 +1671,13 @@ static void ggml_compute_forward_mul_mat_id(
|
||||
continue;
|
||||
}
|
||||
|
||||
if (iqp && ggml_cpu_iqp_mul_mat_id_min_batch(cne1)) {
|
||||
ggml_compute_forward_mul_mat_id_iqp(params, dst, cur_a, cne1, (const int32_t *) &MMID_MATRIX_ROW(cur_a, 0),
|
||||
iqp_panels);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
const char * src0_cur = (const char *) src0->data + cur_a * nb02;
|
||||
const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;
|
||||
const size_t row_size = ggml_row_size(vec_dot_type, ne10);
|
||||
@@ -3694,6 +3719,11 @@ struct ggml_cplan ggml_graph_plan(
|
||||
size_t cur2 = ggml_row_size(vec_dot_type, ggml_nelements(node->src[1]));
|
||||
cur = MAX(cur, cur2);
|
||||
}
|
||||
|
||||
// the IQ panel path needs one scratch panel per thread past the q8_K rows
|
||||
if (ggml_cpu_iqp_supports_mul_mat(node)) {
|
||||
cur = GGML_PAD(cur, 64) + n_tasks * ggml_cpu_iqp_scratch_size(node);
|
||||
}
|
||||
} break;
|
||||
case GGML_OP_MUL_MAT_ID:
|
||||
{
|
||||
@@ -3713,6 +3743,10 @@ struct ggml_cplan ggml_graph_plan(
|
||||
cur += n_as*ids->ne[0]*ids->ne[1]*sizeof(struct mmid_row_mapping) + sizeof(int64_t);
|
||||
// atomic_current_chunk
|
||||
cur += CACHE_LINE_SIZE*n_as + CACHE_LINE_SIZE;
|
||||
// the IQ panel path needs one scratch panel per thread on top of that
|
||||
if (ggml_cpu_iqp_supports_mul_mat_id(node)) {
|
||||
cur += n_tasks * ggml_cpu_iqp_scratch_size(node) + 64;
|
||||
}
|
||||
} break;
|
||||
case GGML_OP_OUT_PROD:
|
||||
{
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,39 @@
|
||||
#pragma once
|
||||
|
||||
#include "ggml-cpu-impl.h"
|
||||
#include "ggml.h"
|
||||
|
||||
// GGML internal header
|
||||
|
||||
// batched mul_mat path for the grid based IQ types: decode 8 src0 rows at a time into per thread scratch
|
||||
// (block_iqp_x8, see iqp.cpp) and run an integer gemm over them against all src1 columns
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
// whether cne1 rows of src1 are enough for the decode to pay for itself, per expert, for MUL_MAT_ID
|
||||
bool ggml_cpu_iqp_mul_mat_id_min_batch(int64_t cne1);
|
||||
|
||||
bool ggml_cpu_iqp_supports_mul_mat(const struct ggml_tensor * dst);
|
||||
|
||||
// node level test only - per expert eligibility is decided with ggml_cpu_iqp_mul_mat_id_min_batch
|
||||
bool ggml_cpu_iqp_supports_mul_mat_id(const struct ggml_tensor * dst);
|
||||
|
||||
// per thread panel scratch bytes, padded
|
||||
size_t ggml_cpu_iqp_scratch_size(const struct ggml_tensor * dst);
|
||||
|
||||
// must be called after src1 has been converted to q8_K into params->wdata and the threads have synchronized on it
|
||||
void ggml_compute_forward_mul_mat_iqp(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
|
||||
// one expert: expert_rows points at its row of the matrix_rows table of (i1, i2) int32 pairs, panels at the base of the per thread panel scratches
|
||||
void ggml_compute_forward_mul_mat_id_iqp(const struct ggml_compute_params * params,
|
||||
struct ggml_tensor * dst,
|
||||
int64_t cur_a,
|
||||
int64_t cne1,
|
||||
const int32_t * expert_rows,
|
||||
void * panels);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
@@ -715,6 +715,9 @@ static __global__ void flash_attn_mask_to_KV_max(
|
||||
KV_max[sequence*ne31 + jt] = KV_max_sj;
|
||||
}
|
||||
|
||||
void ggml_cuda_flash_attn_ext_compact_mask(
|
||||
const ggml_tensor * mask, int32_t * indices, int32_t n_kv_max, cudaStream_t stream);
|
||||
|
||||
template<int D, int ncols1, int ncols2> // D == head size
|
||||
__launch_bounds__(D, 1)
|
||||
static __global__ void flash_attn_stream_k_fixup_uniform(
|
||||
@@ -969,7 +972,8 @@ static __global__ void flash_attn_combine_results(
|
||||
template <int DV, int ncols1, int ncols2>
|
||||
void launch_fattn(
|
||||
ggml_backend_cuda_context & ctx, ggml_tensor * dst, fattn_kernel_t fattn_kernel, const int nwarps, const size_t nbytes_shared,
|
||||
const int nbatch_fa, const bool need_f16_K, const bool need_f16_V, const bool stream_k, const int warp_size = WARP_SIZE
|
||||
const int nbatch_fa, const bool need_f16_K, const bool need_f16_V, const bool stream_k, const bool use_sparse,
|
||||
const int warp_size = WARP_SIZE
|
||||
) {
|
||||
constexpr int ncols = ncols1 * ncols2;
|
||||
|
||||
@@ -1085,10 +1089,20 @@ void launch_fattn(
|
||||
const int ntiles_z_gqa = ((gqa_ratio + ncols2 - 1) / ncols2);
|
||||
const int ntiles_dst = ntiles_x * ntiles_z_gqa * K->ne[2] * Q->ne[3];
|
||||
|
||||
const int32_t n_kv_max = use_sparse ? ggml_get_op_params_i32(KQV, 4) : 0;
|
||||
if (use_sparse) {
|
||||
GGML_ASSERT(mask != nullptr);
|
||||
GGML_ASSERT(n_kv_max > 0);
|
||||
const size_t mask_rows = size_t(mask->ne[1]) * mask->ne[3];
|
||||
|
||||
KV_max.alloc(size_t(n_kv_max) * mask_rows);
|
||||
ggml_cuda_flash_attn_ext_compact_mask(mask, KV_max.ptr, n_kv_max, main_stream);
|
||||
}
|
||||
|
||||
// Optional optimization where the mask is scanned to determine whether part of the calculation can be skipped.
|
||||
// Only worth the overhead if there is at lease one FATTN_KQ_STRIDE x FATTN_KQ_STRIDE square to be skipped or
|
||||
// multiple sequences of possibly different lengths.
|
||||
if (mask && K->ne[1] % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1)) {
|
||||
if (!use_sparse && mask && K->ne[1] % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1)) {
|
||||
const int64_t s31 = mask->nb[1] / sizeof(half2);
|
||||
const int64_t s33 = mask->nb[3] / sizeof(half2);
|
||||
|
||||
@@ -1110,7 +1124,8 @@ void launch_fattn(
|
||||
GGML_ASSERT(max_blocks_per_sm > 0);
|
||||
int parallel_blocks = max_blocks_per_sm;
|
||||
|
||||
const int ntiles_KV = (K->ne[1] + nbatch_fa - 1) / nbatch_fa; // Max. number of parallel blocks limited by KV cache length.
|
||||
const int64_t n_kv = use_sparse ? n_kv_max : K->ne[1];
|
||||
const int ntiles_KV = (n_kv + nbatch_fa - 1) / nbatch_fa; // Max. number of parallel blocks limited by KV cache length.
|
||||
|
||||
dim3 blocks_num;
|
||||
if (stream_k) {
|
||||
@@ -1214,7 +1229,7 @@ void launch_fattn(
|
||||
!stream_k && parallel_blocks > 1 ? dst_tmp.ptr : (float *) KQV->data, dst_tmp_meta.ptr,
|
||||
scale, max_bias, m0, m1, n_head_log2, logit_softcap,
|
||||
Q->ne[0], ne01, Q->ne[2], Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3],
|
||||
K->ne[0], K->ne[1], K->ne[2], K->ne[3], nb11, nb12, nb13,
|
||||
K->ne[0], n_kv, K->ne[2], K->ne[3], nb11, nb12, nb13,
|
||||
nb21, nb22, nb23,
|
||||
mask ? mask->ne[1] : 0, mask ? mask->ne[2] : 0, mask ? mask->ne[3] : 0,
|
||||
mask ? mask->nb[1] : 0, mask ? mask->nb[2] : 0, mask ? mask->nb[3] : 0
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
#include "cp-async.cuh"
|
||||
#include "mma.cuh"
|
||||
#include "fattn-common.cuh"
|
||||
#include "fattn-swizzle.cuh"
|
||||
|
||||
using namespace ggml_cuda_mma;
|
||||
|
||||
@@ -66,7 +67,7 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 32, 128, 2, 32, 96, 64, 64, 2, true);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 64, 128, 2, 32, 96, 64, 64, 2, true);
|
||||
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 8, 64, 4, 64, 128, 128, 128, 2, true);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 8, 128, 2, 64, 128, 128, 128, 2, true);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 64, 4, 32, 128, 128, 128, 2, true);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2, 32, 128, 128, 128, 2, true);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2, 32, 128, 128, 128, 2, true);
|
||||
@@ -349,20 +350,24 @@ static __host__ int ggml_cuda_fattn_mma_get_nstages(const int DKQ, const int DV,
|
||||
return cp_async_available(cc) && ncols2 >= 2 ? ggml_cuda_fattn_mma_get_nstages_target(DKQ, DV, ncols1*ncols2, cc) : 0;
|
||||
}
|
||||
|
||||
static constexpr __device__ int ggml_cuda_fattn_mma_get_nstages(const int DKQ, const int DV, const int ncols1, const int ncols2) {
|
||||
static constexpr __device__ int ggml_cuda_fattn_mma_get_nstages(
|
||||
const int DKQ, const int DV, const int ncols1, const int ncols2, const bool use_sparse) {
|
||||
#ifdef CP_ASYNC_AVAILABLE
|
||||
return ncols2 >= 2 ? ggml_cuda_fattn_mma_get_nstages_target(DKQ, DV, ncols1*ncols2) : 0;
|
||||
const int nstages_target = ncols2 >= 2 ? ggml_cuda_fattn_mma_get_nstages_target(DKQ, DV, ncols1*ncols2) : 0;
|
||||
// sparse gather is not implemented for multi-stage loading
|
||||
return use_sparse && nstages_target > 1 ? 1 : nstages_target;
|
||||
#else
|
||||
GGML_UNUSED_VARS(DKQ, DV, ncols1, ncols2);
|
||||
GGML_UNUSED_VARS(DKQ, DV, ncols1, ncols2, use_sparse);
|
||||
return 0;
|
||||
#endif // CP_ASYNC_AVAILABLE
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
template<int stride_tile, int nwarps, int nbatch_fa, bool use_cp_async, bool oob_check>
|
||||
template<int stride_tile, bool swz, int nwarps, int nbatch_fa, bool use_cp_async, bool oob_check, bool use_sparse>
|
||||
static __device__ __forceinline__ void flash_attn_ext_f16_load_tile(
|
||||
const half2 * const __restrict__ KV, half2 * const __restrict__ tile_KV, const int D2, const int stride_KV, const int i_sup) {
|
||||
const half2 * const __restrict__ KV, half2 * const __restrict__ tile_KV, const int D2, const int stride_KV,
|
||||
const int k_VKQ_0, const int i_sup, const int32_t * const __restrict__ indices) {
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
// K/V data is loaded with decreasing granularity for D for better memory bandwidth.
|
||||
// The minimum granularity is 16 bytes.
|
||||
@@ -370,7 +375,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile(
|
||||
const int chunks_per_row = D2 / h2_per_chunk;
|
||||
if constexpr (use_cp_async) {
|
||||
static_assert(warp_size == 32, "bad warp_size");
|
||||
static_assert(!oob_check, "OOB check not compatible with cp_async");
|
||||
static_assert(!oob_check || use_sparse, "OOB check not compatible with cp_async");
|
||||
constexpr int preload = 64;
|
||||
|
||||
const unsigned int tile_KV_32 = ggml_cuda_cvta_generic_to_shared(tile_KV);
|
||||
@@ -393,11 +398,25 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile(
|
||||
break;
|
||||
}
|
||||
|
||||
int64_t i_KV;
|
||||
if constexpr (use_sparse) {
|
||||
// padded slots gather row 0, the -inf mask removes their contribution
|
||||
const int32_t index = i < i_sup ? indices[k_VKQ_0 + i] : 0;
|
||||
i_KV = index >= 0 ? index : 0;
|
||||
} else {
|
||||
i_KV = k_VKQ_0 + i;
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) {
|
||||
const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k);
|
||||
|
||||
cp_async_cg_16<preload>(tile_KV_32 + i*(stride_tile*sizeof(half2)) + k*16, KV + i*stride_KV + k*h2_per_chunk);
|
||||
if constexpr (swz) {
|
||||
const int smem_offs_b = ggml_cuda_fattn_smem_swizzle::bytes_rc<stride_tile>(i, k*h2_per_chunk);
|
||||
cp_async_cg_16<preload>(tile_KV_32 + smem_offs_b, KV + i_KV*stride_KV + k*h2_per_chunk);
|
||||
} else {
|
||||
cp_async_cg_16<preload>(tile_KV_32 + i*(stride_tile*sizeof(half2)) + k*16, KV + i_KV*stride_KV + k*h2_per_chunk);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -432,8 +451,18 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile(
|
||||
for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) {
|
||||
const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k);
|
||||
|
||||
ggml_cuda_memcpy_1<16>(tile_KV + i*stride_tile + k*4,
|
||||
!oob_check || i < i_sup ? KV + i*stride_KV + k*h2_per_chunk : zero);
|
||||
const half2 * src;
|
||||
if constexpr (use_sparse) {
|
||||
const int32_t index = i < i_sup ? indices[k_VKQ_0 + i] : -1;
|
||||
src = index >= 0 ? KV + int64_t(index)*stride_KV + k*h2_per_chunk : zero;
|
||||
} else {
|
||||
src = !oob_check || i < i_sup ? KV + int64_t(k_VKQ_0 + i)*stride_KV + k*h2_per_chunk : zero;
|
||||
}
|
||||
if constexpr (swz) {
|
||||
ggml_cuda_memcpy_1<16>((char *) tile_KV + ggml_cuda_fattn_smem_swizzle::bytes_rc<stride_tile>(i, k*h2_per_chunk), src);
|
||||
} else {
|
||||
ggml_cuda_memcpy_1<16>(tile_KV + i*stride_tile + k*4, src);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -447,14 +476,16 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile(
|
||||
}
|
||||
}
|
||||
|
||||
template<int ncols1, int nwarps, int nbatch_fa, bool use_cp_async, bool oob_check>
|
||||
template<int ncols1, int nwarps, int nbatch_fa, bool use_cp_async, bool oob_check, bool use_sparse>
|
||||
static __device__ __forceinline__ void flash_attn_ext_f16_load_mask(
|
||||
const half * const __restrict__ mask_h, half * const __restrict__ tile_mask,
|
||||
const int stride_mask, const int i_sup, const int j0, const uint3 ne01) {
|
||||
const int stride_mask, const int k_VKQ_0, const int i_sup, const int j0, const uint3 ne01,
|
||||
const int32_t * const __restrict__ indices) {
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
if constexpr (use_cp_async) {
|
||||
static_assert(nbatch_fa <= 8*warp_size && nbatch_fa % 8 == 0, "bad nbatch_fa");
|
||||
static_assert(!oob_check, "OOB check incompatible with cp_async");
|
||||
static_assert(!use_sparse, "sparse gather incompatible with cp_async");
|
||||
constexpr int preload = nbatch_fa >= 32 ? nbatch_fa * sizeof(half) : 64;
|
||||
constexpr int cols_per_warp = 8*warp_size/nbatch_fa;
|
||||
constexpr int stride_j = nwarps * cols_per_warp;
|
||||
@@ -472,9 +503,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_mask(
|
||||
|
||||
const int i = 8 * (threadIdx.x % (nbatch_fa/8));
|
||||
|
||||
cp_async_cg_16<preload>(tile_mask_32 + j_sram*(nbatch_fa*sizeof(half) + 16) + i*sizeof(half), mask_h + int64_t(j_vram)*stride_mask + i);
|
||||
cp_async_cg_16<preload>(tile_mask_32 + j_sram*(nbatch_fa*sizeof(half) + 16) + i*sizeof(half), mask_h + int64_t(j_vram)*stride_mask + k_VKQ_0 + i);
|
||||
}
|
||||
} else if constexpr (oob_check) {
|
||||
} else if constexpr (oob_check || use_sparse) {
|
||||
#pragma unroll
|
||||
for (int j1 = 0; j1 < ncols1; j1 += nwarps) {
|
||||
const int j_sram = j1 + threadIdx.y;
|
||||
@@ -488,7 +519,12 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_mask(
|
||||
for (int i0 = 0; i0 < nbatch_fa; i0 += warp_size) {
|
||||
const int i = i0 + threadIdx.x;
|
||||
|
||||
tile_mask[j_sram*(nbatch_fa + 8) + i] = i < i_sup ? mask_h[int64_t(j_vram)*stride_mask + i] : half(0.0f);
|
||||
if constexpr (use_sparse) {
|
||||
const int32_t index = i < i_sup ? indices[k_VKQ_0 + i] : -1;
|
||||
tile_mask[j_sram*(nbatch_fa + 8) + i] = index >= 0 ? mask_h[int64_t(j_vram)*stride_mask + index] : half(-INFINITY);
|
||||
} else {
|
||||
tile_mask[j_sram*(nbatch_fa + 8) + i] = i < i_sup ? mask_h[int64_t(j_vram)*stride_mask + k_VKQ_0 + i] : half(0.0f);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if constexpr (nbatch_fa < 2*warp_size) {
|
||||
@@ -505,7 +541,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_mask(
|
||||
|
||||
const int i = threadIdx.x % (warp_size/cols_per_warp);
|
||||
|
||||
ggml_cuda_memcpy_1<sizeof(half2)>(tile_mask + j_sram*(nbatch_fa + 8) + 2*i, mask_h + int64_t(j_vram)*stride_mask + 2*i);
|
||||
ggml_cuda_memcpy_1<sizeof(half2)>(tile_mask + j_sram*(nbatch_fa + 8) + 2*i, mask_h + int64_t(j_vram)*stride_mask + k_VKQ_0 + 2*i);
|
||||
}
|
||||
} else {
|
||||
#pragma unroll
|
||||
@@ -521,20 +557,21 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_mask(
|
||||
for (int i0 = 0; i0 < nbatch_fa; i0 += 2*warp_size) {
|
||||
const int i = i0 + 2*threadIdx.x;
|
||||
|
||||
ggml_cuda_memcpy_1<sizeof(half2)>(tile_mask + j_sram*(nbatch_fa + 8) + i, mask_h + int64_t(j_vram)*stride_mask + i);
|
||||
ggml_cuda_memcpy_1<sizeof(half2)>(tile_mask + j_sram*(nbatch_fa + 8) + i, mask_h + int64_t(j_vram)*stride_mask + k_VKQ_0 + i);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<int DKQ, int DV, int ncols1, int ncols2, int nwarps,
|
||||
bool use_logit_softcap, bool V_is_K_view, bool needs_fixup, bool is_fixup, bool last_iter, bool oob_check,
|
||||
bool use_logit_softcap, bool V_is_K_view, bool use_sparse, bool needs_fixup, bool is_fixup, bool last_iter, bool oob_check,
|
||||
typename T_A_KQ, typename T_B_KQ, typename T_C_KQ, typename T_A_VKQ, typename T_B_VKQ, typename T_C_VKQ>
|
||||
static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
||||
const float2 * const __restrict__ Q_f2,
|
||||
const half2 * const __restrict__ K_h2,
|
||||
const half2 * const __restrict__ V_h2,
|
||||
const half * const __restrict__ mask_h,
|
||||
const int32_t * const __restrict__ indices,
|
||||
float2 * const __restrict__ dstk,
|
||||
float2 * const __restrict__ dstk_fixup,
|
||||
const float scale,
|
||||
@@ -566,11 +603,13 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
||||
constexpr int nbatch_K2 = ggml_cuda_fattn_mma_get_nbatch_K2(DKQ, DV, ncols);
|
||||
constexpr int nbatch_V2 = ggml_cuda_fattn_mma_get_nbatch_V2(DKQ, DV, ncols);
|
||||
constexpr bool Q_in_reg = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols);
|
||||
constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2);
|
||||
constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2, use_sparse);
|
||||
|
||||
constexpr int stride_tile_K = nbatch_K2 + 4;
|
||||
|
||||
constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : nbatch_V2 + 4;
|
||||
// swizzle the tile stride for K and V based on the batch size.
|
||||
constexpr int stride_tile_K = ggml_cuda_fattn_smem_swizzle::tile_stride(nbatch_K2);
|
||||
constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : ggml_cuda_fattn_smem_swizzle::tile_stride(nbatch_V2);
|
||||
constexpr bool swz_K = ggml_cuda_fattn_smem_swizzle::enabled(nbatch_K2);
|
||||
constexpr bool swz_V = V_is_K_view ? swz_K : ggml_cuda_fattn_smem_swizzle::enabled(nbatch_V2);
|
||||
|
||||
const int k_VKQ_0 = kb0 * nbatch_fa;
|
||||
#if defined(TURING_MMA_AVAILABLE)
|
||||
@@ -588,13 +627,14 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
||||
constexpr bool use_cp_async = true;
|
||||
cp_async_wait_all();
|
||||
__syncthreads();
|
||||
flash_attn_ext_f16_load_tile<stride_tile_V, nwarps, nbatch_fa, use_cp_async, oob_check>
|
||||
(V_h2 + int64_t(k_VKQ_0)*stride_V, tile_V, nbatch_V2, stride_V, k_VKQ_sup);
|
||||
flash_attn_ext_f16_load_tile<stride_tile_V, swz_V, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
|
||||
(V_h2, tile_V, nbatch_V2, stride_V, k_VKQ_0, k_VKQ_sup, nullptr);
|
||||
} else {
|
||||
constexpr bool use_cp_async = nstages == 1;
|
||||
// the sparse mask values are gathered per element, always load them synchronously
|
||||
constexpr bool use_cp_async = nstages == 1 && !use_sparse;
|
||||
if (ncols2 > 1 || mask_h) {
|
||||
flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check>
|
||||
(mask_h + k_VKQ_0, tile_mask, stride_mask, k_VKQ_sup, jt*ncols1, ne01);
|
||||
flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
|
||||
(mask_h, tile_mask, stride_mask, k_VKQ_0, k_VKQ_sup, jt*ncols1, ne01, indices);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -607,8 +647,8 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
||||
if constexpr (nstages <= 1) {
|
||||
const int k0_diff = k0_stop - k0_start;
|
||||
constexpr bool use_cp_async = nstages == 1;
|
||||
flash_attn_ext_f16_load_tile<stride_tile_K, nwarps, nbatch_fa, use_cp_async, oob_check>
|
||||
(K_h2 + int64_t(k_VKQ_0)*stride_K + k0_start, tile_K, k0_diff, stride_K, k_VKQ_sup);
|
||||
flash_attn_ext_f16_load_tile<stride_tile_K, swz_K, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
|
||||
(K_h2 + k0_start, tile_K, k0_diff, stride_K, k_VKQ_0, k_VKQ_sup, indices);
|
||||
if (use_cp_async) {
|
||||
cp_async_wait_all();
|
||||
}
|
||||
@@ -623,7 +663,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
||||
#pragma unroll
|
||||
for (int k_KQ_0 = k0_start; k_KQ_0 < k0_stop; k_KQ_0 += T_A_KQ::J) {
|
||||
T_A_KQ K_A;
|
||||
load_ldmatrix(K_A, tile_K + i_KQ_0*stride_tile_K + (k_KQ_0 - k0_start), stride_tile_K);
|
||||
ggml_cuda_fattn_smem_swizzle::load_ldmatrix<stride_tile_K, swz_K>(K_A, tile_K, i_KQ_0, k_KQ_0 - k0_start);
|
||||
if constexpr (cols_per_warp == 8) {
|
||||
mma(KQ_C[i_KQ_00/(np*T_A_KQ::I)], K_A, Q_B[k_KQ_0/T_A_KQ::J]);
|
||||
} else {
|
||||
@@ -649,7 +689,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
||||
const int i_KQ_0 = i_KQ_00 + (threadIdx.y % np)*T_A_KQ::I;
|
||||
|
||||
T_A_KQ K_A;
|
||||
load_ldmatrix(K_A, tile_K + i_KQ_0*stride_tile_K + (k_KQ_0 - k0_start), stride_tile_K);
|
||||
ggml_cuda_fattn_smem_swizzle::load_ldmatrix<stride_tile_K, swz_K>(K_A, tile_K, i_KQ_0, k_KQ_0 - k0_start);
|
||||
|
||||
if constexpr (cols_per_warp == 8) {
|
||||
mma(KQ_C[i_KQ_00/(np*T_A_KQ::I)], K_A, Q_B[0]);
|
||||
@@ -933,6 +973,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
||||
}
|
||||
|
||||
if constexpr (nstages > 1) {
|
||||
static_assert(!use_sparse, "sparse gather not implemented for multi-stage loading");
|
||||
static_assert(!V_is_K_view, "K data reuse not implemented multi-stage loading");
|
||||
// Preload K tile for next iteration:
|
||||
constexpr bool use_cp_async = true;
|
||||
@@ -940,11 +981,11 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
||||
__syncthreads();
|
||||
if (!last_iter) {
|
||||
if (ncols2 > 1 || mask_h) {
|
||||
flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check>
|
||||
(mask_h + k_VKQ_0 + nbatch_fa, tile_mask, stride_mask, k_VKQ_sup, jt*ncols1, ne01);
|
||||
flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
|
||||
(mask_h, tile_mask, stride_mask, k_VKQ_0 + nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr);
|
||||
}
|
||||
flash_attn_ext_f16_load_tile<stride_tile_K, nwarps, nbatch_fa, use_cp_async, oob_check>
|
||||
(K_h2 + int64_t(k_VKQ_0 + nbatch_fa)*stride_K, tile_K, nbatch_K2, stride_K, k_VKQ_sup);
|
||||
flash_attn_ext_f16_load_tile<stride_tile_K, swz_K, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
|
||||
(K_h2, tile_K, nbatch_K2, stride_K, k_VKQ_0 + nbatch_fa, k_VKQ_sup, nullptr);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -959,8 +1000,8 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
||||
const int i0_diff = i0_stop - i0_start;
|
||||
if (!V_is_K_view || i0_stop > 2*nbatch_K2) {
|
||||
constexpr bool use_cp_async = nstages == 1;
|
||||
flash_attn_ext_f16_load_tile<stride_tile_V, nwarps, nbatch_fa, use_cp_async, oob_check>
|
||||
(V_h2 + int64_t(k_VKQ_0)*stride_V + i0_start/2, tile_V, i0_diff/2, stride_V, k_VKQ_sup);
|
||||
flash_attn_ext_f16_load_tile<stride_tile_V, swz_V, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
|
||||
(V_h2 + i0_start/2, tile_V, i0_diff/2, stride_V, k_VKQ_0, k_VKQ_sup, indices);
|
||||
if (use_cp_async) {
|
||||
cp_async_wait_all();
|
||||
}
|
||||
@@ -978,7 +1019,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
||||
const int k0 = k00 + (threadIdx.y % np)*T_A_VKQ::J;
|
||||
|
||||
T_A_VKQ A; // Transposed in SRAM but not in registers, gets transposed on load.
|
||||
load_ldmatrix_trans(A, tile_V_i + 2*k0*stride_tile_V + (i_VKQ_0 - i0_start)/2, stride_tile_V);
|
||||
ggml_cuda_fattn_smem_swizzle::load_ldmatrix_trans<stride_tile_V, swz_V>(A, tile_V, (int)(tile_V_i - tile_V) + 2*k0*stride_tile_V + (i_VKQ_0 - i0_start)/2);
|
||||
if constexpr (T_B_KQ::I == 8) {
|
||||
mma(VKQ_C[i_VKQ_0/T_A_VKQ::I], A, B[k00/(np*T_A_VKQ::J)]);
|
||||
} else {
|
||||
@@ -1004,7 +1045,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
||||
const int k0 = k00 + (threadIdx.y % np)*T_A_VKQ::I;
|
||||
|
||||
T_A_VKQ A; // Transposed in both SRAM and registers, load normally.
|
||||
load_ldmatrix(A, tile_V_i + k0*stride_tile_V + (i_VKQ_0 - i0_start)/2, stride_tile_V);
|
||||
ggml_cuda_fattn_smem_swizzle::load_ldmatrix<stride_tile_V, swz_V>(A, tile_V, (int)(tile_V_i - tile_V) + k0*stride_tile_V + (i_VKQ_0 - i0_start)/2);
|
||||
mma(VKQ_C[i_VKQ_0/i0_stride], B[k00/(np*T_A_VKQ::I)], A);
|
||||
}
|
||||
}
|
||||
@@ -1015,7 +1056,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
||||
}
|
||||
}
|
||||
#else
|
||||
GGML_UNUSED_VARS(Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup,
|
||||
GGML_UNUSED_VARS(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup,
|
||||
scale, slope, logit_softcap, ne01, ne02,
|
||||
stride_K, stride_V, stride_mask,
|
||||
tile_Q, tile_K, tile_V, tile_mask,
|
||||
@@ -1113,12 +1154,13 @@ template<int DV, int ncols> struct mma_tile_sizes {
|
||||
};
|
||||
#endif // defined(TURING_MMA_AVAILABLE)
|
||||
|
||||
template<int DKQ, int DV, int ncols1, int ncols2, int nwarps, bool use_logit_softcap, bool V_is_K_view, bool needs_fixup, bool is_fixup>
|
||||
template<int DKQ, int DV, int ncols1, int ncols2, int nwarps, bool use_logit_softcap, bool V_is_K_view, bool use_sparse, bool needs_fixup, bool is_fixup>
|
||||
static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
const float2 * const __restrict__ Q_f2,
|
||||
const half2 * const __restrict__ K_h2,
|
||||
const half2 * const __restrict__ V_h2,
|
||||
const half * const __restrict__ mask_h,
|
||||
const int32_t * const __restrict__ indices,
|
||||
const float * const __restrict__ sinks_f,
|
||||
float2 * const __restrict__ dstk,
|
||||
float2 * const __restrict__ dstk_fixup,
|
||||
@@ -1158,7 +1200,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
constexpr int nbatch_V2 = ggml_cuda_fattn_mma_get_nbatch_V2 (DKQ, DV, ncols);
|
||||
constexpr int nbatch_combine = ggml_cuda_fattn_mma_get_nbatch_combine(DKQ, DV, ncols);
|
||||
constexpr bool Q_in_reg = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols);
|
||||
constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2);
|
||||
constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2, use_sparse);
|
||||
|
||||
if (cols_per_warp > ncols) {
|
||||
NO_DEVICE_CODE;
|
||||
@@ -1168,10 +1210,12 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
static_assert(nwarps * (cols_per_warp/ncols2) % ncols1 == 0, "bad nwarps");
|
||||
|
||||
constexpr int stride_tile_Q = DKQ/2 + 4;
|
||||
constexpr int stride_tile_K = nbatch_K2 + 4;
|
||||
|
||||
constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : nbatch_V2 + 4;
|
||||
// swizzle the tile stride for K and V based on the batch size.
|
||||
constexpr int stride_tile_K = ggml_cuda_fattn_smem_swizzle::tile_stride(nbatch_K2);
|
||||
constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : ggml_cuda_fattn_smem_swizzle::tile_stride(nbatch_V2);
|
||||
constexpr int stride_tile_KV_max = stride_tile_K > stride_tile_V ? stride_tile_K : stride_tile_V;
|
||||
constexpr bool swz_K = ggml_cuda_fattn_smem_swizzle::enabled(nbatch_K2);
|
||||
constexpr bool swz_V = V_is_K_view ? swz_K : ggml_cuda_fattn_smem_swizzle::enabled(nbatch_V2);
|
||||
|
||||
extern __shared__ half2 tile_Q[];
|
||||
half2 * tile_K = Q_in_reg ? tile_Q : tile_Q + ncols * stride_tile_Q;
|
||||
@@ -1257,37 +1301,38 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
|
||||
// Preload mask and K data for first iteration when using cp_async with multiple stages:
|
||||
if constexpr (nstages > 1) {
|
||||
static_assert(!use_sparse, "sparse gather not implemented for multi-stage loading");
|
||||
static_assert(nbatch_K2 == DKQ/2, "batching not implemented for multi-stage pipeline");
|
||||
constexpr bool use_cp_async = true;
|
||||
constexpr bool oob_check = false;
|
||||
constexpr int k_VKQ_sup = nbatch_fa;
|
||||
if (ncols2 > 1 || mask_h) {
|
||||
flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check>
|
||||
(mask_h + kb0*nbatch_fa, tile_mask, stride_mask, k_VKQ_sup, jt*ncols1, ne01);
|
||||
flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
|
||||
(mask_h, tile_mask, stride_mask, kb0*nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr);
|
||||
}
|
||||
flash_attn_ext_f16_load_tile<stride_tile_K, nwarps, nbatch_fa, use_cp_async, oob_check>
|
||||
(K_h2 + int64_t(kb0)*nbatch_fa*stride_K, tile_K, nbatch_K2, stride_K, k_VKQ_sup);
|
||||
flash_attn_ext_f16_load_tile<stride_tile_K, swz_K, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
|
||||
(K_h2, tile_K, nbatch_K2, stride_K, kb0*nbatch_fa, k_VKQ_sup, nullptr);
|
||||
}
|
||||
|
||||
// kb0_start is always < kb0_stop so the last iter can be executed unconditionally.
|
||||
if constexpr (ncols2 == 1) {
|
||||
if constexpr (ncols2 == 1 || use_sparse) {
|
||||
constexpr bool oob_check = true;
|
||||
for (; kb0 < kb0_stop-1; ++kb0) {
|
||||
constexpr bool last_iter = false;
|
||||
constexpr int k_VKQ_sup = nbatch_fa;
|
||||
flash_attn_ext_f16_iter
|
||||
<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, needs_fixup, is_fixup, last_iter, oob_check,
|
||||
<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, last_iter, oob_check,
|
||||
T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
|
||||
(Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap,
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
|
||||
ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
|
||||
KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);
|
||||
}
|
||||
constexpr bool last_iter = true;
|
||||
const int k_VKQ_sup = ne11 - kb0*nbatch_fa;
|
||||
flash_attn_ext_f16_iter
|
||||
<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, needs_fixup, is_fixup, last_iter, oob_check,
|
||||
<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, last_iter, oob_check,
|
||||
T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
|
||||
(Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap,
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
|
||||
ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
|
||||
KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);
|
||||
} else {
|
||||
@@ -1296,18 +1341,18 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
constexpr bool last_iter = false;
|
||||
constexpr int k_VKQ_sup = nbatch_fa;
|
||||
flash_attn_ext_f16_iter
|
||||
<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, needs_fixup, is_fixup, last_iter, oob_check,
|
||||
<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, last_iter, oob_check,
|
||||
T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
|
||||
(Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap,
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
|
||||
ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
|
||||
KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);
|
||||
}
|
||||
constexpr bool last_iter = true;
|
||||
constexpr int k_VKQ_sup = nbatch_fa;
|
||||
flash_attn_ext_f16_iter
|
||||
<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, needs_fixup, is_fixup, last_iter, oob_check,
|
||||
<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, last_iter, oob_check,
|
||||
T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
|
||||
(Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap,
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
|
||||
ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
|
||||
KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);
|
||||
}
|
||||
@@ -1430,11 +1475,17 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
constexpr int tile_stride = nbatch_combine + 4;
|
||||
static_assert((DV/2) % nbatch_combine == 0, "bad nbatch_combine");
|
||||
|
||||
constexpr bool combine_needs_sync = swz_K || swz_V;
|
||||
|
||||
if constexpr (cols_per_warp == 8) {
|
||||
const int jc_cwmo = (threadIdx.x % (2*T_C_VKQ::J)) / T_C_VKQ::J; // jc combine write meta offset
|
||||
const int jc_cwm = threadIdx.y*(2*T_C_VKQ::J) + 2*T_C_VKQ::get_j(-1) + jc_cwmo; // jc combine write meta
|
||||
const float2 KQ_cmr = make_float2(KQ_max[jc_cwmo], KQ_rowsum[jc_cwmo]); // KQ combine max rowsum
|
||||
|
||||
if constexpr (combine_needs_sync) {
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
if (((!needs_fixup && !is_fixup) || np > 1) && threadIdx.x < 2*T_C_VKQ::J) {
|
||||
// Use the 16 bytes of padding in each row to store the meta data: KQ max, KQ rowsum, KQ max scale.
|
||||
((float2 *) tile_Q)[jc_cwm*(tile_stride/2) + nbatch_combine/2] = KQ_cmr;
|
||||
@@ -1471,6 +1522,10 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
const bool thread_should_write = T_C_KQ::J == 8 || T_C_KQ::get_j(threadIdx.x & 2) < 8;
|
||||
#endif // defined(TURING_MMA_AVAILABLE)
|
||||
|
||||
if constexpr (combine_needs_sync) {
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
if (((!needs_fixup && !is_fixup) || np > 1) && thread_should_write) {
|
||||
((float2 *) tile_Q)[jc_cwm*(tile_stride/2) + nbatch_combine/2] = KQ_cmr;
|
||||
}
|
||||
@@ -1692,7 +1747,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
}
|
||||
}
|
||||
#else
|
||||
GGML_UNUSED_VARS(Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dstk_fixup,
|
||||
GGML_UNUSED_VARS(Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dstk_fixup,
|
||||
scale, slope, logit_softcap, ne01, ne02, gqa_ratio,
|
||||
stride_Q1, stride_Q2, stride_K, stride_V, stride_mask,
|
||||
jt, kb0_start, kb0_stop);
|
||||
@@ -1700,7 +1755,13 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
|
||||
}
|
||||
|
||||
template<int DKQ, int DV, int ncols1, int ncols2, bool use_logit_softcap, bool V_is_K_view>
|
||||
static constexpr __host__ __device__ bool ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(
|
||||
const int DKQ, const int DV, const int ncols1, const int ncols2) {
|
||||
return (DKQ == 512 && DV == 512 && ncols1 == 1 && ncols2 == 8) ||
|
||||
(DKQ == 576 && DV == 512 && ncols1 == 1 && ncols2 == 16);
|
||||
}
|
||||
|
||||
template<int DKQ, int DV, int ncols1, int ncols2, bool use_logit_softcap, bool V_is_K_view, bool use_sparse>
|
||||
__launch_bounds__(ggml_cuda_fattn_mma_get_nthreads(DKQ, DV, ncols1*ncols2), ggml_cuda_fattn_mma_get_occupancy(DKQ, DV, ncols1*ncols2))
|
||||
static __global__ void flash_attn_ext_f16(
|
||||
const char * Q_ptr,
|
||||
@@ -1726,14 +1787,15 @@ static __global__ void flash_attn_ext_f16(
|
||||
const int32_t nb31, const int32_t nb32, const int64_t nb33) {
|
||||
ggml_cuda_pdl_sync(); // TODO optimize placement
|
||||
#if defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE))
|
||||
const char * GGML_CUDA_RESTRICT Q = Q_ptr;
|
||||
const char * GGML_CUDA_RESTRICT K = K_ptr;
|
||||
const char * GGML_CUDA_RESTRICT V = V_ptr;
|
||||
const char * GGML_CUDA_RESTRICT mask = mask_ptr;
|
||||
const char * GGML_CUDA_RESTRICT sinks = sinks_ptr;
|
||||
const int * GGML_CUDA_RESTRICT KV_max = KV_max_ptr;
|
||||
float * GGML_CUDA_RESTRICT dst = dst_ptr;
|
||||
float2 * GGML_CUDA_RESTRICT dst_meta = dst_meta_ptr;
|
||||
const char * GGML_CUDA_RESTRICT Q = Q_ptr;
|
||||
const char * GGML_CUDA_RESTRICT K = K_ptr;
|
||||
const char * GGML_CUDA_RESTRICT V = V_ptr;
|
||||
const char * GGML_CUDA_RESTRICT mask = mask_ptr;
|
||||
const char * GGML_CUDA_RESTRICT sinks = sinks_ptr;
|
||||
const int * GGML_CUDA_RESTRICT KV_max = use_sparse ? nullptr : KV_max_ptr;
|
||||
const int * GGML_CUDA_RESTRICT sparse_indices = use_sparse ? KV_max_ptr : nullptr;
|
||||
float * GGML_CUDA_RESTRICT dst = dst_ptr;
|
||||
float2 * GGML_CUDA_RESTRICT dst_meta = dst_meta_ptr;
|
||||
|
||||
// Skip unused kernel variants for faster compilation:
|
||||
if (use_logit_softcap && !(DKQ == 128 || DKQ == 256 || DKQ == 512)) {
|
||||
@@ -1744,6 +1806,11 @@ static __global__ void flash_attn_ext_f16(
|
||||
NO_DEVICE_CODE;
|
||||
return;
|
||||
}
|
||||
|
||||
if (!ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, ncols1, ncols2) && use_sparse) {
|
||||
NO_DEVICE_CODE;
|
||||
return;
|
||||
}
|
||||
#ifdef VOLTA_MMA_AVAILABLE
|
||||
if (ncols1*ncols2 < 32) {
|
||||
NO_DEVICE_CODE;
|
||||
@@ -1820,6 +1887,7 @@ static __global__ void flash_attn_ext_f16(
|
||||
|
||||
const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV);
|
||||
const float * sinks_f = sinks ? (const float *) sinks + zt_Q : nullptr;
|
||||
const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*ne31 + jt*ncols1)*ne11 : nullptr;
|
||||
|
||||
const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, zt_Q, n_head_log2, m0, m1) : 1.0f;
|
||||
|
||||
@@ -1829,13 +1897,13 @@ static __global__ void flash_attn_ext_f16(
|
||||
constexpr bool is_fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer.
|
||||
if (kb0_start == 0) {
|
||||
constexpr bool needs_fixup = false; // CUDA block is working on an entire tile.
|
||||
flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, needs_fixup, is_fixup>
|
||||
(Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,
|
||||
flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup>
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,
|
||||
ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);
|
||||
} else {
|
||||
constexpr bool needs_fixup = true; // CUDA block is missing the beginning of a tile.
|
||||
flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, needs_fixup, is_fixup>
|
||||
(Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,
|
||||
flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup>
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,
|
||||
ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);
|
||||
}
|
||||
|
||||
@@ -1866,6 +1934,7 @@ static __global__ void flash_attn_ext_f16(
|
||||
|
||||
const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV);
|
||||
const float * sinks_f = sinks ? (const float *) sinks + zt_Q : nullptr;
|
||||
const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*ne31 + jt*ncols1)*ne11 : nullptr;
|
||||
|
||||
const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, zt_Q, n_head_log2, m0, m1) : 1.0f;
|
||||
|
||||
@@ -1875,8 +1944,8 @@ static __global__ void flash_attn_ext_f16(
|
||||
|
||||
constexpr bool is_fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks.
|
||||
constexpr bool needs_fixup = false;
|
||||
flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, needs_fixup, is_fixup>
|
||||
(Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,
|
||||
flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup>
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,
|
||||
ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);
|
||||
#else
|
||||
GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, dst_ptr, dst_meta_ptr, scale,
|
||||
@@ -1892,6 +1961,8 @@ static __global__ void flash_attn_ext_f16(
|
||||
#endif // defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE))
|
||||
}
|
||||
|
||||
bool ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||
|
||||
template <int DKQ, int DV, int ncols1, int ncols2>
|
||||
void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
const ggml_tensor * KQV = dst;
|
||||
@@ -1914,8 +1985,11 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml
|
||||
|
||||
constexpr bool V_is_K_view = DKQ == 576; // Guaranteed by the kernel selection logic in fattn.cu
|
||||
|
||||
const size_t nbytes_shared_KV_1stage = nbatch_fa * std::max(nbatch_K2 + 4, nbatch_V2 + 4) * sizeof(half2);
|
||||
const size_t nbytes_shared_KV_2stage = nbatch_fa * (nbatch_K2 + 4 + nbatch_V2 + 4) * sizeof(half2);
|
||||
// KV tile strides must match flash_attn_ext_f16_iter / _process_tile.
|
||||
const int stride_tile_K = ggml_cuda_fattn_smem_swizzle::tile_stride(nbatch_K2, cc);
|
||||
const int stride_tile_V = V_is_K_view ? stride_tile_K : ggml_cuda_fattn_smem_swizzle::tile_stride(nbatch_V2, cc);
|
||||
const size_t nbytes_shared_KV_1stage = nbatch_fa * std::max(stride_tile_K, stride_tile_V) * sizeof(half2);
|
||||
const size_t nbytes_shared_KV_2stage = nbatch_fa * (stride_tile_K + stride_tile_V) * sizeof(half2);
|
||||
const size_t nbytes_shared_Q = ncols * (DKQ/2 + 4) * sizeof(half2);
|
||||
const size_t nbytes_shared_mask = ncols1 * (nbatch_fa/2 + 4) * sizeof(half2);
|
||||
const size_t nbytes_shared_combine = nwarps*cols_per_warp * (nbatch_combine + 4) * sizeof(half2);
|
||||
@@ -1935,20 +2009,49 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml
|
||||
using fattn_kernel_ptr_t = fattn_kernel_t;
|
||||
#endif // defined(GGML_USE_HIP)
|
||||
fattn_kernel_t fattn_kernel;
|
||||
bool use_sparse = false;
|
||||
if (logit_softcap == 0.0f) {
|
||||
constexpr bool use_logit_softcap = false;
|
||||
fattn_kernel = flash_attn_ext_f16<DKQ, DV, ncols1, ncols2, use_logit_softcap, V_is_K_view>;
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
if constexpr (ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, ncols1, ncols2)) {
|
||||
if (ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(ctx, dst)) {
|
||||
constexpr bool use_sparse_kernel = true;
|
||||
fattn_kernel = flash_attn_ext_f16<DKQ, DV, ncols1, ncols2, use_logit_softcap, V_is_K_view, use_sparse_kernel>;
|
||||
use_sparse = true;
|
||||
|
||||
static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false};
|
||||
if (!shared_memory_limit_raised[id]) {
|
||||
CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast<fattn_kernel_ptr_t>(fattn_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_shared_total));
|
||||
shared_memory_limit_raised[id] = true;
|
||||
}
|
||||
} else {
|
||||
constexpr bool use_sparse_kernel = false;
|
||||
fattn_kernel = flash_attn_ext_f16<DKQ, DV, ncols1, ncols2, use_logit_softcap, V_is_K_view, use_sparse_kernel>;
|
||||
|
||||
static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false};
|
||||
if (!shared_memory_limit_raised[id]) {
|
||||
CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast<fattn_kernel_ptr_t>(fattn_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_shared_total));
|
||||
shared_memory_limit_raised[id] = true;
|
||||
}
|
||||
}
|
||||
} else
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
{
|
||||
constexpr bool use_sparse_kernel = false;
|
||||
fattn_kernel = flash_attn_ext_f16<DKQ, DV, ncols1, ncols2, use_logit_softcap, V_is_K_view, use_sparse_kernel>;
|
||||
|
||||
#if !defined(GGML_USE_MUSA)
|
||||
static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false};
|
||||
if (!shared_memory_limit_raised[id]) {
|
||||
CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast<fattn_kernel_ptr_t>(fattn_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_shared_total));
|
||||
shared_memory_limit_raised[id] = true;
|
||||
}
|
||||
static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false};
|
||||
if (!shared_memory_limit_raised[id]) {
|
||||
CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast<fattn_kernel_ptr_t>(fattn_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_shared_total));
|
||||
shared_memory_limit_raised[id] = true;
|
||||
}
|
||||
#endif // !defined(GGML_USE_MUSA)
|
||||
}
|
||||
} else {
|
||||
constexpr bool use_logit_softcap = true;
|
||||
fattn_kernel = flash_attn_ext_f16<DKQ, DV, ncols1, ncols2, use_logit_softcap, V_is_K_view>;
|
||||
constexpr bool use_sparse_kernel = false;
|
||||
fattn_kernel = flash_attn_ext_f16<DKQ, DV, ncols1, ncols2, use_logit_softcap, V_is_K_view, use_sparse_kernel>;
|
||||
|
||||
#if !defined(GGML_USE_MUSA)
|
||||
static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false};
|
||||
@@ -1960,7 +2063,7 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml
|
||||
}
|
||||
|
||||
launch_fattn<DV, ncols1, ncols2>
|
||||
(ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, nbatch_fa, true, true, true, warp_size_host);
|
||||
(ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, nbatch_fa, true, true, true, use_sparse, warp_size_host);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
#pragma once
|
||||
|
||||
#include "common.cuh"
|
||||
#include "mma.cuh"
|
||||
|
||||
// XOR swizzle for K/V SMEM tiles to avoid bank conflicts without row padding (Turing+ only).
|
||||
// Stride must be a multiple of 32 half2 columns, otherwise we keep +4 row padding.
|
||||
|
||||
namespace ggml_cuda_fattn_smem_swizzle {
|
||||
|
||||
static __host__ __device__ constexpr bool bank_aligned(const int nbatch_2) {
|
||||
return nbatch_2 >= 32 && nbatch_2 % 32 == 0;
|
||||
}
|
||||
|
||||
static __device__ constexpr bool enabled(const int nbatch_2) {
|
||||
#if defined(TURING_MMA_AVAILABLE)
|
||||
return bank_aligned(nbatch_2);
|
||||
#else
|
||||
GGML_UNUSED(nbatch_2);
|
||||
return false;
|
||||
#endif // defined(TURING_MMA_AVAILABLE)
|
||||
}
|
||||
|
||||
static __host__ bool enabled(const int nbatch_2, const int cc) {
|
||||
#ifdef GGML_USE_HIP
|
||||
GGML_UNUSED(nbatch_2);
|
||||
GGML_UNUSED(cc);
|
||||
return false;
|
||||
#else
|
||||
return turing_mma_available(cc) && bank_aligned(nbatch_2);
|
||||
#endif // GGML_USE_HIP
|
||||
}
|
||||
|
||||
static __device__ constexpr int tile_stride(const int nbatch_2) {
|
||||
return enabled(nbatch_2) ? nbatch_2 : nbatch_2 + 4;
|
||||
}
|
||||
|
||||
static __host__ int tile_stride(const int nbatch_2, const int cc) {
|
||||
return enabled(nbatch_2, cc) ? nbatch_2 : nbatch_2 + 4;
|
||||
}
|
||||
|
||||
// Swizzled byte offset for tile element (row, col_h2), same map used for writes and reads.
|
||||
template<int stride_h2>
|
||||
static __device__ __forceinline__ int bytes_rc(const int row, const int col_h2) {
|
||||
static_assert(bank_aligned(stride_h2), "swizzled tile needs a stride that is a multiple of 32");
|
||||
return ((row * stride_h2 + col_h2) * (int) sizeof(half2)) ^ ((row & 7) << 4);
|
||||
}
|
||||
|
||||
// ldmatrix.x4 via 64-bit generic pointer.
|
||||
static __device__ __forceinline__ void ldmatrix_x4(int * xi, const half2 * addr) {
|
||||
#if defined(TURING_MMA_AVAILABLE)
|
||||
asm volatile("ldmatrix.sync.aligned.m8n8.x4.b16 {%0, %1, %2, %3}, [%4];"
|
||||
: "=r"(xi[0]), "=r"(xi[1]), "=r"(xi[2]), "=r"(xi[3])
|
||||
: "l"(addr));
|
||||
#else
|
||||
GGML_UNUSED_VARS(xi, addr);
|
||||
NO_DEVICE_CODE;
|
||||
#endif // defined(TURING_MMA_AVAILABLE)
|
||||
}
|
||||
|
||||
static __device__ __forceinline__ void ldmatrix_x4_trans(int * xi, const half2 * addr) {
|
||||
#if defined(TURING_MMA_AVAILABLE)
|
||||
asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.b16 {%0, %1, %2, %3}, [%4];"
|
||||
: "=r"(xi[0]), "=r"(xi[2]), "=r"(xi[1]), "=r"(xi[3])
|
||||
: "l"(addr));
|
||||
#else
|
||||
GGML_UNUSED_VARS(xi, addr);
|
||||
NO_DEVICE_CODE;
|
||||
#endif // defined(TURING_MMA_AVAILABLE)
|
||||
}
|
||||
|
||||
// Per-lane swizzled address for one tile<16, 8, half2> ldmatrix: 16 rows, 4 half2 columns per lane.
|
||||
template<int stride_h2>
|
||||
static __device__ __forceinline__ const half2 * lane_addr(
|
||||
const half2 * tile_base, const int base_row, const int base_col_h2, const int I, const int J) {
|
||||
static_assert(bank_aligned(stride_h2), "swizzled tile needs a stride that is a multiple of 32");
|
||||
const int lane_row = threadIdx.x % I;
|
||||
const int lane_col = (threadIdx.x / I) * (J / 2);
|
||||
uint32_t byte_off = (uint32_t) ((base_row + lane_row)*stride_h2 + base_col_h2 + lane_col) * (uint32_t) sizeof(half2);
|
||||
byte_off ^= (uint32_t) (((base_row + lane_row) & 7) << 4);
|
||||
return (const half2 *) ((const char *) tile_base + byte_off);
|
||||
}
|
||||
|
||||
template<int stride_h2, bool swz, typename TileT>
|
||||
static __device__ __forceinline__ void load_ldmatrix(
|
||||
TileT & t, const half2 * tile_base, const int base_row, const int base_col_h2) {
|
||||
if constexpr (swz) {
|
||||
static_assert(std::is_same_v<TileT, ggml_cuda_mma::tile<16, 8, half2>>,
|
||||
"the swizzled layout is only supported for tile<16, 8, half2>");
|
||||
ldmatrix_x4((int *) t.x, lane_addr<stride_h2>(tile_base, base_row, base_col_h2, TileT::I, TileT::J));
|
||||
} else {
|
||||
ggml_cuda_mma::load_ldmatrix(t, tile_base + base_row*stride_h2 + base_col_h2, stride_h2);
|
||||
}
|
||||
}
|
||||
|
||||
template<int stride_h2, bool swz, typename TileT>
|
||||
static __device__ __forceinline__ void load_ldmatrix(TileT & t, const half2 * tile_base, const int off_h2) {
|
||||
if constexpr (swz) {
|
||||
load_ldmatrix<stride_h2, swz>(t, tile_base, off_h2 / stride_h2, off_h2 % stride_h2);
|
||||
} else {
|
||||
ggml_cuda_mma::load_ldmatrix(t, tile_base + off_h2, stride_h2);
|
||||
}
|
||||
}
|
||||
|
||||
template<int stride_h2, bool swz, typename TileT>
|
||||
static __device__ __forceinline__ void load_ldmatrix_trans(
|
||||
TileT & t, const half2 * tile_base, const int base_row, const int base_col_h2) {
|
||||
if constexpr (swz) {
|
||||
static_assert(std::is_same_v<TileT, ggml_cuda_mma::tile<16, 8, half2>>,
|
||||
"the swizzled layout is only supported for tile<16, 8, half2>");
|
||||
ldmatrix_x4_trans((int *) t.x, lane_addr<stride_h2>(tile_base, base_row, base_col_h2, TileT::I, TileT::J));
|
||||
} else {
|
||||
ggml_cuda_mma::load_ldmatrix_trans(t, tile_base + base_row*stride_h2 + base_col_h2, stride_h2);
|
||||
}
|
||||
}
|
||||
|
||||
template<int stride_h2, bool swz, typename TileT>
|
||||
static __device__ __forceinline__ void load_ldmatrix_trans(TileT & t, const half2 * tile_base, const int off_h2) {
|
||||
if constexpr (swz) {
|
||||
load_ldmatrix_trans<stride_h2, swz>(t, tile_base, off_h2 / stride_h2, off_h2 % stride_h2);
|
||||
} else {
|
||||
ggml_cuda_mma::load_ldmatrix_trans(t, tile_base + off_h2, stride_h2);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace ggml_cuda_fattn_smem_swizzle
|
||||
@@ -1163,7 +1163,7 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm
|
||||
const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);
|
||||
fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;
|
||||
launch_fattn<DV, cols_per_block/ncols2, ncols2>
|
||||
(ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);
|
||||
(ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -1179,7 +1179,7 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm
|
||||
const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);
|
||||
fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;
|
||||
launch_fattn<DV, cols_per_block/ncols2, ncols2>
|
||||
(ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);
|
||||
(ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -1191,7 +1191,7 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm
|
||||
const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);
|
||||
fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;
|
||||
launch_fattn<DV, cols_per_block/ncols2, ncols2>
|
||||
(ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);
|
||||
(ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -1203,7 +1203,7 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm
|
||||
const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);
|
||||
fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;
|
||||
launch_fattn<DV, cols_per_block/ncols2, ncols2>
|
||||
(ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);
|
||||
(ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -1215,7 +1215,7 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm
|
||||
const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);
|
||||
fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;
|
||||
launch_fattn<DV, cols_per_block/ncols2, ncols2>
|
||||
(ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);
|
||||
(ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -1226,7 +1226,7 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm
|
||||
const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);
|
||||
fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;
|
||||
launch_fattn<DV, cols_per_block/ncols2, ncols2>
|
||||
(ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);
|
||||
(ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -540,7 +540,7 @@ void ggml_cuda_flash_attn_ext_vec_case_impl(ggml_backend_cuda_context & ctx, ggm
|
||||
const bool need_f16_K = type_K == GGML_TYPE_F16;
|
||||
const bool need_f16_V = type_V == GGML_TYPE_F16;
|
||||
constexpr size_t nbytes_shared = 0;
|
||||
launch_fattn<D, cols_per_block, 1>(ctx, dst, fattn_kernel, nwarps, nbytes_shared, D, need_f16_K, need_f16_V, false);
|
||||
launch_fattn<D, cols_per_block, 1>(ctx, dst, fattn_kernel, nwarps, nbytes_shared, D, need_f16_K, need_f16_V, false, false);
|
||||
}
|
||||
|
||||
template <int D, ggml_type type_K, ggml_type type_V>
|
||||
|
||||
@@ -5,11 +5,144 @@
|
||||
#include "fattn-vec.cuh"
|
||||
#include "fattn.cuh"
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
__launch_bounds__(256, 1)
|
||||
static __global__ void flash_attn_mask_to_sparse_indices(
|
||||
const half * mask_ptr, int32_t * indices_ptr, const int ne30, const int n_kv_max,
|
||||
const int64_t s31, const int64_t s33) {
|
||||
ggml_cuda_pdl_sync();
|
||||
|
||||
constexpr int values_per_lane = 8;
|
||||
const int tid = threadIdx.x;
|
||||
const int warp = tid / WARP_SIZE;
|
||||
const int lane = tid % WARP_SIZE;
|
||||
const int sequence = blockIdx.y;
|
||||
const int query = blockIdx.x;
|
||||
|
||||
const half * mask = mask_ptr + sequence*s33 + query*s31;
|
||||
int32_t * indices = indices_ptr + (int64_t(sequence)*gridDim.x + query)*n_kv_max;
|
||||
|
||||
__shared__ int warp_offsets[256/WARP_SIZE];
|
||||
__shared__ int row_count;
|
||||
__shared__ int chunk_count;
|
||||
|
||||
if (tid == 0) {
|
||||
row_count = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
for (int i0 = 0; i0 < ne30; i0 += blockDim.x*values_per_lane) {
|
||||
uint32_t selected_warp[values_per_lane];
|
||||
int warp_count = 0;
|
||||
#pragma unroll
|
||||
for (int item = 0; item < values_per_lane; ++item) {
|
||||
const int i = i0 + (warp*values_per_lane + item)*WARP_SIZE + lane;
|
||||
const bool selected = i < ne30 && isfinite(__half2float(mask[i]));
|
||||
selected_warp[item] = __ballot_sync(0xFFFFFFFF, selected);
|
||||
warp_count += __popc(selected_warp[item]);
|
||||
}
|
||||
|
||||
if (lane == 0) {
|
||||
warp_offsets[warp] = warp_count;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
if (tid == 0) {
|
||||
int offset = 0;
|
||||
#pragma unroll
|
||||
for (int iw = 0; iw < 256/WARP_SIZE; ++iw) {
|
||||
const int count = warp_offsets[iw];
|
||||
warp_offsets[iw] = offset;
|
||||
offset += count;
|
||||
}
|
||||
chunk_count = offset;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const uint32_t lane_mask = lane == 0 ? 0 : (1u << lane) - 1;
|
||||
int warp_item_offset = 0;
|
||||
#pragma unroll
|
||||
for (int item = 0; item < values_per_lane; ++item) {
|
||||
const int i = i0 + (warp*values_per_lane + item)*WARP_SIZE + lane;
|
||||
const int dst = row_count + warp_offsets[warp] + warp_item_offset + __popc(selected_warp[item] & lane_mask);
|
||||
if ((selected_warp[item] & (uint32_t(1) << lane)) && dst < n_kv_max) {
|
||||
indices[dst] = i;
|
||||
}
|
||||
warp_item_offset += __popc(selected_warp[item]);
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
if (tid == 0) {
|
||||
row_count += chunk_count;
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
const int count = row_count;
|
||||
for (int i = count + tid; i < n_kv_max; i += blockDim.x) {
|
||||
indices[i] = -1;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// the dependent grid reads indices, signal once the row is complete
|
||||
ggml_cuda_pdl_lc();
|
||||
}
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
void ggml_cuda_flash_attn_ext_compact_mask(
|
||||
const ggml_tensor * mask, int32_t * indices, int32_t n_kv_max, cudaStream_t stream) {
|
||||
#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
|
||||
GGML_UNUSED_VARS(mask, indices, n_kv_max, stream);
|
||||
GGML_ABORT("sparse flash attention is only supported on NVIDIA CUDA");
|
||||
#else
|
||||
const int64_t s31 = mask->nb[1] / sizeof(half);
|
||||
const int64_t s33 = mask->nb[3] / sizeof(half);
|
||||
const dim3 blocks_num(mask->ne[1], mask->ne[3], 1);
|
||||
const dim3 block_dim(256, 1, 1);
|
||||
const ggml_cuda_kernel_launch_params launch_params(blocks_num, block_dim, 0, stream);
|
||||
ggml_cuda_kernel_launch(flash_attn_mask_to_sparse_indices, launch_params,
|
||||
(const half *) mask->data, indices, int(mask->ne[0]), n_kv_max, s31, s33);
|
||||
CUDA_CHECK(cudaGetLastError());
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
}
|
||||
|
||||
bool ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
|
||||
GGML_UNUSED_VARS(ctx, dst);
|
||||
return false;
|
||||
#else
|
||||
const ggml_tensor * Q = dst->src[0];
|
||||
const ggml_tensor * K = dst->src[1];
|
||||
const ggml_tensor * mask = dst->src[3];
|
||||
const int cc = ggml_cuda_info().devices[ctx.device].cc;
|
||||
|
||||
float max_bias = 0.0f;
|
||||
float logit_softcap = 0.0f;
|
||||
memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float));
|
||||
memcpy(&logit_softcap, (const float *) dst->op_params + 2, sizeof(float));
|
||||
|
||||
const int32_t n_kv_max = ggml_get_op_params_i32(dst, 4);
|
||||
return GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc) &&
|
||||
mask != nullptr && n_kv_max > 0 && max_bias == 0.0f && logit_softcap == 0.0f &&
|
||||
mask->ne[0] == K->ne[1] && mask->ne[1] >= Q->ne[1] && mask->ne[2] == 1 &&
|
||||
K->ne[1] >= std::max<int64_t>(4096, 2LL*n_kv_max);
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
}
|
||||
|
||||
template <int DKQ, int DV, int ncols2>
|
||||
static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
|
||||
const ggml_tensor * Q = dst->src[0];
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
if constexpr (ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, 1, ncols2)) {
|
||||
if (ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(ctx, dst)) {
|
||||
ggml_cuda_flash_attn_ext_mma_f16_case<DKQ, DV, 1, ncols2>(ctx, dst);
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
if constexpr (ncols2 <= 8) {
|
||||
if (turing_mma_available(cc) && Q->ne[1] <= 8/ncols2) {
|
||||
ggml_cuda_flash_attn_ext_mma_f16_case<DKQ, DV, 8/ncols2, ncols2>(ctx, dst);
|
||||
|
||||
@@ -34,6 +34,7 @@ bool g_mul_mat_q = true;
|
||||
#include "ggml-cuda/mmq.cuh"
|
||||
#include "ggml-cuda/mmvf.cuh"
|
||||
#include "ggml-cuda/mmvq.cuh"
|
||||
#include "ggml-cuda/moe-weighted-reduction.cuh"
|
||||
#include "ggml-cuda/norm.cuh"
|
||||
#include "ggml-cuda/opt-step-adamw.cuh"
|
||||
#include "ggml-cuda/opt-step-sgd.cuh"
|
||||
@@ -3034,6 +3035,150 @@ static bool ggml_cuda_check_fusion_memory_ranges(const ggml_cgraph * cgraph,
|
||||
return is_ok;
|
||||
}
|
||||
|
||||
// The long form spans 2*k + 1 nodes. ggml_can_fuse_subgraph() accepts at most
|
||||
// 31 nodes, so k <= 15; larger values use the per-operation path.
|
||||
static constexpr int MOE_WEIGHTED_REDUCTION_MAX_EXPERTS = 15;
|
||||
|
||||
struct ggml_cuda_moe_weighted_reduction_match {
|
||||
const ggml_tensor * experts = nullptr;
|
||||
const ggml_tensor * expert_scale = nullptr;
|
||||
const ggml_tensor * weights = nullptr;
|
||||
ggml_tensor * dst = nullptr;
|
||||
int node_count = 0;
|
||||
};
|
||||
|
||||
static bool ggml_cuda_match_moe_weighted_reduction(
|
||||
const ggml_cgraph * cgraph,
|
||||
int node_idx,
|
||||
ggml_cuda_moe_weighted_reduction_match & match) {
|
||||
const ggml_tensor * first = cgraph->nodes[node_idx];
|
||||
if (first->op != GGML_OP_MUL || first->type != GGML_TYPE_F32 || !ggml_is_contiguous(first)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
auto split_mul = [](const ggml_tensor * mul, const ggml_tensor *& full, const ggml_tensor *& broadcast) {
|
||||
auto is_weights = [mul](const ggml_tensor * tensor) {
|
||||
return tensor && tensor->type == GGML_TYPE_F32 && ggml_is_contiguous(tensor) && tensor->ne[0] == 1 &&
|
||||
tensor->ne[1] == mul->ne[1] && tensor->ne[2] == mul->ne[2] && tensor->ne[3] == mul->ne[3];
|
||||
};
|
||||
auto is_experts = [mul](const ggml_tensor * tensor) {
|
||||
return tensor && tensor->type == GGML_TYPE_F32 && ggml_is_contiguous(tensor) &&
|
||||
ggml_are_same_shape(tensor, mul);
|
||||
};
|
||||
|
||||
if (is_experts(mul->src[0]) && is_weights(mul->src[1])) {
|
||||
full = mul->src[0];
|
||||
broadcast = mul->src[1];
|
||||
return true;
|
||||
}
|
||||
if (is_experts(mul->src[1]) && is_weights(mul->src[0])) {
|
||||
full = mul->src[1];
|
||||
broadcast = mul->src[0];
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
const ggml_tensor * weighted = first;
|
||||
const ggml_tensor * experts = nullptr;
|
||||
const ggml_tensor * expert_scale = nullptr;
|
||||
const ggml_tensor * weights = nullptr;
|
||||
int mul_count = 1;
|
||||
|
||||
// Match both structural forms:
|
||||
// (experts * expert_scale) * router_weight
|
||||
// experts * router_weight
|
||||
// The matcher does not depend on the model or quantization type.
|
||||
if (node_idx + 1 < cgraph->n_nodes) {
|
||||
const ggml_tensor * second = cgraph->nodes[node_idx + 1];
|
||||
const ggml_tensor * scaled = nullptr;
|
||||
const ggml_tensor * route = nullptr;
|
||||
const ggml_tensor * raw = nullptr;
|
||||
const ggml_tensor * scale = nullptr;
|
||||
if (second->op == GGML_OP_MUL && second->type == GGML_TYPE_F32 && ggml_is_contiguous(second) &&
|
||||
split_mul(second, scaled, route) && scaled == first && split_mul(first, raw, scale)) {
|
||||
weighted = second;
|
||||
experts = raw;
|
||||
expert_scale = scale;
|
||||
weights = route;
|
||||
mul_count = 2;
|
||||
}
|
||||
}
|
||||
|
||||
if (experts == nullptr && !split_mul(first, experts, weights)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const int n_expert_used = (int) weighted->ne[1];
|
||||
const int64_t n_tokens = weighted->ne[2] * weighted->ne[3];
|
||||
if (n_expert_used < 2 || n_expert_used > MOE_WEIGHTED_REDUCTION_MAX_EXPERTS || n_tokens <= 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const int node_count = 2 * n_expert_used + mul_count - 1;
|
||||
if (node_idx + node_count > cgraph->n_nodes) {
|
||||
return false;
|
||||
}
|
||||
|
||||
std::vector<ggml_op> ops(node_count, GGML_OP_VIEW);
|
||||
ops[0] = GGML_OP_MUL;
|
||||
if (mul_count == 2) {
|
||||
ops[1] = GGML_OP_MUL;
|
||||
}
|
||||
std::vector<const ggml_tensor *> views;
|
||||
views.reserve(n_expert_used);
|
||||
const ggml_tensor * previous = nullptr;
|
||||
int n_adds = 0;
|
||||
for (int offset = mul_count; offset < node_count; ++offset) {
|
||||
const ggml_tensor * candidate = cgraph->nodes[node_idx + offset];
|
||||
ops[offset] = candidate->op;
|
||||
|
||||
if (candidate->op == GGML_OP_VIEW) {
|
||||
const int expert = (int) views.size();
|
||||
if (expert >= n_expert_used || candidate->src[0] != weighted || candidate->view_src != weighted ||
|
||||
candidate->type != GGML_TYPE_F32 || candidate->ne[0] != weighted->ne[0] ||
|
||||
candidate->ne[1] != n_tokens || candidate->ne[2] != 1 || candidate->ne[3] != 1 ||
|
||||
candidate->nb[0] != weighted->nb[0] || candidate->nb[1] != weighted->nb[2] ||
|
||||
candidate->view_offs != (size_t) expert * weighted->nb[1]) {
|
||||
return false;
|
||||
}
|
||||
views.push_back(candidate);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (candidate->op != GGML_OP_ADD || views.size() < 2 || n_adds + 1 >= (int) views.size()) {
|
||||
return false;
|
||||
}
|
||||
const ggml_tensor * lhs = n_adds == 0 ? views[0] : previous;
|
||||
const ggml_tensor * rhs = views[n_adds + 1];
|
||||
if (candidate->src[0] != lhs || candidate->src[1] != rhs || candidate->type != GGML_TYPE_F32) {
|
||||
return false;
|
||||
}
|
||||
previous = candidate;
|
||||
++n_adds;
|
||||
}
|
||||
|
||||
if ((int) views.size() != n_expert_used || n_adds != n_expert_used - 1 || previous == nullptr) {
|
||||
return false;
|
||||
}
|
||||
if (!ggml_is_contiguous(previous) || previous->ne[0] != weighted->ne[0] ||
|
||||
previous->ne[1] != n_tokens || previous->ne[2] != 1 || previous->ne[3] != 1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const int output_idx = node_idx + node_count - 1;
|
||||
if (!ggml_can_fuse_subgraph(cgraph, node_idx, node_count, ops.data(), &output_idx, 1)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
match.experts = experts;
|
||||
match.expert_scale = expert_scale;
|
||||
match.weights = weights;
|
||||
match.dst = cgraph->nodes[output_idx];
|
||||
match.node_count = node_count;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph,
|
||||
int node_idx,
|
||||
@@ -3296,6 +3441,18 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
|
||||
|
||||
ggml_tensor * node = cgraph->nodes[i];
|
||||
|
||||
if (node->op == GGML_OP_MUL) {
|
||||
ggml_cuda_moe_weighted_reduction_match match;
|
||||
if (ggml_cuda_match_moe_weighted_reduction(cgraph, i, match)) {
|
||||
const int output_idx = i + match.node_count - 1;
|
||||
if (ggml_cuda_check_fusion_memory_ranges(cgraph, i, match.node_count, &output_idx, 1)) {
|
||||
ggml_cuda_op_moe_weighted_reduction(
|
||||
*cuda_ctx, match.experts, match.expert_scale, match.weights, match.dst);
|
||||
return match.node_count - 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// gated_delta_net -> cpy: scatter recurrent-state snapshots into the cache
|
||||
if (node->op == GGML_OP_GATED_DELTA_NET) {
|
||||
ggml_cuda_gated_delta_net_fused_cache fused_state_cpy;
|
||||
@@ -4353,10 +4510,30 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev
|
||||
}
|
||||
|
||||
static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph, ggml_backend_graph_optimize_params * params) {
|
||||
GGML_UNUSED(params);
|
||||
|
||||
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
|
||||
|
||||
static const bool disable_fusion = getenv("GGML_CUDA_DISABLE_FUSION") != nullptr && std::atoi(getenv("GGML_CUDA_DISABLE_FUSION"));
|
||||
if (!disable_fusion) {
|
||||
for (int i = 0; i < cgraph->n_nodes; ++i) {
|
||||
if (cgraph->nodes[i]->op != GGML_OP_MUL) {
|
||||
continue;
|
||||
}
|
||||
|
||||
ggml_cuda_moe_weighted_reduction_match match;
|
||||
if (!ggml_cuda_match_moe_weighted_reduction(cgraph, i, match)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
params->add_alloc_dep(params->user_data, const_cast<ggml_tensor *>(match.experts), match.dst);
|
||||
params->add_alloc_dep(params->user_data, const_cast<ggml_tensor *>(match.weights), match.dst);
|
||||
if (match.expert_scale != nullptr) {
|
||||
params->add_alloc_dep(
|
||||
params->user_data, const_cast<ggml_tensor *>(match.expert_scale), match.dst);
|
||||
}
|
||||
i += match.node_count - 1;
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef USE_CUDA_GRAPH
|
||||
const void * graph_key = ggml_cuda_graph_get_key(cgraph);
|
||||
const bool use_cuda_graph = ggml_cuda_graph_set_enabled(cuda_ctx, graph_key);
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
#include "moe-weighted-reduction.cuh"
|
||||
|
||||
static __global__ void moe_weighted_reduction_f32(const float * __restrict__ experts,
|
||||
const float * __restrict__ expert_scale,
|
||||
const float * __restrict__ weights,
|
||||
float * __restrict__ dst,
|
||||
const int64_t n_embd,
|
||||
const int n_expert_used) {
|
||||
const int64_t token = blockIdx.x;
|
||||
const int64_t col = (int64_t) blockIdx.y * blockDim.x + threadIdx.x;
|
||||
if (col >= n_embd) {
|
||||
return;
|
||||
}
|
||||
|
||||
const uint64_t first_row = (uint64_t) token * n_expert_used;
|
||||
const float first_scale = expert_scale != nullptr ? expert_scale[first_row] : 1.0f;
|
||||
float sum = (experts[first_row * n_embd + col] * first_scale) * weights[first_row];
|
||||
|
||||
for (int expert = 1; expert < n_expert_used; ++expert) {
|
||||
const uint64_t row = first_row + expert;
|
||||
const float scale = expert_scale != nullptr ? expert_scale[row] : 1.0f;
|
||||
sum += (experts[row * n_embd + col] * scale) * weights[row];
|
||||
}
|
||||
dst[token * n_embd + col] = sum;
|
||||
}
|
||||
|
||||
static void launch_moe_weighted_reduction(const float * experts,
|
||||
const float * expert_scale,
|
||||
const float * weights,
|
||||
float * dst,
|
||||
int64_t n_embd,
|
||||
int64_t n_tokens,
|
||||
int n_expert_used,
|
||||
cudaStream_t stream) {
|
||||
constexpr int threads = 256;
|
||||
const dim3 blocks(n_tokens, (n_embd + threads - 1) / threads, 1);
|
||||
moe_weighted_reduction_f32
|
||||
<<<blocks, threads, 0, stream>>>(experts, expert_scale, weights, dst, n_embd, n_expert_used);
|
||||
}
|
||||
|
||||
void ggml_cuda_op_moe_weighted_reduction(ggml_backend_cuda_context & ctx,
|
||||
const ggml_tensor * experts,
|
||||
const ggml_tensor * expert_scale,
|
||||
const ggml_tensor * weights,
|
||||
ggml_tensor * dst) {
|
||||
GGML_ASSERT(experts->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(weights->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(expert_scale == nullptr || expert_scale->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(ggml_is_contiguous(experts));
|
||||
GGML_ASSERT(ggml_is_contiguous(weights));
|
||||
GGML_ASSERT(expert_scale == nullptr || ggml_is_contiguous(expert_scale));
|
||||
GGML_ASSERT(ggml_is_contiguous(dst));
|
||||
|
||||
const int64_t n_embd = experts->ne[0];
|
||||
const int64_t n_expert_used = experts->ne[1];
|
||||
const int64_t n_tokens = experts->ne[2] * experts->ne[3];
|
||||
cudaStream_t stream = ctx.stream();
|
||||
|
||||
launch_moe_weighted_reduction((const float *) experts->data,
|
||||
expert_scale ? (const float *) expert_scale->data : nullptr,
|
||||
(const float *) weights->data,
|
||||
(float *) dst->data, n_embd, n_tokens, (int) n_expert_used, stream);
|
||||
CUDA_CHECK(cudaGetLastError());
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
#include "common.cuh"
|
||||
|
||||
void ggml_cuda_op_moe_weighted_reduction(ggml_backend_cuda_context & ctx,
|
||||
const ggml_tensor * experts,
|
||||
const ggml_tensor * expert_scale,
|
||||
const ggml_tensor * weights,
|
||||
ggml_tensor * dst);
|
||||
@@ -69,6 +69,10 @@ struct ggml_metal {
|
||||
// extra command buffers for things like getting, setting and copying tensors
|
||||
NSMutableArray * cmd_bufs_ext;
|
||||
|
||||
// buffers to release after async Metal operations complete
|
||||
// if Metal released them, it would do so on a Metal-internal thread without an autorelease pool, which could cause leaks
|
||||
NSMutableArray * buf_refs;
|
||||
|
||||
// the last command buffer queued into the Metal queue with operations relevant to the current Metal backend
|
||||
id<MTLCommandBuffer> cmd_buf_last;
|
||||
|
||||
@@ -179,6 +183,7 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) {
|
||||
}
|
||||
|
||||
res->cmd_bufs_ext = [[NSMutableArray alloc] init];
|
||||
res->buf_refs = [[NSMutableArray alloc] init];
|
||||
|
||||
res->cmd_buf_last = nil;
|
||||
|
||||
@@ -206,6 +211,11 @@ void ggml_metal_free(ggml_metal_t ctx) {
|
||||
[ctx->cmd_bufs_ext removeAllObjects];
|
||||
[ctx->cmd_bufs_ext release];
|
||||
|
||||
@autoreleasepool {
|
||||
[ctx->buf_refs removeAllObjects];
|
||||
[ctx->buf_refs release];
|
||||
}
|
||||
|
||||
if (ctx->pipelines_ext) {
|
||||
ggml_metal_pipelines_free(ctx->pipelines_ext);
|
||||
ctx->pipelines_ext = nil;
|
||||
@@ -294,6 +304,10 @@ void ggml_metal_synchronize(ggml_metal_t ctx) {
|
||||
|
||||
[ctx->cmd_bufs_ext removeAllObjects];
|
||||
}
|
||||
|
||||
@autoreleasepool {
|
||||
[ctx->buf_refs removeAllObjects];
|
||||
}
|
||||
}
|
||||
|
||||
static struct ggml_metal_buffer_id ggml_metal_get_buffer_id(const struct ggml_tensor * t) {
|
||||
@@ -337,6 +351,8 @@ void ggml_metal_set_tensor_async(ggml_metal_t ctx, struct ggml_tensor * tensor,
|
||||
|
||||
[encoder endEncoding];
|
||||
[cmd_buf commit];
|
||||
|
||||
[ctx->buf_refs addObject:buf_src];
|
||||
[buf_src release];
|
||||
|
||||
// do not wait here for completion
|
||||
@@ -381,6 +397,8 @@ void ggml_metal_get_tensor_async(ggml_metal_t ctx, const struct ggml_tensor * te
|
||||
|
||||
[encoder endEncoding];
|
||||
[cmd_buf commit];
|
||||
|
||||
[ctx->buf_refs addObject:buf_dst];
|
||||
[buf_dst release];
|
||||
|
||||
// do not wait here for completion
|
||||
|
||||
@@ -259,6 +259,7 @@ enum ggml_metal_device_id {
|
||||
GGML_METAL_DEVICE_M5_PRO,
|
||||
GGML_METAL_DEVICE_M5_MAX,
|
||||
GGML_METAL_DEVICE_M5_ULTRA,
|
||||
GGML_METAL_DEVICE_A18_PRO,
|
||||
};
|
||||
|
||||
const char * ggml_metal_device_id_token(enum ggml_metal_device_id id);
|
||||
|
||||
@@ -27,6 +27,9 @@
|
||||
static const NSInteger MTLGPUFamilyMetal3_GGML = 5001;
|
||||
static const NSInteger MTLGPUFamilyMetal4_GGML = 5002;
|
||||
|
||||
// MTLLanguageVersion4_0 is not present in older SDKs
|
||||
static const NSUInteger MTLLanguageVersion4_0_GGML = 4 << 16;
|
||||
|
||||
#if !GGML_METAL_EMBED_LIBRARY
|
||||
// Here to assist with NSBundle Path Hack
|
||||
@interface GGMLMetalClass : NSObject
|
||||
@@ -154,6 +157,9 @@ struct ggml_metal_library {
|
||||
// nil in single_library mode (everything resolves to objs[0]).
|
||||
NSMutableDictionary<NSString *, NSNumber *> * fn_to_lib;
|
||||
|
||||
// kernels from a second metallib, resolved ahead of the combined library
|
||||
NSSet<NSString *> * override_fns;
|
||||
|
||||
ggml_metal_device_t dev;
|
||||
ggml_metal_pipelines_t pipelines; // cache of compiled pipelines
|
||||
|
||||
@@ -174,6 +180,18 @@ static void ggml_metal_library_build_index(ggml_metal_library_t lib) {
|
||||
}
|
||||
}
|
||||
|
||||
// note: defined below, after struct ggml_metal_device
|
||||
static void ggml_metal_device_disable_tensor(ggml_metal_device_t dev);
|
||||
|
||||
// the tensor API headers are exposed to the shader compiler only at Metal language version 4.0
|
||||
static void ggml_metal_compile_options_set_lang(MTLCompileOptions * options, bool has_tensor) {
|
||||
if (!has_tensor) {
|
||||
return;
|
||||
}
|
||||
|
||||
options.languageVersion = (MTLLanguageVersion) MTLLanguageVersion4_0_GGML;
|
||||
}
|
||||
|
||||
// Parse a `#include "name"` line. Returns the quoted name in *include_name on
|
||||
// success. Whitespace-tolerant; ignores `#include <...>` (system headers).
|
||||
static bool ggml_metal_library_parse_quoted_include(NSString * line, NSString ** include_name) {
|
||||
@@ -313,6 +331,7 @@ static bool ggml_metal_library_compile_all(
|
||||
@autoreleasepool {
|
||||
MTLCompileOptions * options = [MTLCompileOptions new];
|
||||
options.preprocessorMacros = prep;
|
||||
ggml_metal_compile_options_set_lang(options, ggml_metal_device_get_props(res->dev)->has_tensor);
|
||||
|
||||
lib = [device newLibraryWithSource:src options:options error:&error];
|
||||
|
||||
@@ -369,6 +388,46 @@ static bool ggml_metal_library_compile_all(
|
||||
return ok;
|
||||
}
|
||||
|
||||
// look for <name>.metallib as a bundle resource, then next to the running binary
|
||||
static NSString * ggml_metal_find_metallib(NSBundle * bundle, NSString * name) {
|
||||
NSError * error = nil;
|
||||
|
||||
NSString * path_lib = [bundle pathForResource:name ofType:@"metallib"];
|
||||
if (path_lib == nil) {
|
||||
// Try to find the resource in the directory where the current binary located.
|
||||
NSString * bin_cur = [[NSProcessInfo processInfo] arguments][0];
|
||||
NSString * bin_dir = [bin_cur stringByDeletingLastPathComponent];
|
||||
|
||||
NSString * path_lib_default = [NSString pathWithComponents:@[bin_dir, [name stringByAppendingPathExtension:@"metallib"]]];
|
||||
if ([[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {
|
||||
GGML_LOG_INFO("%s: found '%s'\n", __func__, [path_lib_default UTF8String]);
|
||||
|
||||
NSDictionary * atts = [[NSFileManager defaultManager] attributesOfItemAtPath:path_lib_default error:&error];
|
||||
if (atts && atts[NSFileType] == NSFileTypeSymbolicLink) {
|
||||
// Optionally, if this is a symlink, try to resolve it.
|
||||
path_lib_default = [[NSFileManager defaultManager] destinationOfSymbolicLinkAtPath:path_lib_default error:&error];
|
||||
if (path_lib_default && [path_lib_default length] > 0 && ![[path_lib_default substringToIndex:1] isEqualToString:@"/"]) {
|
||||
// It is a relative path, adding the binary directory as directory prefix.
|
||||
path_lib_default = [NSString pathWithComponents:@[bin_dir, path_lib_default]];
|
||||
}
|
||||
if (!path_lib_default || ![[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {
|
||||
// Link to the resource could not be resolved.
|
||||
path_lib_default = nil;
|
||||
} else {
|
||||
GGML_LOG_INFO("%s: symlink resolved '%s'\n", __func__, [path_lib_default UTF8String]);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// The resource couldn't be found in the binary's directory.
|
||||
path_lib_default = nil;
|
||||
}
|
||||
|
||||
path_lib = path_lib_default;
|
||||
}
|
||||
|
||||
return path_lib;
|
||||
}
|
||||
|
||||
ggml_metal_library_t ggml_metal_library_init(ggml_metal_device_t dev) {
|
||||
id<MTLDevice> device = ggml_metal_device_get_obj(dev);
|
||||
|
||||
@@ -432,38 +491,7 @@ ggml_metal_library_t ggml_metal_library_init(ggml_metal_device_t dev) {
|
||||
const int64_t t_start = ggml_time_us();
|
||||
|
||||
NSError * error = nil;
|
||||
NSString * path_lib = [bundle pathForResource:@"default" ofType:@"metallib"];
|
||||
if (path_lib == nil) {
|
||||
// Try to find the resource in the directory where the current binary located.
|
||||
NSString * bin_cur = [[NSProcessInfo processInfo] arguments][0];
|
||||
NSString * bin_dir = [bin_cur stringByDeletingLastPathComponent];
|
||||
|
||||
NSString * path_lib_default = [NSString pathWithComponents:@[bin_dir, @"default.metallib"]];
|
||||
if ([[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {
|
||||
GGML_LOG_INFO("%s: found '%s'\n", __func__, [path_lib_default UTF8String]);
|
||||
|
||||
NSDictionary * atts = [[NSFileManager defaultManager] attributesOfItemAtPath:path_lib_default error:&error];
|
||||
if (atts && atts[NSFileType] == NSFileTypeSymbolicLink) {
|
||||
// Optionally, if this is a symlink, try to resolve it.
|
||||
path_lib_default = [[NSFileManager defaultManager] destinationOfSymbolicLinkAtPath:path_lib_default error:&error];
|
||||
if (path_lib_default && [path_lib_default length] > 0 && ![[path_lib_default substringToIndex:1] isEqualToString:@"/"]) {
|
||||
// It is a relative path, adding the binary directory as directory prefix.
|
||||
path_lib_default = [NSString pathWithComponents:@[bin_dir, path_lib_default]];
|
||||
}
|
||||
if (!path_lib_default || ![[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {
|
||||
// Link to the resource could not be resolved.
|
||||
path_lib_default = nil;
|
||||
} else {
|
||||
GGML_LOG_INFO("%s: symlink resolved '%s'\n", __func__, [path_lib_default UTF8String]);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// The resource couldn't be found in the binary's directory.
|
||||
path_lib_default = nil;
|
||||
}
|
||||
|
||||
path_lib = path_lib_default;
|
||||
}
|
||||
NSString * path_lib = ggml_metal_find_metallib(bundle, @"default");
|
||||
|
||||
if (path_lib != nil) {
|
||||
// pre-compiled library found: a single combined default.metallib
|
||||
@@ -478,6 +506,30 @@ ggml_metal_library_t ggml_metal_library_init(ggml_metal_device_t dev) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// the tensor API kernels are built into a separate metallib
|
||||
if (ggml_metal_device_get_props(dev)->has_tensor) {
|
||||
NSString * path_mm = ggml_metal_find_metallib(bundle, @"ggml-tensor");
|
||||
|
||||
id<MTLLibrary> lib_mm = nil;
|
||||
if (path_mm != nil) {
|
||||
lib_mm = [device newLibraryWithURL:[NSURL fileURLWithPath:path_mm] error:&error];
|
||||
if (!lib_mm && error) {
|
||||
GGML_LOG_ERROR("%s: %s\n", __func__, [[error description] UTF8String]);
|
||||
}
|
||||
}
|
||||
|
||||
if (lib_mm) {
|
||||
GGML_LOG_INFO("%s: loaded '%s'\n", __func__, [path_mm UTF8String]);
|
||||
|
||||
res->objs[GGML_METAL_LIB_MUL_MM] = [lib_mm retain];
|
||||
res->override_fns = [[NSSet setWithArray:[lib_mm functionNames]] retain];
|
||||
} else {
|
||||
GGML_LOG_INFO("%s: ggml-tensor.metallib not found - disabling the tensor API\n", __func__);
|
||||
|
||||
ggml_metal_device_disable_tensor(dev);
|
||||
}
|
||||
}
|
||||
|
||||
GGML_LOG_INFO("%s: loaded in %.3f sec\n", __func__, (ggml_time_us() - t_start) / 1e6);
|
||||
return res;
|
||||
}
|
||||
@@ -557,6 +609,7 @@ ggml_metal_library_t ggml_metal_library_init_from_source(ggml_metal_device_t dev
|
||||
|
||||
MTLCompileOptions * options = [MTLCompileOptions new];
|
||||
options.preprocessorMacros = prep;
|
||||
ggml_metal_compile_options_set_lang(options, ggml_metal_device_get_props(dev)->has_tensor);
|
||||
|
||||
library = [device newLibraryWithSource:src options:options error:&error];
|
||||
if (error) {
|
||||
@@ -615,6 +668,10 @@ void ggml_metal_library_free(ggml_metal_library_t lib) {
|
||||
[lib->fn_to_lib release];
|
||||
}
|
||||
|
||||
if (lib->override_fns) {
|
||||
[lib->override_fns release];
|
||||
}
|
||||
|
||||
ggml_metal_pipelines_free(lib->pipelines);
|
||||
|
||||
[lib->lock release];
|
||||
@@ -676,7 +733,9 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_compile_pipeline(ggml_
|
||||
// route to the library that actually defines this kernel; fn_to_lib is
|
||||
// built from -[MTLLibrary functionNames] so it's always in sync
|
||||
int lib_idx = 0;
|
||||
if (!lib->single_library) {
|
||||
if (lib->override_fns && [lib->override_fns containsObject:base_func]) {
|
||||
lib_idx = GGML_METAL_LIB_MUL_MM;
|
||||
} else if (!lib->single_library) {
|
||||
NSNumber * idx = lib->fn_to_lib[base_func];
|
||||
if (!idx) {
|
||||
[lib->lock unlock];
|
||||
@@ -1003,6 +1062,7 @@ static const struct {
|
||||
DEV("M5 Pro", GGML_METAL_DEVICE_M5_PRO),
|
||||
DEV("M5 Max", GGML_METAL_DEVICE_M5_MAX),
|
||||
DEV("M5 Ultra", GGML_METAL_DEVICE_M5_ULTRA),
|
||||
DEV("A18 Pro", GGML_METAL_DEVICE_A18_PRO),
|
||||
#undef DEV
|
||||
};
|
||||
|
||||
@@ -1293,19 +1353,21 @@ ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) {
|
||||
void ggml_metal_device_free(ggml_metal_device_t dev) {
|
||||
assert(dev != NULL);
|
||||
|
||||
ggml_metal_rsets_free(dev->rsets);
|
||||
@autoreleasepool {
|
||||
ggml_metal_rsets_free(dev->rsets);
|
||||
|
||||
ggml_metal_library_free(dev->library);
|
||||
dev->library = NULL;
|
||||
ggml_metal_library_free(dev->library);
|
||||
dev->library = NULL;
|
||||
|
||||
if (dev->mtl_queue) {
|
||||
[dev->mtl_queue release];
|
||||
dev->mtl_queue = nil;
|
||||
}
|
||||
if (dev->mtl_queue) {
|
||||
[dev->mtl_queue release];
|
||||
dev->mtl_queue = nil;
|
||||
}
|
||||
|
||||
if (dev->mtl_device) {
|
||||
[dev->mtl_device release];
|
||||
dev->mtl_device = nil;
|
||||
if (dev->mtl_device) {
|
||||
[dev->mtl_device release];
|
||||
dev->mtl_device = nil;
|
||||
}
|
||||
}
|
||||
|
||||
free(dev);
|
||||
@@ -1393,12 +1455,14 @@ ggml_metal_event_t ggml_metal_device_event_init(ggml_metal_device_t dev) {
|
||||
}
|
||||
|
||||
void ggml_metal_device_event_free(ggml_metal_device_t dev, ggml_metal_event_t ev) {
|
||||
id<MTLSharedEvent> event = ev->obj;
|
||||
[event release];
|
||||
@autoreleasepool {
|
||||
id<MTLSharedEvent> event = ev->obj;
|
||||
[event release];
|
||||
|
||||
free(ev);
|
||||
free(ev);
|
||||
|
||||
GGML_UNUSED(dev);
|
||||
GGML_UNUSED(dev);
|
||||
}
|
||||
}
|
||||
|
||||
void ggml_metal_device_event_synchronize(ggml_metal_device_t dev, ggml_metal_event_t ev) {
|
||||
@@ -1544,6 +1608,12 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
|
||||
return true;
|
||||
case GGML_TYPE_BF16:
|
||||
return has_bfloat;
|
||||
case GGML_TYPE_Q4_0:
|
||||
case GGML_TYPE_Q4_1:
|
||||
case GGML_TYPE_Q5_0:
|
||||
case GGML_TYPE_Q5_1:
|
||||
case GGML_TYPE_Q8_0:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
@@ -1862,6 +1932,10 @@ const struct ggml_metal_device_props * ggml_metal_device_get_props(ggml_metal_de
|
||||
return &dev->props;
|
||||
}
|
||||
|
||||
static void ggml_metal_device_disable_tensor(ggml_metal_device_t dev) {
|
||||
dev->props.has_tensor = false;
|
||||
}
|
||||
|
||||
//
|
||||
// device buffers
|
||||
//
|
||||
@@ -2163,14 +2237,16 @@ ggml_metal_buffer_t ggml_metal_buffer_map(ggml_metal_device_t dev, void * ptr, s
|
||||
}
|
||||
|
||||
void ggml_metal_buffer_free(ggml_metal_buffer_t buf) {
|
||||
ggml_metal_device_rsets_rm(buf->dev, buf->rset);
|
||||
@autoreleasepool {
|
||||
ggml_metal_device_rsets_rm(buf->dev, buf->rset);
|
||||
|
||||
for (int i = 0; i < buf->n_buffers; i++) {
|
||||
[buf->buffers[i].metal release];
|
||||
for (int i = 0; i < buf->n_buffers; i++) {
|
||||
[buf->buffers[i].metal release];
|
||||
}
|
||||
|
||||
ggml_metal_buffer_rset_free(buf);
|
||||
}
|
||||
|
||||
ggml_metal_buffer_rset_free(buf);
|
||||
|
||||
if (buf->is_shared && buf->owned) {
|
||||
#if TARGET_OS_OSX
|
||||
vm_deallocate((vm_map_t)mach_task_self(), (vm_address_t)buf->all_data, buf->all_size);
|
||||
|
||||
@@ -552,8 +552,24 @@ int ggml_metal_op_concat(ggml_metal_op_t ctx, int idx) {
|
||||
|
||||
const int32_t dim = ((const int32_t *) op->op_params)[0];
|
||||
|
||||
const bool is_q = ggml_is_quantized(op->type);
|
||||
|
||||
// for quantized types, concat is done at the block level (nb0 == type_size == block size)
|
||||
int32_t ne00_arg = ne00;
|
||||
int32_t ne10_arg = ne10;
|
||||
int32_t ne0_arg = ne0;
|
||||
if (is_q) {
|
||||
const int32_t blck = ggml_blck_size(op->type);
|
||||
GGML_ASSERT(ne00 % blck == 0);
|
||||
GGML_ASSERT(ne10 % blck == 0);
|
||||
GGML_ASSERT(ne0 % blck == 0);
|
||||
ne00_arg = ne00/blck;
|
||||
ne10_arg = ne10/blck;
|
||||
ne0_arg = ne0/blck;
|
||||
}
|
||||
|
||||
ggml_metal_kargs_concat args = {
|
||||
/*.ne00 =*/ ne00,
|
||||
/*.ne00 =*/ ne00_arg,
|
||||
/*.ne01 =*/ ne01,
|
||||
/*.ne02 =*/ ne02,
|
||||
/*.ne03 =*/ ne03,
|
||||
@@ -561,7 +577,7 @@ int ggml_metal_op_concat(ggml_metal_op_t ctx, int idx) {
|
||||
/*.nb01 =*/ nb01,
|
||||
/*.nb02 =*/ nb02,
|
||||
/*.nb03 =*/ nb03,
|
||||
/*.ne10 =*/ ne10,
|
||||
/*.ne10 =*/ ne10_arg,
|
||||
/*.ne11 =*/ ne11,
|
||||
/*.ne12 =*/ ne12,
|
||||
/*.ne13 =*/ ne13,
|
||||
@@ -569,7 +585,7 @@ int ggml_metal_op_concat(ggml_metal_op_t ctx, int idx) {
|
||||
/*.nb11 =*/ nb11,
|
||||
/*.nb12 =*/ nb12,
|
||||
/*.nb13 =*/ nb13,
|
||||
/*.ne0 =*/ ne0,
|
||||
/*.ne0 =*/ ne0_arg,
|
||||
/*.ne1 =*/ ne1,
|
||||
/*.ne2 =*/ ne2,
|
||||
/*.ne3 =*/ ne3,
|
||||
@@ -588,7 +604,7 @@ int ggml_metal_op_concat(ggml_metal_op_t ctx, int idx) {
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);
|
||||
|
||||
int nth = std::min(256, ne0);
|
||||
int nth = std::min(256, ne0_arg);
|
||||
|
||||
// when rows are small, we can batch them together in a single threadgroup
|
||||
int nrptg = 1;
|
||||
|
||||
@@ -657,6 +657,183 @@ constexpr fa_vec_entry_t fa_vec_tuned_table[] = {
|
||||
{ { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 128, 128, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 128, 128, 3, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 192, 128, 1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 192, 128, 1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 192, 128, 1, 3 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 320, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 320, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_F16, 320, 256, 1, 4 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 32, 32, 1, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 32, 32, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 32, 32, 2, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 32, 32, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 32, 32, 3, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 96, 96, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 96, 96, 2, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 96, 96, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 96, 96, 3, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 32, 32, 1, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 32, 32, 2, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 32, 32, 3, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 96, 96, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 96, 96, 3, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 128, 128, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 128, 128, 3, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 128, 128, 3, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 256, 256, 3, 2 }, { 1, 1 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 512, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 512, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 32, 32, 1, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 32, 32, 1, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 192, 192, 1, 4 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 192, 192, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 192, 192, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 256, 256, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 320, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 320, 256, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 320, 256, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 576, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 32, 32, 2, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 32, 32, 2, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 32, 32, 2, 4 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 32, 32, 3, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 32, 32, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 32, 32, 3, 4 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 96, 96, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 96, 96, 1, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 96, 96, 1, 4 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 256, 256, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 320, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 320, 256, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 320, 256, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 320, 256, 2, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 320, 256, 2, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 320, 256, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 320, 256, 3, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 576, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q5_1, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 32, 32, 1, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 32, 32, 2, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 32, 32, 3, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 64, 64, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 64, 64, 3, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 96, 96, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 96, 96, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 96, 96, 3, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 96, 96, 3, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 320, 256, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 320, 256, 1, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 320, 256, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 320, 256, 2, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 320, 256, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 320, 256, 3, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M1_ULTRA, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
|
||||
{ { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 32, 32, 1, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } },
|
||||
@@ -852,6 +1029,274 @@ constexpr fa_vec_entry_t fa_vec_tuned_table[] = {
|
||||
{ { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 128, 128, 2, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 128, 128, 2, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 128, 128, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 128, 128, 2, 3 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 128, 128, 2, 4 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 128, 128, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 192, 128, 1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 192, 128, 1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 192, 128, 1, 3 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 320, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 320, 256, 1, 1 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 320, 256, 1, 3 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_F16, 320, 256, 1, 4 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 32, 32, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 96, 96, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 96, 96, 2, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 96, 96, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 96, 96, 3, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 320, 256, 3, 3 }, { 4, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 320, 256, 3, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 32, 32, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 32, 32, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 32, 32, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 96, 96, 2, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 96, 96, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 96, 96, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 192, 192, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 192, 192, 1, 4 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 192, 192, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 192, 192, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 512, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q4_1, 512, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 96, 96, 2, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 96, 96, 3, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 128, 128, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 128, 128, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 128, 128, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 128, 128, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 4, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 192, 128, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 192, 128, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 192, 128, 1, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 192, 128, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 192, 128, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 320, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 576, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 128, 128, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 128, 128, 2, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 128, 128, 2, 3 }, { 4, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 128, 128, 3, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 128, 128, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 192, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 192, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 192, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 4, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 128, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 128, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 128, 1, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 128, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 192, 128, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 320, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 576, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q5_1, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 96, 96, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 96, 96, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 320, 256, 1, 2 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 320, 256, 1, 4 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 320, 256, 2, 2 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 320, 256, 2, 4 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 320, 256, 3, 2 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 32, 32, 1, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 64, 64, 1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 64, 64, 2, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 64, 64, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 64, 64, 3, 2 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 64, 64, 3, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 64, 64, 3, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 96, 96, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 128, 128, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 192, 192, 3, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 192, 128, 1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 192, 128, 2, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 192, 128, 1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 192, 128, 2, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 192, 128, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 192, 128, 3, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 320, 256, 2, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 320, 256, 3, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 320, 256, 2, 3 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 320, 256, 2, 4 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 512, 512, 2, 0 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 512, 512, 2, 1 }, { 4, 1 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_F16, 512, 512, 2, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 32, 32, 2, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 64, 64, 1, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 64, 64, 2, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 96, 96, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 96, 96, 1, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 96, 96, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 96, 96, 2, 4 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 128, 128, 2, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 128, 128, 2, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 128, 128, 3, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 192, 128, 1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 192, 128, 3, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 192, 128, 3, 3 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 192, 128, 3, 4 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 256, 256, 2, 2 }, { 4, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 256, 256, 2, 3 }, { 4, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 320, 256, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 320, 256, 1, 3 }, { 4, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 320, 256, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 320, 256, 2, 2 }, { 4, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 320, 256, 2, 3 }, { 4, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 320, 256, 2, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 320, 256, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 320, 256, 3, 3 }, { 4, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 512, 512, 3, 1 }, { 1, 1 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
|
||||
{ { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M2_ULTRA, GGML_TYPE_F16, 128, 128, 2, 1 }, { 1, 4 } },
|
||||
@@ -2727,6 +3172,239 @@ constexpr fa_vec_entry_t fa_vec_tuned_table[] = {
|
||||
{ { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 576, 512, 1, 4 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_M5_MAX, GGML_TYPE_Q8_0, 576, 512, 2, 1 }, { 4, 4 } },
|
||||
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 32, 32, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 32, 32, 1, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 32, 32, 2, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 32, 32, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 64, 64, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 64, 64, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 64, 64, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 96, 96, 1, 4 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 96, 96, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 96, 96, 2, 4 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 128, 128, 1, 2 }, { 1, 1 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 128, 128, 2, 2 }, { 1, 1 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 128, 128, 2, 4 }, { 1, 1 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 192, 192, 1, 4 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 192, 128, 1, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 192, 128, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 192, 128, 2, 2 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 256, 256, 2, 0 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 256, 256, 3, 0 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 256, 256, 1, 2 }, { 1, 1 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 256, 256, 1, 4 }, { 1, 1 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 512, 512, 3, 0 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 512, 512, 3, 2 }, { 4, 1 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 512, 512, 3, 3 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 576, 512, 2, 0 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 576, 512, 2, 2 }, { 4, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_F16, 576, 512, 3, 2 }, { 4, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 32, 32, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 32, 32, 1, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 32, 32, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 32, 32, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 64, 64, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 64, 64, 1, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 64, 64, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 64, 64, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 96, 96, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 96, 96, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 96, 96, 2, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 96, 96, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 128, 128, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 192, 192, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 192, 192, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 192, 192, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 192, 128, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 576, 512, 2, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 576, 512, 3, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 576, 512, 1, 2 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_0, 576, 512, 1, 3 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 32, 32, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 64, 64, 1, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 64, 64, 2, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 64, 64, 2, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 64, 64, 3, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 64, 64, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 192, 192, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 192, 192, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 192, 192, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 320, 256, 3, 1 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 512, 512, 2, 0 }, { 4, 1 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 512, 512, 3, 1 }, { 4, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 512, 512, 3, 2 }, { 4, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 576, 512, 2, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 576, 512, 3, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 128, 128, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 128, 128, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 128, 128, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 4, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 192, 128, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 192, 128, 1, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 192, 128, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 192, 128, 2, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 192, 128, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 320, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 320, 256, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 512, 512, 2, 3 }, { 2, 1 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 512, 512, 3, 1 }, { 2, 1 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 512, 512, 3, 3 }, { 2, 1 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 576, 512, 3, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 576, 512, 1, 2 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 576, 512, 1, 3 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_0, 576, 512, 1, 4 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 128, 128, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 128, 128, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 128, 128, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 4, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 192, 128, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 192, 128, 1, 4 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 192, 128, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 192, 128, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 320, 256, 1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 2, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 320, 256, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 512, 512, 3, 3 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 576, 512, 2, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q5_1, 576, 512, 3, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 32, 32, 2, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 64, 64, 1, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 64, 64, 2, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 96, 96, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 96, 96, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 128, 128, 2, 3 }, { 4, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 192, 192, 1, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 192, 192, 2, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 192, 192, 3, 2 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 576, 512, 1, 1 }, { 1, 2 } },
|
||||
{ { GGML_METAL_DEVICE_A18_PRO, GGML_TYPE_Q8_0, 576, 512, 1, 4 }, { 1, 2 } },
|
||||
};
|
||||
|
||||
static enum ggml_metal_device_id fa_vec_family_representative(int gpu_family) {
|
||||
|
||||
@@ -207,6 +207,51 @@ template [[host_name("kernel_concat_i16")]] kernel kernel_concat_t kernel_conca
|
||||
template [[host_name("kernel_concat_i32")]] kernel kernel_concat_t kernel_concat<int>;
|
||||
template [[host_name("kernel_concat_i64")]] kernel kernel_concat_t kernel_concat<long>;
|
||||
|
||||
template<typename block_q>
|
||||
kernel void kernel_concat_q(
|
||||
constant ggml_metal_kargs_concat & args,
|
||||
device const char * src0,
|
||||
device const char * src1,
|
||||
device char * dst,
|
||||
uint3 tgpig[[threadgroup_position_in_grid]],
|
||||
ushort3 tpitg[[thread_position_in_threadgroup]],
|
||||
ushort3 ntg[[threads_per_threadgroup]]) {
|
||||
|
||||
// note: for quantized types, the args are in units of blocks (nb0 == type_size)
|
||||
const int i3 = tgpig.z;
|
||||
const int i2 = tgpig.y;
|
||||
const int i1 = ntg.y == 1 ? tgpig.x : tgpig.x*ntg.y + tpitg.y;
|
||||
|
||||
if (i1 >= args.ne1) {
|
||||
return;
|
||||
}
|
||||
|
||||
int o[4] = {0, 0, 0, 0};
|
||||
o[args.dim] = args.dim == 0 ? args.ne00 : (args.dim == 1 ? args.ne01 : (args.dim == 2 ? args.ne02 : args.ne03));
|
||||
|
||||
for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) {
|
||||
device const block_q * x;
|
||||
|
||||
if (i0 < args.ne00 && i1 < args.ne01 && i2 < args.ne02 && i3 < args.ne03) {
|
||||
x = (device const block_q *)(src0 + (i3 )*args.nb03 + (i2 )*args.nb02 + (i1 )*args.nb01 + (i0 )*args.nb00);
|
||||
} else {
|
||||
x = (device const block_q *)(src1 + (i3 - o[3])*args.nb13 + (i2 - o[2])*args.nb12 + (i1 - o[1])*args.nb11 + (i0 - o[0])*args.nb10);
|
||||
}
|
||||
|
||||
device block_q * y = (device block_q *)(dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0);
|
||||
|
||||
*y = *x;
|
||||
}
|
||||
}
|
||||
|
||||
typedef decltype(kernel_concat_q<block_q4_0>) kernel_concat_q_t;
|
||||
|
||||
template [[host_name("kernel_concat_q4_0")]] kernel kernel_concat_q_t kernel_concat_q<block_q4_0>;
|
||||
template [[host_name("kernel_concat_q4_1")]] kernel kernel_concat_q_t kernel_concat_q<block_q4_1>;
|
||||
template [[host_name("kernel_concat_q5_0")]] kernel kernel_concat_q_t kernel_concat_q<block_q5_0>;
|
||||
template [[host_name("kernel_concat_q5_1")]] kernel kernel_concat_q_t kernel_concat_q<block_q5_1>;
|
||||
template [[host_name("kernel_concat_q8_0")]] kernel kernel_concat_q_t kernel_concat_q<block_q8_0>;
|
||||
|
||||
template<typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread float4x4 &)>
|
||||
kernel void kernel_get_rows_q(
|
||||
constant ggml_metal_kargs_get_rows & args,
|
||||
|
||||
@@ -6920,7 +6920,8 @@ static vk_device ggml_vk_get_device(size_t idx) {
|
||||
}
|
||||
|
||||
#if defined(VK_KHR_shader_bfloat16) && defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT)
|
||||
if (prop.AType == VK_COMPONENT_TYPE_BFLOAT16_KHR &&
|
||||
if (bfloat16_support &&
|
||||
prop.AType == VK_COMPONENT_TYPE_BFLOAT16_KHR &&
|
||||
prop.BType == VK_COMPONENT_TYPE_BFLOAT16_KHR &&
|
||||
prop.CType == VK_COMPONENT_TYPE_FLOAT32_KHR &&
|
||||
prop.ResultType == VK_COMPONENT_TYPE_FLOAT32_KHR) {
|
||||
@@ -7040,7 +7041,8 @@ static vk_device ggml_vk_get_device(size_t idx) {
|
||||
device->coopmat_int_k = prop.KSize;
|
||||
}
|
||||
#if defined(VK_KHR_shader_bfloat16) && defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT)
|
||||
if (prop.AType == VK_COMPONENT_TYPE_BFLOAT16_KHR &&
|
||||
if (bfloat16_support &&
|
||||
prop.AType == VK_COMPONENT_TYPE_BFLOAT16_KHR &&
|
||||
prop.BType == VK_COMPONENT_TYPE_BFLOAT16_KHR &&
|
||||
prop.CType == VK_COMPONENT_TYPE_FLOAT32_KHR &&
|
||||
prop.ResultType == VK_COMPONENT_TYPE_FLOAT32_KHR &&
|
||||
@@ -7065,19 +7067,11 @@ static vk_device ggml_vk_get_device(size_t idx) {
|
||||
GGML_LOG_DEBUG("ggml_vulkan: WARNING: No suitable matrix core mode found. Disabling matrix cores.\n");
|
||||
device->coopmat_support = false;
|
||||
}
|
||||
if (getenv("GGML_VK_DISABLE_BFLOAT16")) {
|
||||
device->coopmat_bf16_support = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (device->coopmat_support) {
|
||||
device_extensions.push_back("VK_KHR_cooperative_matrix");
|
||||
}
|
||||
#if defined(VK_KHR_shader_bfloat16)
|
||||
if (device->coopmat_bf16_support) {
|
||||
device_extensions.push_back("VK_KHR_shader_bfloat16");
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
device->name = GGML_VK_NAME + std::to_string(idx);
|
||||
|
||||
|
||||
@@ -7,7 +7,14 @@ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
||||
|
||||
FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
|
||||
|
||||
void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
|
||||
// invocations per superblock. with many columns, 8 invocations need too many
|
||||
// registers and spill, so use 16 to halve the per-invocation B working set
|
||||
const uint TPB = NUM_COLS <= 4 ? 8 : 16;
|
||||
const uint NL = 32 / TPB; // l steps per invocation
|
||||
|
||||
void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
|
||||
const uint ib32 = itid / (TPB / 8);
|
||||
const uint l0 = (itid % (TPB / 8)) * NL;
|
||||
const uint y_idx = i * QUANT_K + 32 * ib32;
|
||||
|
||||
uint ibi = a_offset + first_row * num_blocks_per_row + i;
|
||||
@@ -16,11 +23,8 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32,
|
||||
const uint scale = (data_a[ibi].scales[ib32/2] >> (4 * (ib32 & 1))) & 0xF;
|
||||
const float dscale = d * (1 + 2 * scale);
|
||||
const uint qh = data_a[ibi].qh[ib32];
|
||||
FLOAT_TYPE sum[NUM_COLS];
|
||||
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
||||
sum[j] = 0.0;
|
||||
}
|
||||
[[unroll]] for (uint l = 0; l < 4; ++l) {
|
||||
[[unroll]] for (uint ll = 0; ll < NL; ++ll) {
|
||||
const uint l = l0 + ll;
|
||||
const u8vec2 qs = unpack8(uint32_t(data_a_packed16[ibi].qs[4 * ib32 + l])).xy; // vec4 used due to #12147
|
||||
const uint sign = data_a[ibi].signs[4 * ib32 + l];
|
||||
const vec4 grid0 = vec4(unpack8(iq3s_grid[qs.x | ((qh << (8 - 2*l)) & 0x100)]));
|
||||
@@ -30,7 +34,7 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32,
|
||||
const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]);
|
||||
const vec4 b4 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 1]);
|
||||
|
||||
sum[j] =
|
||||
const FLOAT_TYPE sum =
|
||||
fma(FLOAT_TYPE(b0.x), FLOAT_TYPE((sign & 1) != 0 ? -grid0.x : grid0.x),
|
||||
fma(FLOAT_TYPE(b0.y), FLOAT_TYPE((sign & 2) != 0 ? -grid0.y : grid0.y),
|
||||
fma(FLOAT_TYPE(b0.z), FLOAT_TYPE((sign & 4) != 0 ? -grid0.z : grid0.z),
|
||||
@@ -39,12 +43,11 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32,
|
||||
fma(FLOAT_TYPE(b4.y), FLOAT_TYPE((sign & 32) != 0 ? -grid1.y : grid1.y),
|
||||
fma(FLOAT_TYPE(b4.z), FLOAT_TYPE((sign & 64) != 0 ? -grid1.z : grid1.z),
|
||||
fma(FLOAT_TYPE(b4.w), FLOAT_TYPE((sign & 128) != 0 ? -grid1.w : grid1.w),
|
||||
sum[j]))))))));
|
||||
FLOAT_TYPE(0.0)))))))));
|
||||
|
||||
temp[j][n] = fma(dscale, sum, temp[j][n]);
|
||||
}
|
||||
}
|
||||
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
||||
temp[j][n] = fma(dscale, sum[j], temp[j][n]);
|
||||
}
|
||||
ibi += num_blocks_per_row;
|
||||
}
|
||||
}
|
||||
@@ -55,11 +58,11 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
|
||||
|
||||
const uint num_blocks_per_row = p.ncols / QUANT_K;
|
||||
|
||||
// 8 threads are used to process each block
|
||||
const uint blocks_per_wg = gl_WorkGroupSize.x/8;
|
||||
// TPB invocations are used to process each block
|
||||
const uint blocks_per_wg = gl_WorkGroupSize.x/TPB;
|
||||
const uint tid = gl_LocalInvocationID.x;
|
||||
const uint itid = tid % 8; // 0...7
|
||||
const uint ix = tid / 8;
|
||||
const uint itid = tid % TPB;
|
||||
const uint ix = tid / TPB;
|
||||
|
||||
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
||||
[[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
|
||||
|
||||
@@ -5522,6 +5522,15 @@ enum ggml_prec ggml_flash_attn_ext_get_prec(
|
||||
return (enum ggml_prec) prec_i32;
|
||||
}
|
||||
|
||||
void ggml_flash_attn_ext_set_n_kv_max(
|
||||
struct ggml_tensor * a,
|
||||
int32_t n_kv_max) {
|
||||
GGML_ASSERT(a->op == GGML_OP_FLASH_ATTN_EXT);
|
||||
GGML_ASSERT(n_kv_max >= 0);
|
||||
|
||||
ggml_set_op_params_i32(a, 4, n_kv_max);
|
||||
}
|
||||
|
||||
void ggml_flash_attn_ext_add_sinks(
|
||||
struct ggml_tensor * a,
|
||||
struct ggml_tensor * sinks) {
|
||||
|
||||
@@ -950,6 +950,9 @@ class MODEL_TENSOR(IntEnum):
|
||||
V_RESMPL_PROJ = auto() # minicpmv
|
||||
V_RESMPL_QUERY = auto() # minicpmv
|
||||
V_TOK_EMBD_IMG_BREAK = auto() # pixtral
|
||||
V_TOK_EMBD_IMG_START = auto() # deepseek4v
|
||||
V_TOK_EMBD_IMG_END = auto() # deepseek4v
|
||||
V_TOK_EMBD_IMG_PAD = auto() # deepseek4v
|
||||
V_MM_PATCH_MERGER = auto() # mistral small 3.1
|
||||
V_DS_NORM = auto() # qwen3vl
|
||||
V_DS_FC1 = auto() # qwen3vl
|
||||
@@ -1696,6 +1699,9 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
|
||||
MODEL_TENSOR.V_RESMPL_PROJ: "resampler.proj",
|
||||
MODEL_TENSOR.V_RESMPL_QUERY: "resampler.query",
|
||||
MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK: "v.token_embd.img_break", # pixtral
|
||||
MODEL_TENSOR.V_TOK_EMBD_IMG_START: "v.token_embd.img_start", # deepseek4v
|
||||
MODEL_TENSOR.V_TOK_EMBD_IMG_END: "v.token_embd.img_end", # deepseek4v
|
||||
MODEL_TENSOR.V_TOK_EMBD_IMG_PAD: "v.token_embd.img_pad", # deepseek4v
|
||||
MODEL_TENSOR.V_MM_PATCH_MERGER: "mm.patch_merger", # mistral small 3.1
|
||||
MODEL_TENSOR.V_DS_NORM: "v.deepstack.{bid}.norm",
|
||||
MODEL_TENSOR.V_DS_FC1: "v.deepstack.{bid}.fc1",
|
||||
@@ -2030,6 +2036,9 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
|
||||
MODEL_TENSOR.V_RESMPL_PROJ,
|
||||
MODEL_TENSOR.V_RESMPL_QUERY,
|
||||
MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK,
|
||||
MODEL_TENSOR.V_TOK_EMBD_IMG_START,
|
||||
MODEL_TENSOR.V_TOK_EMBD_IMG_END,
|
||||
MODEL_TENSOR.V_TOK_EMBD_IMG_PAD,
|
||||
MODEL_TENSOR.V_MM_PATCH_MERGER,
|
||||
MODEL_TENSOR.V_MM_MERGER_FC1,
|
||||
MODEL_TENSOR.V_MM_MERGER_FC2,
|
||||
@@ -5645,6 +5654,7 @@ class VisionProjectorType:
|
||||
DOTS3NOTE_A = "dots3note_a" # audio
|
||||
DEEPSEEKOCR = "deepseekocr"
|
||||
DEEPSEEKOCR2 = "deepseekocr2"
|
||||
DEEPSEEK4V = "deepseek4v"
|
||||
LFM2A = "lfm2a" # audio
|
||||
MUSIC_FLAMINGO = "musicflamingo" # audio
|
||||
GLM4V = "glm4v"
|
||||
|
||||
@@ -1476,6 +1476,7 @@ class TensorNameMap:
|
||||
## Vision encoder
|
||||
|
||||
MODEL_TENSOR.V_MMPROJ: (
|
||||
"aligner.w{bid}", # deepseek4v (w1 -> mm.1, w2 -> mm.2)
|
||||
"multi_modal_projector.linear_{bid}",
|
||||
"mm_projector.proj.linear_{bid}", # Kimi-K2.5
|
||||
"visual.merger.mlp.{bid}", # qwen2vl
|
||||
@@ -1515,6 +1516,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_EMBD_PATCH: (
|
||||
"vision.patch_embed.proj", # deepseek4v
|
||||
"model.vision_tower.vision_model.embeddings.patch_embedding", # Granite4Vision
|
||||
"vision_tower.vision_model.embeddings.patch_embedding",
|
||||
"model.vision_tower.embeddings.patch_embedding", # minicpmv4_6
|
||||
@@ -1570,6 +1572,7 @@ class TensorNameMap:
|
||||
|
||||
# TODO: I think these should all be moved to mapping_cfg?
|
||||
MODEL_TENSOR.V_ENC_EMBD_IMGNL: (
|
||||
"image_newline", # deepseek4v
|
||||
"model.image_newline", # Deepseek-OCR, Granite4Vision
|
||||
"vit.perceive.image_newline", # HunyuanVL
|
||||
),
|
||||
@@ -1580,6 +1583,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_ATTN_QKV: (
|
||||
"vision.blocks.{bid}.attn.wqkv", # deepseek4v
|
||||
"visual.blocks.{bid}.attn.qkv", # qwen3vl
|
||||
"vision_tower.blocks.{bid}.attn.qkv", # dots.ocr
|
||||
"vision_encoder.blocks.{bid}.attn.qkv", # dots3note
|
||||
@@ -1667,6 +1671,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_INPUT_NORM: (
|
||||
"vision.blocks.{bid}.norm1", # deepseek4v
|
||||
"model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm1", # Granite4Vision
|
||||
"vision_tower.vision_model.encoder.layers.{bid}.layer_norm1",
|
||||
"model.vision_tower.encoder.layers.{bid}.layer_norm1", # minicpmv4_6
|
||||
@@ -1692,6 +1697,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_ATTN_O: (
|
||||
"vision.blocks.{bid}.attn.wo", # deepseek4v
|
||||
"model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj", # Granite4Vision
|
||||
"vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj",
|
||||
"model.vision_tower.encoder.layers.{bid}.self_attn.out_proj", # minicpmv4_6
|
||||
@@ -1723,6 +1729,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_POST_ATTN_NORM: (
|
||||
"vision.blocks.{bid}.norm2", # deepseek4v
|
||||
"model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm2", # Granite4Vision
|
||||
"vision_tower.vision_model.encoder.layers.{bid}.layer_norm2",
|
||||
"model.vision_tower.encoder.layers.{bid}.layer_norm2", # minicpmv4_6
|
||||
@@ -1749,6 +1756,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_FFN_UP: (
|
||||
"vision.blocks.{bid}.mlp.w1_up", # deepseek4v (split from fused w1)
|
||||
"vision_encoder.blocks.{bid}.mlp.fc3", # dots3note
|
||||
"model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1", # Granite4Vision
|
||||
"vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1",
|
||||
@@ -1775,6 +1783,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_FFN_GATE: (
|
||||
"vision.blocks.{bid}.mlp.w1_gate", # deepseek4v (split from fused w1)
|
||||
"vision_encoder.blocks.{bid}.mlp.fc1", # dots3note
|
||||
"vision_tower.transformer.layers.{bid}.feed_forward.gate_proj", # pixtral-hf
|
||||
"vision_encoder.transformer.layers.{bid}.feed_forward.w1", # pixtral
|
||||
@@ -1784,6 +1793,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_FFN_DOWN: (
|
||||
"vision.blocks.{bid}.mlp.w2", # deepseek4v
|
||||
"vision_encoder.blocks.{bid}.mlp.fc2", # dots3note
|
||||
"model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2", # Granite4Vision
|
||||
"vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2",
|
||||
@@ -1869,6 +1879,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_POST_NORM: (
|
||||
"vision.norm", # deepseek4v
|
||||
"model.vision_tower.vision_model.post_layernorm", # Granite4Vision
|
||||
"vision_tower.vision_model.post_layernorm",
|
||||
"model.vision_tower.post_layernorm", # minicpmv4_6
|
||||
@@ -1960,6 +1971,18 @@ class TensorNameMap:
|
||||
"v.token_embd.img_break", # for pixtral, this is a generated vector
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_TOK_EMBD_IMG_START: (
|
||||
"image_start", # deepseek4v
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_TOK_EMBD_IMG_END: (
|
||||
"image_end", # deepseek4v
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_TOK_EMBD_IMG_PAD: (
|
||||
"image_pad", # deepseek4v
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_MM_PATCH_MERGER: (
|
||||
"multi_modal_projector.patch_merger.merging_layer", # mistral small 3.1 - hf
|
||||
"patch_merger.merging_layer", # mistral
|
||||
|
||||
@@ -1100,6 +1100,7 @@ bool llm_arch_supports_rs_rollback(const llm_arch & arch) {
|
||||
switch (arch) {
|
||||
case LLM_ARCH_QWEN35:
|
||||
case LLM_ARCH_QWEN35MOE:
|
||||
case LLM_ARCH_QWEN4EXP:
|
||||
case LLM_ARCH_DEEPSEEK4:
|
||||
case LLM_ARCH_NEMOTRON_H:
|
||||
case LLM_ARCH_NEMOTRON_H_MOE:
|
||||
@@ -1142,6 +1143,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
|
||||
case LLM_ARCH_BAILINGMOE3:
|
||||
case LLM_ARCH_KIMI_K3:
|
||||
case LLM_ARCH_QWEN3TTS:
|
||||
case LLM_ARCH_QWEN4EXP: // TODO: fix test-llama-archs
|
||||
return false;
|
||||
default:
|
||||
return true;
|
||||
|
||||
+15
-7
@@ -128,8 +128,9 @@ llama_context::llama_context(
|
||||
cparams.embeddings_layer_inp.resize(hparams.n_layer() + 1, false);
|
||||
embd_layer_inp.resize(hparams.n_layer() + 1);
|
||||
|
||||
cparams.ctx_type = params.ctx_type;
|
||||
cparams.pooling_type = params.pooling_type;
|
||||
cparams.ctx_type = params.ctx_type;
|
||||
cparams.rope_scaling_type = params.rope_scaling_type;
|
||||
cparams.pooling_type = params.pooling_type;
|
||||
|
||||
cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx;
|
||||
cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base;
|
||||
@@ -163,17 +164,16 @@ llama_context::llama_context(
|
||||
}
|
||||
}
|
||||
|
||||
auto rope_scaling_type = params.rope_scaling_type;
|
||||
if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) {
|
||||
rope_scaling_type = hparams.rope_scaling_type_train;
|
||||
if (cparams.rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) {
|
||||
cparams.rope_scaling_type = hparams.rope_scaling_type_train;
|
||||
}
|
||||
|
||||
if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_NONE) {
|
||||
if (cparams.rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_NONE) {
|
||||
cparams.rope_freq_scale = 1.0f; // never scale if scaling type is none
|
||||
}
|
||||
|
||||
if (cparams.yarn_ext_factor < 0.0f) { // negative indicates 'not set'
|
||||
cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f;
|
||||
cparams.yarn_ext_factor = cparams.rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f;
|
||||
}
|
||||
|
||||
if (cparams.yarn_ext_factor != 0) {
|
||||
@@ -3744,6 +3744,14 @@ llama_context * llama_init_from_model(
|
||||
|
||||
try {
|
||||
auto * ctx = new llama_context(*model, params);
|
||||
const auto & cparams = ctx->get_cparams();
|
||||
|
||||
if (cparams.rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN && cparams.rope_freq_scale != model->hparams.rope_freq_scale_train) {
|
||||
LLAMA_LOG_INFO("%s: custom YaRN scaling detected, re-adjusting n_ctx_train(%u)...\n", __func__, model->hparams.n_ctx_train);
|
||||
model->hparams.n_ctx_train = cparams.n_ctx_orig_yarn / cparams.rope_freq_scale;
|
||||
LLAMA_LOG_INFO("%s: n_ctx_train adjusted to %u\n", __func__, model->hparams.n_ctx_train);
|
||||
}
|
||||
|
||||
return ctx;
|
||||
} catch (const std::exception & err) {
|
||||
LLAMA_LOG_ERROR("%s: failed to initialize the context: %s\n", __func__, err.what());
|
||||
|
||||
@@ -57,6 +57,7 @@ struct llama_cparams {
|
||||
std::vector<bool> embeddings_layer_inp; // [n_layer()] extract input embeddings for layer
|
||||
|
||||
enum llama_context_type ctx_type;
|
||||
enum llama_rope_scaling_type rope_scaling_type;
|
||||
enum llama_pooling_type pooling_type;
|
||||
|
||||
ggml_backend_sched_eval_callback cb_eval;
|
||||
|
||||
+10
-7
@@ -2541,6 +2541,7 @@ ggml_tensor * llm_graph_context::build_attn_mha(
|
||||
ggml_tensor * kq_mask,
|
||||
ggml_tensor * sinks,
|
||||
ggml_tensor * v_mla,
|
||||
int64_t n_kv_max,
|
||||
float kq_scale,
|
||||
int il) const {
|
||||
const bool v_trans = v->nb[1] > v->nb[2];
|
||||
@@ -2578,6 +2579,8 @@ ggml_tensor * llm_graph_context::build_attn_mha(
|
||||
res->add_fused_node({LLM_FUSED_OP_FLASH_ATTN, cur, il});
|
||||
|
||||
ggml_flash_attn_ext_add_sinks(cur, sinks);
|
||||
GGML_ASSERT(n_kv_max >= 0 && n_kv_max <= INT32_MAX);
|
||||
ggml_flash_attn_ext_set_n_kv_max(cur, static_cast<int32_t>(n_kv_max));
|
||||
ggml_flash_attn_ext_set_prec (cur, GGML_PREC_F32);
|
||||
|
||||
if (v_mla) {
|
||||
@@ -2727,7 +2730,7 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
ggml_tensor * k = k_cur;
|
||||
ggml_tensor * v = v_cur;
|
||||
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, 0, kq_scale, il);
|
||||
cb(cur, "kqv_out", il);
|
||||
|
||||
if (wo) {
|
||||
@@ -2826,7 +2829,7 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
|
||||
ggml_tensor * v = mctx_cur->get_v(ctx0, il);
|
||||
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, 0, kq_scale, il);
|
||||
cb(cur, "kqv_out", il);
|
||||
|
||||
if (inp->self_v_rot) {
|
||||
@@ -2917,7 +2920,7 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
|
||||
ggml_tensor * v = ggml_view_4d(ctx0, k, v_cur->ne[0], k->ne[1], k->ne[2], k->ne[3], k->nb[1], k->nb[2], k->nb[3], 0);
|
||||
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, 0, kq_scale, il);
|
||||
cb(cur, "kqv_out", il);
|
||||
|
||||
if (wo) {
|
||||
@@ -3002,7 +3005,7 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
|
||||
ggml_tensor * v = ggml_view_4d(ctx0, k, v_cur->ne[0], k->ne[1], k->ne[2], k->ne[3], k->nb[1], k->nb[2], k->nb[3], 0);
|
||||
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask_top_k, sinks, v_mla, kq_scale, il);
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask_top_k, sinks, v_mla, top_k->ne[0], kq_scale, il);
|
||||
cb(cur, "kqv_out", il);
|
||||
|
||||
if (wo) {
|
||||
@@ -3081,7 +3084,7 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
|
||||
ggml_tensor * v = mctx_cur->get_v(ctx0, il);
|
||||
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, 0, kq_scale, il);
|
||||
cb(cur, "kqv_out", il);
|
||||
|
||||
if (v_rot) {
|
||||
@@ -3152,7 +3155,7 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
|
||||
ggml_tensor * v = ggml_view_4d(ctx0, k, v_cur->ne[0], k->ne[1], k->ne[2], k->ne[3], k->nb[1], k->nb[2], k->nb[3], 0);
|
||||
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, 0, kq_scale, il);
|
||||
cb(cur, "kqv_out", il);
|
||||
|
||||
if (k_rot) {
|
||||
@@ -3211,7 +3214,7 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
ggml_tensor * k = k_cur;
|
||||
ggml_tensor * v = v_cur;
|
||||
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, 0, kq_scale, il);
|
||||
cb(cur, "kqv_out", il);
|
||||
|
||||
if (wo) {
|
||||
|
||||
@@ -1171,6 +1171,7 @@ struct llm_graph_context {
|
||||
ggml_tensor * kq_mask,
|
||||
ggml_tensor * sinks, // [n_head_q]
|
||||
ggml_tensor * v_mla, // [n_embd_head_v_mla, n_embd_head_v, n_head_v]
|
||||
int64_t n_kv_max,
|
||||
float kq_scale,
|
||||
int il) const;
|
||||
|
||||
|
||||
+21
-54
@@ -6,7 +6,6 @@
|
||||
#include "llama-context.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
@@ -1841,58 +1840,10 @@ void llama_kv_cache::get_prev_tokens(const llama_ubatch & ubatch, uint32_t n, st
|
||||
return;
|
||||
}
|
||||
|
||||
// note: apply_ubatch() has already stored the current ubatch
|
||||
// the window below thus covers tokens of this very ubatch as well, which is what we want
|
||||
llama_pos p_min = std::numeric_limits<llama_pos>::max();
|
||||
llama_pos p_max = std::numeric_limits<llama_pos>::min();
|
||||
|
||||
std::bitset<LLAMA_MAX_SEQ> seqs;
|
||||
|
||||
for (uint32_t i = 0; i < n_tokens; ++i) {
|
||||
p_min = std::min(p_min, ubatch.pos[i]);
|
||||
p_max = std::max(p_max, ubatch.pos[i]);
|
||||
}
|
||||
|
||||
for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) {
|
||||
seqs.set(ubatch.seq_id_unq[s]);
|
||||
}
|
||||
|
||||
const llama_pos w0 = p_min - (llama_pos) n;
|
||||
|
||||
// (seq_id, pos) -> token, for every cell that could be a predecessor of a ubatch token
|
||||
std::unordered_map<uint64_t, llama_token> hist;
|
||||
|
||||
const auto key = [](llama_seq_id seq_id, llama_pos pos) {
|
||||
return ((uint64_t) seq_id << 32) | (uint32_t) pos;
|
||||
};
|
||||
|
||||
// handle M-RoPE gaps: multiple tokens share the same temporal pos
|
||||
// TODO @ngxson : improve this in the future
|
||||
std::array<std::pair<llama_pos, llama_token>, LLAMA_MAX_SEQ> below;
|
||||
below.fill({ -1, LLAMA_TOKEN_NULL });
|
||||
|
||||
for (uint32_t s = 0; s < n_stream; ++s) {
|
||||
// p_max inclusive: an embd token looks up cells at its own (shared) position
|
||||
v_cells[s].for_each_token_in(seqs, 0, p_max + 1,
|
||||
[&](llama_seq_id seq_id, llama_pos pos, llama_token tok) {
|
||||
if (pos >= w0) {
|
||||
hist[key(seq_id, pos)] = tok;
|
||||
} else if (pos > below[seq_id].first) {
|
||||
below[seq_id] = { pos, tok };
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// the token at pos p, or the nearest earlier one when p falls in an M-RoPE gap
|
||||
const auto lookup = [&](llama_seq_id seq_id, llama_pos p) -> llama_token {
|
||||
for (llama_pos q = p; q >= w0; --q) {
|
||||
const auto it = hist.find(key(seq_id, q));
|
||||
if (it != hist.end()) {
|
||||
return it->second;
|
||||
}
|
||||
}
|
||||
return below[seq_id].second;
|
||||
};
|
||||
// note: apply_ubatch() has already stored the current ubatch, so the cells cover the tokens
|
||||
// of this very ubatch as well, which is what we want
|
||||
// the nearest cell at or before a position also resolves M-RoPE gaps, where multiple tokens
|
||||
// share the same temporal pos
|
||||
|
||||
// an embd (multimodal) ubatch can repeat one position for a whole image, so positions
|
||||
// do not encode the token order; resolve its predecessors by ubatch order instead
|
||||
@@ -1930,7 +1881,7 @@ void llama_kv_cache::get_prev_tokens(const llama_ubatch & ubatch, uint32_t n, st
|
||||
continue;
|
||||
}
|
||||
|
||||
res[i*n + j] = lookup(seq_id, p);
|
||||
res[i*n + j] = v_cells[seq_to_stream[seq_id]].seq_pos_tok_le(seq_id, p);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2392,6 +2343,12 @@ bool llama_kv_cache::state_read_meta(llama_io_read_i & io, uint32_t strm, uint32
|
||||
|
||||
ubatch.seq_id_unq[0] = dest_seq_id;
|
||||
|
||||
// the ext as it was saved, to put back after apply_ubatch()
|
||||
std::vector<llama_kv_cell_ext> exts;
|
||||
if (has_cell_ext()) {
|
||||
exts.resize(cell_count);
|
||||
}
|
||||
|
||||
for (uint32_t i = 0; i < cell_count; ++i) {
|
||||
llama_pos pos;
|
||||
uint32_t n_seq_id;
|
||||
@@ -2415,6 +2372,8 @@ bool llama_kv_cache::state_read_meta(llama_io_read_i & io, uint32_t strm, uint32
|
||||
|
||||
// apply_ubatch() below restores ext.tok from the ubatch tokens
|
||||
ubatch.token[i] = ext.tok;
|
||||
|
||||
exts[i] = ext;
|
||||
}
|
||||
|
||||
// read the sequence id, but directly discard it - we will use dest_seq_id instead
|
||||
@@ -2466,6 +2425,14 @@ bool llama_kv_cache::state_read_meta(llama_io_read_i & io, uint32_t strm, uint32
|
||||
// see: https://github.com/ggml-org/llama.cpp/pull/16825#issuecomment-3460868350
|
||||
apply_ubatch(sinfo, ubatch);
|
||||
|
||||
// apply_ubatch() takes the 2D position from the ubatch, and that ubatch is built with this
|
||||
// cache's own n_pos_per_embd. a cache that does not use M-RoPE itself but mirrors one that
|
||||
// does (the qwen4exp QSA indexer) would drop x and y. put the saved ext back instead, which
|
||||
// is what the whole-context path below already does.
|
||||
for (uint32_t i = 0; i < (uint32_t) exts.size(); ++i) {
|
||||
cells.ext_set(sinfo.idxs[0][i], exts[i]);
|
||||
}
|
||||
|
||||
LLAMA_LOG_DEBUG("%s: cell_count = %d, dest_seq_id = %d\n", __func__, cell_count, dest_seq_id);
|
||||
|
||||
// DEBUG CHECK: verify that all cells were allocated and have correct seq_id and pos values
|
||||
|
||||
+37
-43
@@ -6,7 +6,7 @@
|
||||
#include <bitset>
|
||||
#include <cassert>
|
||||
#include <cstring>
|
||||
#include <map>
|
||||
#include <limits>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
|
||||
@@ -35,6 +35,8 @@ struct llama_kv_cell_ext {
|
||||
// TODO: add unit tests
|
||||
class llama_kv_cells {
|
||||
public:
|
||||
using seq_set_t = std::bitset<LLAMA_MAX_SEQ>;
|
||||
|
||||
void reset() {
|
||||
for (uint32_t i = 0; i < pos.size(); ++i) {
|
||||
pos[i] = -1;
|
||||
@@ -246,7 +248,7 @@ public:
|
||||
assert(seq_id >= 0);
|
||||
|
||||
seq[i].reset(seq_id);
|
||||
seq_pos_dec(seq_id, pos[i]);
|
||||
seq_pos_dec(seq_id, i);
|
||||
|
||||
if (seq[i].none()) {
|
||||
pos[i] = -1;
|
||||
@@ -270,7 +272,7 @@ public:
|
||||
seq[i].reset();
|
||||
|
||||
seq[i].set(seq_id);
|
||||
seq_pos_inc(seq_id, pos[i]);
|
||||
seq_pos_inc(seq_id, i);
|
||||
|
||||
return false;
|
||||
}
|
||||
@@ -301,6 +303,13 @@ public:
|
||||
return seq[i].count();
|
||||
}
|
||||
|
||||
// the full set of sequences this cell is visible to
|
||||
const seq_set_t & seq_get_all(uint32_t i) const {
|
||||
assert(i < pos.size());
|
||||
|
||||
return seq[i];
|
||||
}
|
||||
|
||||
// check if the cell contains seq_id
|
||||
bool seq_has(uint32_t i, llama_seq_id seq_id) const {
|
||||
assert(i < pos.size());
|
||||
@@ -309,28 +318,22 @@ public:
|
||||
return seq[i].test(seq_id);
|
||||
}
|
||||
|
||||
// gather the token ids of the cells in `seqs` with position in [p0, p1)
|
||||
// the callback receives (seq_id, pos, token) for every such (cell, seq) pair
|
||||
// the token of the cell of sequence seq_id at the largest position <= p
|
||||
// when several cells share that position, the one with the highest index wins
|
||||
// return LLAMA_TOKEN_NULL if the sequence has no cell at or before p
|
||||
// note: used by n-gram input embeddings to recover the tokens preceding a ubatch
|
||||
template<typename F>
|
||||
void for_each_token_in(const std::bitset<LLAMA_MAX_SEQ> & seqs, llama_pos p0, llama_pos p1, F && f) const {
|
||||
for (const auto & i : used) {
|
||||
if (pos[i] < p0 || pos[i] >= p1) {
|
||||
continue;
|
||||
}
|
||||
llama_token seq_pos_tok_le(llama_seq_id seq_id, llama_pos p) const {
|
||||
assert(seq_id >= 0);
|
||||
assert(seq_id < LLAMA_MAX_SEQ);
|
||||
|
||||
const auto m = seq[i] & seqs;
|
||||
const auto & sp = seq_pos[seq_id];
|
||||
|
||||
// a cell carries a handful of sequences at most, out of LLAMA_MAX_SEQ
|
||||
size_t left = m.count();
|
||||
|
||||
for (llama_seq_id s = 0; left > 0 && s < (llama_seq_id) LLAMA_MAX_SEQ; ++s) {
|
||||
if (m.test(s)) {
|
||||
f(s, pos[i], ext[i].tok);
|
||||
--left;
|
||||
}
|
||||
}
|
||||
auto it = sp.upper_bound({ p, std::numeric_limits<uint32_t>::max() });
|
||||
if (it == sp.begin()) {
|
||||
return LLAMA_TOKEN_NULL;
|
||||
}
|
||||
|
||||
return ext[(--it)->second].tok;
|
||||
}
|
||||
|
||||
// note: call only if the cell is not empty and the seq_id is not in the cell
|
||||
@@ -340,7 +343,7 @@ public:
|
||||
assert(!seq[i].test(seq_id));
|
||||
|
||||
seq[i].set(seq_id);
|
||||
seq_pos_inc(seq_id, pos[i]);
|
||||
seq_pos_inc(seq_id, i);
|
||||
}
|
||||
|
||||
// return the sequence id of this cell
|
||||
@@ -367,8 +370,6 @@ public:
|
||||
return -1;
|
||||
}
|
||||
|
||||
assert(seq_pos[seq_id].begin()->second > 0);
|
||||
|
||||
return seq_pos[seq_id].begin()->first;
|
||||
}
|
||||
|
||||
@@ -382,8 +383,6 @@ public:
|
||||
return -1;
|
||||
}
|
||||
|
||||
assert(seq_pos[seq_id].rbegin()->second > 0);
|
||||
|
||||
return seq_pos[seq_id].rbegin()->first;
|
||||
}
|
||||
|
||||
@@ -511,41 +510,36 @@ private:
|
||||
//
|
||||
std::vector<llama_pos> shift;
|
||||
|
||||
using seq_set_t = std::bitset<LLAMA_MAX_SEQ>;
|
||||
|
||||
// the bitset seq[i] tells us which sequences are currently occupying the i-th cell
|
||||
std::vector<seq_set_t> seq;
|
||||
|
||||
// the set seq_pos[s][p] tells us how many times the position p is currently present for sequence s
|
||||
// if the position p is not present, seq_pos[s][p] is not set
|
||||
// the set seq_pos[s] holds one (pos, cell) pair per cell that carries sequence s, ordered by position
|
||||
// this way seq_pos[s].begin() and seq_pos[s].rbegin() give us the min/max positions currently in the cache
|
||||
// and upper_bound() on a position finds the nearest cell of the sequence in logarithmic time
|
||||
//
|
||||
// note that we cannot a use an std::set because in some cases a position can occur more than once for the same seq:
|
||||
// the cell index is part of the key because a position can occur more than once for the same seq:
|
||||
// - during performing a cache reuse via (rm + add)
|
||||
// - some vision models have input embeddings with repeating positions
|
||||
//
|
||||
std::map<llama_pos, int> seq_pos[LLAMA_MAX_SEQ];
|
||||
std::set<std::pair<llama_pos, uint32_t>> seq_pos[LLAMA_MAX_SEQ];
|
||||
|
||||
// helper functions for updating `seq_pos`, once cell at a time:
|
||||
|
||||
void seq_pos_dec(llama_seq_id s, llama_pos p) {
|
||||
auto it = seq_pos[s].find(p);
|
||||
assert(it != seq_pos[s].end());
|
||||
|
||||
if (--it->second == 0) {
|
||||
seq_pos[s].erase(it);
|
||||
}
|
||||
void seq_pos_dec(llama_seq_id s, uint32_t i) {
|
||||
const auto n = seq_pos[s].erase({ pos[i], i });
|
||||
assert(n == 1);
|
||||
GGML_UNUSED(n);
|
||||
}
|
||||
|
||||
void seq_pos_inc(llama_seq_id s, llama_pos p) {
|
||||
seq_pos[s][p]++;
|
||||
void seq_pos_inc(llama_seq_id s, uint32_t i) {
|
||||
seq_pos[s].insert({ pos[i], i });
|
||||
}
|
||||
|
||||
// remove cell i
|
||||
void seq_pos_rm(uint32_t i) {
|
||||
for (int s = 0; s < LLAMA_MAX_SEQ; ++s) {
|
||||
if (seq[i].test(s)) {
|
||||
seq_pos_dec(s, pos[i]);
|
||||
seq_pos_dec(s, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -554,7 +548,7 @@ private:
|
||||
void seq_pos_add(uint32_t i) {
|
||||
for (int s = 0; s < LLAMA_MAX_SEQ; ++s) {
|
||||
if (seq[i].test(s)) {
|
||||
seq_pos_inc(s, pos[i]);
|
||||
seq_pos_inc(s, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+329
-115
@@ -5,6 +5,7 @@
|
||||
#include "llama-io.h"
|
||||
#include "llama-model.h"
|
||||
|
||||
|
||||
#include <algorithm>
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
@@ -50,6 +51,10 @@ llama_memory_hybrid_idx::llama_memory_hybrid_idx(
|
||||
std::fill(hparams_idx.n_head_kv_arr.begin(), hparams_idx.n_head_kv_arr.end(), 1);
|
||||
hparams_idx.n_embd_head_k_full = model.hparams.indexer_head_size;
|
||||
|
||||
// the cached indexer keys are raw, rotation happens after pooling at read time, so a
|
||||
// K-shift must not rotate them while the stream copies in the same update still apply
|
||||
hparams_idx.rope_type = LLAMA_ROPE_TYPE_NONE;
|
||||
|
||||
LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size);
|
||||
|
||||
return new llama_kv_cache(
|
||||
@@ -261,6 +266,324 @@ llama_kv_cache * llama_memory_hybrid_idx::get_mem_idx() const {
|
||||
return mem_idx.get();
|
||||
}
|
||||
|
||||
void llama_memory_hybrid_idx::set_input_qsa(
|
||||
ggml_tensor * cell_blk,
|
||||
ggml_tensor * blk_cells,
|
||||
ggml_tensor * blk_pos,
|
||||
ggml_tensor * bias,
|
||||
const llama_ubatch * ubatch,
|
||||
uint32_t ratio,
|
||||
bool blk_bias) const {
|
||||
GGML_ASSERT(ratio > 0);
|
||||
GGML_ASSERT(get_mem_idx() != nullptr);
|
||||
|
||||
GGML_ASSERT(ggml_backend_buffer_is_host(cell_blk->buffer));
|
||||
|
||||
const int64_t n_kv = cell_blk->ne[0];
|
||||
const int64_t n_ns = cell_blk->ne[1]; // streams in this ubatch
|
||||
const int64_t n_blocks = blk_pos->ne[0]/(4*n_ns);
|
||||
const int64_t n_tokens = ubatch->n_tokens;
|
||||
const int64_t r = ratio;
|
||||
|
||||
GGML_ASSERT(n_tokens % n_ns == 0);
|
||||
const int64_t n_tps = n_tokens/n_ns; // tokens per stream
|
||||
|
||||
int32_t * dst_cell_blk = (int32_t *) cell_blk->data;
|
||||
int32_t * dst_blk_cells = (int32_t *) blk_cells->data;
|
||||
int32_t * dst_blk_pos = (int32_t *) blk_pos->data;
|
||||
float * dst_bias = (float *) bias->data;
|
||||
|
||||
// a block is keyed on (sequence set, index bucket): a unified cache counts every sequence
|
||||
// from zero, so the bucket alone would pool two sequences into one block
|
||||
GGML_ASSERT(r <= 64);
|
||||
const uint64_t slots_full = r == 64 ? ~uint64_t(0) : ((uint64_t(1) << r) - 1);
|
||||
|
||||
// TODO: this runs per ubatch and is O(n_kv) per stream, about 865 us at 33k context. the cost
|
||||
// is the per-cell scan rather than these allocations, so hoisting them buys nothing
|
||||
std::vector<int32_t> blk_of(n_kv);
|
||||
std::vector<int32_t> cell_grp(n_kv);
|
||||
std::vector<int32_t> grp_head(n_blocks);
|
||||
std::vector<int32_t> grp_next;
|
||||
std::vector<int32_t> grp_first;
|
||||
std::vector<int32_t> grp_slot0;
|
||||
std::vector<uint64_t> grp_slots;
|
||||
std::vector<int32_t> grp_bid;
|
||||
std::vector<int32_t> bid_idx;
|
||||
std::vector<int32_t> bid_cell;
|
||||
std::vector<int32_t> bid_slot0;
|
||||
|
||||
std::vector<int32_t> order;
|
||||
std::vector<int32_t> rank;
|
||||
|
||||
std::fill(dst_blk_pos, dst_blk_pos + 4*n_blocks*n_ns, 0);
|
||||
|
||||
for (int64_t s = 0; s < n_ns; ++s) {
|
||||
// ubatch index s*n_tps belongs to this stream; ask which cells array it uses
|
||||
const llama_seq_id seq_of_stream = ubatch->seq_id[s*n_tps][0];
|
||||
const auto & cells = get_mem_idx()->get_cells(seq_of_stream);
|
||||
|
||||
int32_t * cur_cell_blk = dst_cell_blk + s*n_kv;
|
||||
int32_t * cur_blk_cells = dst_blk_cells + s*(r*n_blocks);
|
||||
|
||||
std::fill(cur_blk_cells, cur_blk_cells + r*n_blocks, 0);
|
||||
|
||||
bid_idx .clear();
|
||||
bid_cell .clear();
|
||||
bid_slot0.clear();
|
||||
|
||||
int n_seq_present = 0;
|
||||
|
||||
for (int sq = 0; sq < LLAMA_MAX_SEQ && n_seq_present < 2; ++sq) {
|
||||
if (cells.seq_pos_min(sq) >= 0) {
|
||||
n_seq_present++;
|
||||
}
|
||||
}
|
||||
|
||||
const bool one_seq = n_seq_present <= 1;
|
||||
|
||||
// a cell no block covers needs its own -inf, which a per-block bias cannot carry
|
||||
// every cache path keeps the position below the cell window, so this stays false
|
||||
bool oor = false;
|
||||
|
||||
bool dup = false;
|
||||
|
||||
bool ranked = false;
|
||||
|
||||
auto group_cells = [&]() {
|
||||
// -1 means no usable block: an incomplete or short group cannot be pooled
|
||||
std::fill(blk_of.begin(), blk_of.end(), -1);
|
||||
std::fill(cell_grp.begin(), cell_grp.end(), -1);
|
||||
std::fill(grp_head.begin(), grp_head.end(), -1);
|
||||
|
||||
grp_next .clear();
|
||||
grp_first.clear();
|
||||
grp_slot0.clear();
|
||||
grp_slots.clear();
|
||||
grp_bid .clear();
|
||||
|
||||
oor = false;
|
||||
dup = false;
|
||||
|
||||
for (int64_t j = 0; j < n_kv; ++j) {
|
||||
if (cells.is_empty(j)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const int64_t idx = ranked ? rank[j] : cells.pos_get(j);
|
||||
const int64_t pb = idx/r;
|
||||
|
||||
if (pb >= n_blocks) {
|
||||
oor = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
int32_t g = -1;
|
||||
|
||||
for (int32_t c = grp_head[pb]; c >= 0; c = grp_next[c]) {
|
||||
if (one_seq || cells.seq_get_all((uint32_t) grp_first[c]) == cells.seq_get_all((uint32_t) j)) {
|
||||
g = c;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (g < 0) {
|
||||
g = (int32_t) grp_first.size();
|
||||
|
||||
grp_next .push_back(grp_head[pb]);
|
||||
grp_first.push_back((int32_t) j);
|
||||
grp_slot0.push_back(-1);
|
||||
grp_slots.push_back(0);
|
||||
grp_bid .push_back(-1);
|
||||
|
||||
grp_head[pb] = g;
|
||||
}
|
||||
|
||||
const uint64_t bit = uint64_t(1) << (idx%r);
|
||||
|
||||
dup |= (grp_slots[g] & bit) != 0;
|
||||
|
||||
cell_grp[j] = g;
|
||||
grp_slots[g] |= bit;
|
||||
|
||||
if (idx%r == 0) {
|
||||
grp_slot0[g] = (int32_t) j;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
group_cells();
|
||||
|
||||
// mrope repeats one position across an image, so rank cells instead of using the position
|
||||
if (dup && ubatch->is_pos_2d() && one_seq) {
|
||||
order.clear();
|
||||
order.reserve(n_kv);
|
||||
|
||||
for (int64_t j = 0; j < n_kv; ++j) {
|
||||
if (!cells.is_empty(j)) {
|
||||
order.push_back((int32_t) j);
|
||||
}
|
||||
}
|
||||
|
||||
// same total order the mrope causal mask uses: pos, then ext.y, then ext.x
|
||||
std::sort(order.begin(), order.end(), [&cells](int32_t a, int32_t b) {
|
||||
const llama_pos pa = cells.pos_get(a);
|
||||
const llama_pos pb = cells.pos_get(b);
|
||||
|
||||
if (pa != pb) {
|
||||
return pa < pb;
|
||||
}
|
||||
|
||||
const auto & ea = cells.ext_get(a);
|
||||
|
||||
return cells.ext_get(b).is_2d_gt(ea.x, ea.y);
|
||||
});
|
||||
|
||||
rank.assign(n_kv, -1);
|
||||
|
||||
for (int64_t k = 0; k < (int64_t) order.size(); ++k) {
|
||||
rank[order[k]] = (int32_t) k;
|
||||
}
|
||||
|
||||
ranked = true;
|
||||
|
||||
group_cells();
|
||||
}
|
||||
|
||||
GGML_ASSERT((!blk_bias || !oor) && "qsa: cell position runs past the cell window");
|
||||
|
||||
int32_t n_bid = 0;
|
||||
|
||||
for (int64_t pb = 0; pb < n_blocks; ++pb) {
|
||||
for (int32_t g = grp_head[pb]; g >= 0; g = grp_next[g]) {
|
||||
if (grp_slots[g] != slots_full) {
|
||||
continue;
|
||||
}
|
||||
|
||||
grp_bid[g] = n_bid++;
|
||||
|
||||
bid_idx .push_back((int32_t) (pb*r));
|
||||
bid_cell .push_back(grp_first[g]);
|
||||
bid_slot0.push_back(grp_slot0[g]);
|
||||
}
|
||||
}
|
||||
|
||||
GGML_ASSERT(n_bid <= n_blocks);
|
||||
|
||||
for (int32_t b = 0; b < n_bid; ++b) {
|
||||
int32_t sec_pos[4] = { bid_idx[b], bid_idx[b], bid_idx[b], bid_idx[b] };
|
||||
|
||||
if (ranked) {
|
||||
const int32_t c = bid_slot0[b];
|
||||
const llama_pos p = cells.pos_get(c);
|
||||
const auto & e = cells.ext_get(c);
|
||||
|
||||
sec_pos[0] = p;
|
||||
sec_pos[1] = e.y;
|
||||
sec_pos[2] = e.x;
|
||||
sec_pos[3] = p;
|
||||
}
|
||||
|
||||
for (int64_t sec = 0; sec < 4; ++sec) {
|
||||
dst_blk_pos[sec*(n_blocks*n_ns) + s*n_blocks + b] = sec_pos[sec];
|
||||
}
|
||||
}
|
||||
|
||||
// unpooled cells all point at one spare block. a spare block exists only when some
|
||||
// cell is unpooled: n_bid == n_blocks means every cell sits in a full block.
|
||||
const bool have_dead = n_bid < n_blocks;
|
||||
const int32_t dead_bid = have_dead ? n_bid : n_blocks - 1;
|
||||
|
||||
for (int64_t j = 0; j < n_kv; ++j) {
|
||||
const int32_t g = cell_grp[j];
|
||||
|
||||
blk_of[j] = g < 0 ? -1 : grp_bid[g];
|
||||
|
||||
if (blk_of[j] >= 0) {
|
||||
const int64_t idx = ranked ? rank[j] : cells.pos_get(j);
|
||||
|
||||
cur_blk_cells[blk_of[j]*r + (idx%r)] = (int32_t) j;
|
||||
}
|
||||
|
||||
cur_cell_blk[j] = blk_of[j] < 0 ? dead_bid : blk_of[j];
|
||||
}
|
||||
|
||||
for (int64_t ii = 0; ii < n_tps; ++ii) {
|
||||
const int64_t i = s*n_tps + ii;
|
||||
const llama_seq_id seq_id = ubatch->seq_id[i][0];
|
||||
|
||||
int64_t q = ubatch->pos[i];
|
||||
|
||||
if (ranked) {
|
||||
const llama_pos qt = ubatch->pos[i];
|
||||
const llama_pos qy = ubatch->pos[i + n_tokens];
|
||||
const llama_pos qx = ubatch->pos[i + n_tokens*2];
|
||||
|
||||
int64_t lo = 0;
|
||||
int64_t hi = (int64_t) order.size();
|
||||
|
||||
while (lo < hi) {
|
||||
const int64_t mid = (lo + hi)/2;
|
||||
const int32_t c = order[mid];
|
||||
const llama_pos pc = cells.pos_get(c);
|
||||
|
||||
if (pc < qt || (pc == qt && !cells.ext_get(c).is_2d_gt(qx, qy))) {
|
||||
lo = mid + 1;
|
||||
} else {
|
||||
hi = mid;
|
||||
}
|
||||
}
|
||||
|
||||
q = lo - 1;
|
||||
}
|
||||
|
||||
// the tail is an incomplete block and is always visible, as in the reference
|
||||
const int64_t tail_start = (q + 1)/r*r;
|
||||
|
||||
if (blk_bias) {
|
||||
// a block sits wholly inside or outside the tail, so one value covers it
|
||||
// the caller adds the attention mask, which drops empty, foreign and future cells
|
||||
float * cur_blk_bias = dst_bias + i*n_blocks;
|
||||
|
||||
for (int64_t b = 0; b < n_blocks; ++b) {
|
||||
if (b >= n_bid || !cells.seq_has((uint32_t) bid_cell[b], seq_id)) {
|
||||
cur_blk_bias[b] = -INFINITY;
|
||||
continue;
|
||||
}
|
||||
|
||||
// finite, so it can never meet a -inf and produce a nan
|
||||
cur_blk_bias[b] = bid_idx[b] >= tail_start ? 1e9f : 0.0f;
|
||||
}
|
||||
|
||||
// the spare block holds the unpooled cells, which are the incomplete tail, so
|
||||
// it gets the tail value. it must stay finite: a sequence with fewer than
|
||||
// `ratio` cells owns no full block, and a row of -inf only gives a nan.
|
||||
if (have_dead) {
|
||||
cur_blk_bias[dead_bid] = 1e9f;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
float * cur_bias = dst_bias + i*n_kv;
|
||||
|
||||
for (int64_t j = 0; j < n_kv; ++j) {
|
||||
float v = -INFINITY;
|
||||
|
||||
if (!cells.is_empty(j) && cells.seq_has(j, seq_id)) {
|
||||
const int64_t idx = ranked ? rank[j] : cells.pos_get(j);
|
||||
|
||||
if (idx <= q) {
|
||||
// finite, so it can never meet a -inf and produce a nan
|
||||
v = idx >= tail_start ? 1e9f : (blk_of[j] < 0 ? -INFINITY : 0.0f);
|
||||
}
|
||||
}
|
||||
|
||||
cur_bias[j] = v;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// llama_memory_hybrid_idx_context
|
||||
//
|
||||
@@ -295,7 +618,10 @@ llama_memory_hybrid_idx_context::llama_memory_hybrid_idx_context(
|
||||
llama_context * lctx,
|
||||
bool optimize) :
|
||||
llama_memory_hybrid_context(mem, lctx, optimize),
|
||||
mem(mem) {}
|
||||
mem(mem),
|
||||
// update() applies a pending cross-stream seq_cp, else the copy keeps stale indexer keys
|
||||
ctx_idx(mem->get_mem_idx() == nullptr ? nullptr :
|
||||
mem->get_mem_idx()->init_update(lctx, optimize)) {}
|
||||
|
||||
llama_memory_hybrid_idx_context::llama_memory_hybrid_idx_context(
|
||||
llama_memory_hybrid_idx * mem,
|
||||
@@ -347,119 +673,7 @@ void llama_memory_hybrid_idx_context::set_input_qsa(
|
||||
const llama_ubatch * ubatch,
|
||||
uint32_t ratio,
|
||||
bool blk_bias) const {
|
||||
GGML_ASSERT(ratio > 0);
|
||||
GGML_ASSERT(mem != nullptr && mem->get_mem_idx() != nullptr);
|
||||
GGML_ASSERT(mem != nullptr);
|
||||
|
||||
GGML_ASSERT(ggml_backend_buffer_is_host(cell_blk->buffer));
|
||||
|
||||
const int64_t n_kv = cell_blk->ne[0];
|
||||
const int64_t n_ns = cell_blk->ne[1]; // streams in this ubatch
|
||||
const int64_t n_blocks = blk_pos->ne[0]/(4*n_ns);
|
||||
const int64_t n_tokens = ubatch->n_tokens;
|
||||
const int64_t r = ratio;
|
||||
|
||||
GGML_ASSERT(n_tokens % n_ns == 0);
|
||||
const int64_t n_tps = n_tokens/n_ns; // tokens per stream
|
||||
|
||||
int32_t * dst_cell_blk = (int32_t *) cell_blk->data;
|
||||
int32_t * dst_blk_cells = (int32_t *) blk_cells->data;
|
||||
int32_t * dst_blk_pos = (int32_t *) blk_pos->data;
|
||||
float * dst_bias = (float *) bias->data;
|
||||
|
||||
// block b covers [b*ratio, (b+1)*ratio), so its first token is at b*ratio
|
||||
// all mrope sections carry it: exact for text, approximate for images
|
||||
for (int64_t sec = 0; sec < 4; ++sec) {
|
||||
for (int64_t s = 0; s < n_ns; ++s) {
|
||||
for (int64_t b = 0; b < n_blocks; ++b) {
|
||||
dst_blk_pos[sec*(n_blocks*n_ns) + s*n_blocks + b] = (int32_t) (b*r);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// one pass per stream: cell j is a different token in each, so no mapping is shared
|
||||
std::vector<int32_t> blk_of(n_kv);
|
||||
std::vector<int32_t> filled(n_blocks);
|
||||
|
||||
for (int64_t s = 0; s < n_ns; ++s) {
|
||||
// ubatch index s*n_tps belongs to this stream; ask which cells array it uses
|
||||
const llama_seq_id seq_of_stream = ubatch->seq_id[s*n_tps][0];
|
||||
const auto & cells = mem->get_mem_idx()->get_cells(seq_of_stream);
|
||||
|
||||
int32_t * cur_cell_blk = dst_cell_blk + s*n_kv;
|
||||
int32_t * cur_blk_cells = dst_blk_cells + s*(r*n_blocks);
|
||||
|
||||
// an incomplete block cannot be pooled; the bias below forces those tail cells in
|
||||
// -1 means no usable block, and block 0 only keeps the gather in range
|
||||
std::fill(blk_of.begin(), blk_of.end(), -1);
|
||||
std::fill(filled.begin(), filled.end(), 0);
|
||||
std::fill(cur_blk_cells, cur_blk_cells + r*n_blocks, 0);
|
||||
|
||||
// a cell no block covers needs its own -inf, which a per-block bias cannot carry
|
||||
// every cache path keeps the position below the cell window, so this stays false
|
||||
bool oor = false;
|
||||
|
||||
for (int64_t j = 0; j < n_kv; ++j) {
|
||||
if (cells.is_empty(j)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const llama_pos p = cells.pos_get(j);
|
||||
const int64_t b = p/r;
|
||||
|
||||
if (b >= n_blocks) {
|
||||
oor = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
blk_of[j] = (int32_t) b;
|
||||
cur_blk_cells[b*r + (p%r)] = (int32_t) j;
|
||||
filled[b]++;
|
||||
}
|
||||
|
||||
GGML_ASSERT((!blk_bias || !oor) && "qsa: cell position runs past the cell window");
|
||||
|
||||
// per-block mode keeps an unpooled cell's real block, so the block's own -inf reaches it
|
||||
// per-cell mode carries that -inf itself and only needs the gather in range
|
||||
for (int64_t j = 0; j < n_kv; ++j) {
|
||||
if (blk_of[j] >= 0 && filled[blk_of[j]] < r && !blk_bias) {
|
||||
blk_of[j] = -1;
|
||||
}
|
||||
cur_cell_blk[j] = blk_of[j] < 0 ? 0 : blk_of[j];
|
||||
}
|
||||
|
||||
for (int64_t ii = 0; ii < n_tps; ++ii) {
|
||||
const int64_t i = s*n_tps + ii;
|
||||
const llama_seq_id seq_id = ubatch->seq_id[i][0];
|
||||
const llama_pos q = ubatch->pos[i];
|
||||
|
||||
// the tail is an incomplete block and is always visible, as in the reference
|
||||
const llama_pos tail_start = (q + 1)/r*r;
|
||||
|
||||
if (blk_bias) {
|
||||
// a block sits wholly inside or outside the tail, so one value covers it
|
||||
// the caller adds the attention mask, which drops empty, foreign and future cells
|
||||
float * cur_blk_bias = dst_bias + i*n_blocks;
|
||||
|
||||
for (int64_t b = 0; b < n_blocks; ++b) {
|
||||
// finite, so it can never meet a -inf and produce a nan
|
||||
cur_blk_bias[b] = b*r >= tail_start ? 1e9f : (filled[b] < r ? -INFINITY : 0.0f);
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
float * cur_bias = dst_bias + i*n_kv;
|
||||
|
||||
for (int64_t j = 0; j < n_kv; ++j) {
|
||||
float v = -INFINITY;
|
||||
|
||||
if (!cells.is_empty(j) && cells.seq_has(j, seq_id) && cells.pos_get(j) <= q) {
|
||||
// finite, so it can never meet a -inf and produce a nan
|
||||
v = cells.pos_get(j) >= tail_start ? 1e9f : (blk_of[j] < 0 ? -INFINITY : 0.0f);
|
||||
}
|
||||
|
||||
cur_bias[j] = v;
|
||||
}
|
||||
}
|
||||
}
|
||||
mem->set_input_qsa(cell_blk, blk_cells, blk_pos, bias, ubatch, ratio, blk_bias);
|
||||
}
|
||||
|
||||
@@ -75,6 +75,18 @@ public:
|
||||
|
||||
llama_kv_cache * get_mem_idx() const; // nullptr when the model carries no indexer
|
||||
|
||||
// block-compressed sparse attention (qwen4exp QSA) over the cells of the indexer cache.
|
||||
// Blocks cut the position line, not the cell array, so no caller assumes a contiguous layout:
|
||||
// cell_blk I32 [n_kv, ns] block each cell belongs to
|
||||
// blk_cells I32 [ratio*n_blocks, ns] cells making up each block
|
||||
// blk_pos I32 [4*n_blocks*ns] mrope position rows of each block's first token
|
||||
// bias F32 [n_kv, n_tokens/ns, ns] -inf where invisible, large where always visible
|
||||
// blk_bias asks for the bias per block instead: [n_blocks, n_tokens/ns, ns]
|
||||
// the caller then adds the attention mask, the only part of the bias that varies within a block
|
||||
void set_input_qsa(ggml_tensor * cell_blk, ggml_tensor * blk_cells, ggml_tensor * blk_pos,
|
||||
ggml_tensor * bias, const llama_ubatch * ubatch, uint32_t ratio,
|
||||
bool blk_bias) const;
|
||||
|
||||
private:
|
||||
// forget seq_id (all of it if seq_id < 0) in every cache at once, so a failed restore cannot leave the caches out of step
|
||||
// seq_id < 0 drops the whole context, as the caches themselves do on a failed restore
|
||||
@@ -123,20 +135,12 @@ public:
|
||||
// llama_memory_hybrid_idx_context specific API
|
||||
//
|
||||
|
||||
// nullptr with no indexer, and for the update context, which builds no sparse graph
|
||||
// nullptr with no indexer
|
||||
const llama_kv_cache_context * get_idx() const;
|
||||
|
||||
// streams in the current slot info, the `ns` of get_k/get_v; 1 if unified
|
||||
uint32_t get_n_stream() const;
|
||||
|
||||
// block-compressed sparse attention (qwen4exp QSA) over the cells of the indexer cache.
|
||||
// Blocks cut the position line, not the cell array, so no caller assumes a contiguous layout:
|
||||
// cell_blk I32 [n_kv, ns] block each cell belongs to
|
||||
// blk_cells I32 [ratio*n_blocks, ns] cells making up each block
|
||||
// blk_pos I32 [4*n_blocks*ns] mrope position rows of each block's first token
|
||||
// bias F32 [n_kv, n_tokens/ns, ns] -inf where invisible, large where always visible
|
||||
// blk_bias asks for the bias per block instead: [n_blocks, n_tokens/ns, ns]
|
||||
// the caller then adds the attention mask, the only part of the bias that varies within a block
|
||||
void set_input_qsa(ggml_tensor * cell_blk, ggml_tensor * blk_cells, ggml_tensor * blk_pos,
|
||||
ggml_tensor * bias, const llama_ubatch * ubatch, uint32_t ratio,
|
||||
bool blk_bias) const;
|
||||
@@ -148,7 +152,7 @@ private:
|
||||
// declared first, so it is initialised while sinfos_idx is still intact
|
||||
const std::vector<uint32_t> ns_ubatch;
|
||||
|
||||
// null unless the model has an indexer and this is a batch or full context
|
||||
// null unless the model has an indexer
|
||||
const llama_memory_context_ptr ctx_idx;
|
||||
|
||||
// mirrors the base class's ubatch cursor, which is private there
|
||||
|
||||
+5
-3
@@ -1375,6 +1375,8 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);
|
||||
ml.get_key(LLM_KV_BLOCK_COUNT, hparams.n_layer_all);
|
||||
GGML_ASSERT(hparams.n_layer_all > 0 && hparams.n_layer_all <= LLAMA_MAX_LAYERS);
|
||||
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
|
||||
GGML_ASSERT(hparams.n_layer_nextn <= hparams.n_layer_all);
|
||||
ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false);
|
||||
ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false);
|
||||
ml.get_key(LLM_KV_EXPERT_GROUP_COUNT, hparams.n_expert_groups, false);
|
||||
@@ -1434,8 +1436,8 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
|
||||
std::fill(hparams.swiglu_clamp_exp.begin(), hparams.swiglu_clamp_exp.end(), 0.0f);
|
||||
std::fill(hparams.swiglu_clamp_shexp.begin(), hparams.swiglu_clamp_shexp.end(), 0.0f);
|
||||
|
||||
ml.get_key_or_arr(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, hparams.n_layer(), false);
|
||||
ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head_arr, hparams.n_layer(), false);
|
||||
ml.get_key_or_arr(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, hparams.n_layer_all, false);
|
||||
ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head_arr, hparams.n_layer_all, false);
|
||||
|
||||
// Populate deepstack_mapping_arr - initialized to -1 (no deepstack)
|
||||
std::fill(hparams.deepstack_mapping_arr.begin(), hparams.deepstack_mapping_arr.end(), -1);
|
||||
@@ -1443,7 +1445,7 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
|
||||
// n_head_kv is optional, default to n_head
|
||||
hparams.n_head_kv_arr = hparams.n_head_arr;
|
||||
|
||||
ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT_KV, hparams.n_head_kv_arr, hparams.n_layer(), false);
|
||||
ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT_KV, hparams.n_head_kv_arr, hparams.n_layer_all, false);
|
||||
|
||||
bool rope_finetuned = false;
|
||||
ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false);
|
||||
|
||||
+58
-42
@@ -744,12 +744,28 @@ static ggml_type llama_tensor_get_type(quantize_state_impl & qs, const llama_mod
|
||||
// quantization implementation
|
||||
//
|
||||
|
||||
static size_t llama_tensor_quantize_impl(enum ggml_type new_type, const float * f32_data, void * new_data, const int64_t chunk_size, int64_t nrows, int64_t n_per_row, const float * imatrix, std::vector<std::thread> & workers, const int nthread) {
|
||||
// quantize rows [first_row, first_row + nrows), indexed globally across all expert matrices
|
||||
// note: chunks never cross an expert boundary since each expert has its own imatrix slice
|
||||
static size_t llama_tensor_quantize_impl(enum ggml_type new_type, const float * f32_data, void * new_data, const int64_t chunk_size, int64_t first_row, int64_t nrows, int64_t nrows_per_expert, int64_t n_per_row, const float * imatrix, std::vector<std::thread> & workers, const int nthread) {
|
||||
const size_t row_size = ggml_row_size(new_type, n_per_row);
|
||||
|
||||
auto imatrix_for_row = [=](int64_t row_global) {
|
||||
return imatrix ? imatrix + (row_global / nrows_per_expert) * n_per_row : nullptr;
|
||||
};
|
||||
|
||||
if (nthread < 2) {
|
||||
// single-thread
|
||||
size_t new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, nrows, n_per_row, imatrix);
|
||||
if (!ggml_validate_row_data(new_type, new_data, new_size)) {
|
||||
throw std::runtime_error("quantized data validation failed");
|
||||
size_t new_size = 0;
|
||||
for (int64_t row = 0; row < nrows;) {
|
||||
const int64_t row_global = first_row + row;
|
||||
const int64_t this_nrow = std::min(nrows - row, nrows_per_expert - row_global % nrows_per_expert);
|
||||
void * this_data = (char *) new_data + row * row_size;
|
||||
size_t this_size = ggml_quantize_chunk(new_type, f32_data + row * n_per_row, this_data, 0, this_nrow, n_per_row, imatrix_for_row(row_global));
|
||||
if (!ggml_validate_row_data(new_type, this_data, this_size)) {
|
||||
throw std::runtime_error("quantized data validation failed");
|
||||
}
|
||||
new_size += this_size;
|
||||
row += this_nrow;
|
||||
}
|
||||
return new_size;
|
||||
}
|
||||
@@ -759,26 +775,29 @@ static size_t llama_tensor_quantize_impl(enum ggml_type new_type, const float *
|
||||
size_t new_size = 0;
|
||||
bool valid = true;
|
||||
auto compute = [&mutex, &counter, &new_size, &valid, new_type, f32_data, new_data, chunk_size,
|
||||
nrows, n_per_row, imatrix]() {
|
||||
first_row, nrows, nrows_per_expert, n_per_row, row_size, imatrix_for_row]() {
|
||||
const int64_t nrows_per_chunk = chunk_size / n_per_row;
|
||||
size_t local_size = 0;
|
||||
while (true) {
|
||||
std::unique_lock<std::mutex> lock(mutex);
|
||||
int64_t first_row = counter; counter += nrows_per_chunk;
|
||||
if (first_row >= nrows) {
|
||||
if (counter >= nrows) {
|
||||
if (local_size > 0) {
|
||||
new_size += local_size;
|
||||
}
|
||||
break;
|
||||
}
|
||||
const int64_t row = counter;
|
||||
const int64_t row_global = first_row + row;
|
||||
// stop at the expert boundary
|
||||
const int64_t this_nrow = std::min(std::min(nrows - row, nrows_per_chunk), nrows_per_expert - row_global % nrows_per_expert);
|
||||
counter += this_nrow;
|
||||
lock.unlock();
|
||||
const int64_t this_nrow = std::min(nrows - first_row, nrows_per_chunk);
|
||||
size_t this_size = ggml_quantize_chunk(new_type, f32_data, new_data, first_row * n_per_row, this_nrow, n_per_row, imatrix);
|
||||
|
||||
void * this_data = (char *) new_data + row * row_size;
|
||||
size_t this_size = ggml_quantize_chunk(new_type, f32_data + row * n_per_row, this_data, 0, this_nrow, n_per_row, imatrix_for_row(row_global));
|
||||
local_size += this_size;
|
||||
|
||||
// validate the quantized data
|
||||
const size_t row_size = ggml_row_size(new_type, n_per_row);
|
||||
void * this_data = (char *) new_data + first_row * row_size;
|
||||
if (!ggml_validate_row_data(new_type, this_data, this_size)) {
|
||||
std::unique_lock<std::mutex> lock(mutex);
|
||||
valid = false;
|
||||
@@ -1260,52 +1279,49 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std::
|
||||
fflush(stdout);
|
||||
|
||||
const int64_t n_per_row = tensor->ne[0];
|
||||
const int64_t nrows = tensor->ne[1];
|
||||
const int64_t nrows_per_expert = tensor->ne[1];
|
||||
const int64_t nrows_total = tensor->ne[1] * tensor->ne[2];
|
||||
|
||||
const size_t row_size_src = ggml_row_size(tensor->type, n_per_row);
|
||||
const size_t row_size_dst = ggml_row_size(new_type, n_per_row);
|
||||
|
||||
// process the rows in slabs, so that the buffers stay below max_buf_size
|
||||
const size_t bytes_per_row = row_size_src + row_size_dst + (tensor->type == GGML_TYPE_F32 ? 0 : n_per_row*sizeof(float));
|
||||
const int64_t nrows_slab = std::max<int64_t>(1, std::min<int64_t>(nrows, max_buf_size/bytes_per_row));
|
||||
const int64_t nrows_slab = std::max<int64_t>(1, std::min<int64_t>(nrows_total, max_buf_size/bytes_per_row));
|
||||
|
||||
static const int64_t min_chunk_size = 32 * 512;
|
||||
const int64_t chunk_size = (n_per_row >= min_chunk_size ? n_per_row : n_per_row * ((min_chunk_size + n_per_row - 1)/n_per_row));
|
||||
|
||||
// quantize each expert separately since they have different importance matrices
|
||||
// process rows across all experts in one pass to keep all threads busy
|
||||
new_size = 0;
|
||||
for (int64_t i03 = 0; i03 < tensor->ne[2]; ++i03) {
|
||||
const float * imatrix_03 = imatrix ? imatrix + i03 * n_per_row : nullptr;
|
||||
for (int64_t ir = 0; ir < nrows_total; ir += nrows_slab) {
|
||||
const int64_t nrows_cur = std::min(nrows_slab, nrows_total - ir);
|
||||
const int64_t nelements_cur = nrows_cur * n_per_row;
|
||||
|
||||
for (int64_t ir = 0; ir < nrows; ir += nrows_slab) {
|
||||
const int64_t nrows_cur = std::min(nrows_slab, nrows - ir);
|
||||
const int64_t nelements_cur = nrows_cur * n_per_row;
|
||||
const void * src = load_range(ir*row_size_src, nrows_cur*row_size_src);
|
||||
|
||||
const void * src = load_range((i03*nrows + ir)*row_size_src, nrows_cur*row_size_src);
|
||||
|
||||
const float * f32_data;
|
||||
if (tensor->type == GGML_TYPE_F32) {
|
||||
f32_data = (const float *) src;
|
||||
} else {
|
||||
if (f32_conv_buf.size() < (size_t) nelements_cur) {
|
||||
f32_conv_buf.resize(nelements_cur);
|
||||
}
|
||||
llama_tensor_dequantize_impl(tensor->type, src, (float *) f32_conv_buf.data(), workers, nelements_cur, nthread);
|
||||
f32_data = (const float *) f32_conv_buf.data();
|
||||
const float * f32_data;
|
||||
if (tensor->type == GGML_TYPE_F32) {
|
||||
f32_data = (const float *) src;
|
||||
} else {
|
||||
if (f32_conv_buf.size() < (size_t) nelements_cur) {
|
||||
f32_conv_buf.resize(nelements_cur);
|
||||
}
|
||||
|
||||
if (work.size() < nrows_cur*row_size_dst) {
|
||||
work.resize(nrows_cur*row_size_dst);
|
||||
}
|
||||
|
||||
const int64_t nchunk = (nelements_cur + chunk_size - 1)/chunk_size;
|
||||
const int64_t nthread_use = nthread > 1 ? std::max((int64_t)1, std::min((int64_t)nthread, nchunk)) : 1;
|
||||
|
||||
const size_t size_cur = llama_tensor_quantize_impl(new_type, f32_data, work.data(), chunk_size, nrows_cur, n_per_row, imatrix_03, workers, nthread_use);
|
||||
|
||||
fout.write((const char *) work.data(), size_cur);
|
||||
new_size += size_cur;
|
||||
llama_tensor_dequantize_impl(tensor->type, src, (float *) f32_conv_buf.data(), workers, nelements_cur, nthread);
|
||||
f32_data = (const float *) f32_conv_buf.data();
|
||||
}
|
||||
|
||||
if (work.size() < nrows_cur*row_size_dst) {
|
||||
work.resize(nrows_cur*row_size_dst);
|
||||
}
|
||||
|
||||
const int64_t nchunk = (nelements_cur + chunk_size - 1)/chunk_size;
|
||||
const int64_t nthread_use = nthread > 1 ? std::max((int64_t)1, std::min((int64_t)nthread, nchunk)) : 1;
|
||||
|
||||
const size_t size_cur = llama_tensor_quantize_impl(new_type, f32_data, work.data(), chunk_size, ir, nrows_cur, nrows_per_expert, n_per_row, imatrix, workers, nthread_use);
|
||||
|
||||
fout.write((const char *) work.data(), size_cur);
|
||||
new_size += size_cur;
|
||||
}
|
||||
LLAMA_LOG_INFO("size = %8.2f MiB -> %8.2f MiB\n", tensor_size/1024.0/1024.0, new_size/1024.0/1024.0);
|
||||
}
|
||||
|
||||
@@ -9,9 +9,6 @@ void llama_model_bailingmoe2::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);
|
||||
ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);
|
||||
ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func);
|
||||
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
|
||||
|
||||
GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
|
||||
|
||||
switch (hparams.n_layer()) {
|
||||
case 20: type = LLM_TYPE_16B_A1B; break;
|
||||
|
||||
@@ -22,7 +22,6 @@ void llama_model_bailingmoe3::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);
|
||||
ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);
|
||||
ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func);
|
||||
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
|
||||
ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, hparams.n_layer_all, false);
|
||||
ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer_all, false);
|
||||
|
||||
@@ -87,7 +86,7 @@ void llama_model_bailingmoe3::load_arch_tensors(llama_model_loader & ml) {
|
||||
create_tensor_qkv(layer, il, n_embd, d_inner, d_inner, d_inner, trunk_flags);
|
||||
layer.ssm_f_a = create_tensor(tn(LLM_TENSOR_SSM_F_A, "weight", il), { n_embd, d_inner }, trunk_flags);
|
||||
layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", il), { n_embd, n_head }, trunk_flags);
|
||||
layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, il), { 1, n_head }, trunk_flags);
|
||||
layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, il), { 1, n_head }, trunk_flags);
|
||||
layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", il), { d_inner }, trunk_flags);
|
||||
layer.ssm_g_a = create_tensor(tn(LLM_TENSOR_SSM_G_A, "weight", il), { n_embd, d_inner }, trunk_flags);
|
||||
layer.ssm_o_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", il), { head_dim }, trunk_flags);
|
||||
|
||||
@@ -20,9 +20,6 @@ void llama_model_cohere2moe::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);
|
||||
ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false);
|
||||
|
||||
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
|
||||
GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer");
|
||||
|
||||
if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) {
|
||||
hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;
|
||||
}
|
||||
|
||||
@@ -37,11 +37,6 @@ void llama_model_deepseek2::load_arch_hparams(llama_model_loader & ml) {
|
||||
hparams.rope_yarn_log_mul /= 0.1f;
|
||||
}
|
||||
|
||||
// NextN/MTP
|
||||
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
|
||||
GGML_ASSERT(hparams.n_layer_nextn == 0 ||
|
||||
hparams.n_layer() + hparams.n_layer_nextn == hparams.n_layer_all);
|
||||
|
||||
// (optional) temperature tuning - used by mistral-large
|
||||
ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_SCALE, hparams.f_attn_temp_scale, false);
|
||||
ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_LENGTH, hparams.n_attn_temp_floor_scale, false); // FIXME why not use temperature_length?
|
||||
|
||||
@@ -37,10 +37,6 @@ void llama_model_deepseek32::load_arch_hparams(llama_model_loader & ml) {
|
||||
hparams.rope_yarn_log_mul /= 0.1f;
|
||||
}
|
||||
|
||||
// NextN/MTP parameters
|
||||
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
|
||||
GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer");
|
||||
|
||||
switch (hparams.n_layer()) {
|
||||
case 61: type = LLM_TYPE_685B_A37B; break;
|
||||
default: type = LLM_TYPE_UNKNOWN;
|
||||
|
||||
@@ -17,15 +17,13 @@ static float dsv4_rope_attn_factor(float freq_scale, float ext_factor) {
|
||||
}
|
||||
|
||||
void llama_model_deepseek4::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
|
||||
if (hparams.n_layer_nextn > 0 && hparams.n_layer_nextn < hparams.n_layer_all) {
|
||||
if (hparams.n_layer_nextn > 0) {
|
||||
const uint32_t n_layer_main = hparams.n_layer_all - hparams.n_layer_nextn;
|
||||
const std::string mtp_probe = "blk." + std::to_string(n_layer_main) + ".nextn.eh_proj.weight";
|
||||
if (ml.get_weight(mtp_probe.c_str()) == nullptr) {
|
||||
hparams.n_layer_nextn = 0;
|
||||
}
|
||||
}
|
||||
GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < block_count");
|
||||
|
||||
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
|
||||
ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q);
|
||||
@@ -754,7 +752,8 @@ ggml_tensor * llama_model_deepseek4::graph::build_csa_lid_attention(
|
||||
ggml_tensor * kq_mask = ggml_concat(ctx0, raw_mask, csa_mask, 0);
|
||||
cb(kq_mask, "csa_lid_kq_mask", il);
|
||||
|
||||
ggml_tensor * out = build_attn_mha(q, k_all, k_all, nullptr, kq_mask, sinks, nullptr, kq_scale, il);
|
||||
const int64_t n_kv_max = std::min<int64_t>(raw_mask->ne[0], hparams.n_swa) + top_k->ne[0];
|
||||
ggml_tensor * out = build_attn_mha(q, k_all, k_all, nullptr, kq_mask, sinks, nullptr, n_kv_max, kq_scale, il);
|
||||
if (k_rot) {
|
||||
out = llama_mul_mat_hadamard(ctx0, out, k_rot);
|
||||
}
|
||||
@@ -809,7 +808,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_hca_attention(
|
||||
ggml_tensor * kq_mask = ggml_concat(ctx0, raw_mask, hca_mask, 0);
|
||||
cb(kq_mask, "hca_kq_mask", il);
|
||||
|
||||
ggml_tensor * out = build_attn_mha(q, k_all, k_all, nullptr, kq_mask, sinks, nullptr, kq_scale, il);
|
||||
ggml_tensor * out = build_attn_mha(q, k_all, k_all, nullptr, kq_mask, sinks, nullptr, 0, kq_scale, il);
|
||||
if (k_rot) {
|
||||
out = llama_mul_mat_hadamard(ctx0, out, k_rot);
|
||||
}
|
||||
@@ -845,7 +844,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_raw_attention(
|
||||
|
||||
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
|
||||
|
||||
ggml_tensor * out = build_attn_mha(q, k, k, nullptr, kq_mask, sinks, nullptr, kq_scale, il);
|
||||
ggml_tensor * out = build_attn_mha(q, k, k, nullptr, kq_mask, sinks, nullptr, 0, kq_scale, il);
|
||||
if (k_rot) {
|
||||
out = llama_mul_mat_hadamard(ctx0, out, k_rot);
|
||||
}
|
||||
|
||||
@@ -9,10 +9,6 @@ void llama_model_dots3note::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
|
||||
hparams.f_norm_eps = 1e-6; // eps for the indexer k_norm layer norm
|
||||
|
||||
// TODO: use MTP layer
|
||||
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
|
||||
GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all");
|
||||
|
||||
// MoE parameters
|
||||
ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);
|
||||
ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);
|
||||
|
||||
@@ -20,9 +20,6 @@ void llama_model_exaone_moe::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);
|
||||
ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false);
|
||||
|
||||
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
|
||||
GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
|
||||
|
||||
switch (hparams.n_layer()) {
|
||||
case 32: type = LLM_TYPE_30B_A3B; break;
|
||||
case 48: type = LLM_TYPE_235B_A22B; break;
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
#include "models.h"
|
||||
|
||||
void llama_model_exaone4::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
|
||||
GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer");
|
||||
|
||||
if (hparams.n_layer() == 64) { // 32B
|
||||
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
|
||||
hparams.n_swa = 4096;
|
||||
|
||||
@@ -4,16 +4,13 @@ void llama_model_gemma4_assistant::load_arch_hparams(llama_model_loader & ml) {
|
||||
hparams.n_embd_inp_impl = hparams.n_embd_out();
|
||||
|
||||
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
|
||||
ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer());
|
||||
ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer_all);
|
||||
|
||||
uint32_t n_kv_shared_layers = 0;
|
||||
ml.get_key(LLM_KV_ATTENTION_SHARED_KV_LAYERS, n_kv_shared_layers, false);
|
||||
|
||||
hparams.f_attention_scale = 1.0f;
|
||||
|
||||
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
|
||||
GGML_ASSERT(hparams.n_layer_nextn == hparams.n_layer_all && "n_layer_nextn must be == n_layer_impl");
|
||||
|
||||
ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);
|
||||
ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
|
||||
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
|
||||
|
||||
@@ -56,10 +56,6 @@ void llama_model_glm_dsa::load_arch_hparams(llama_model_loader & ml) {
|
||||
hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;
|
||||
}
|
||||
|
||||
// NextN/MTP parameters
|
||||
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
|
||||
GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all");
|
||||
|
||||
// BC for GLM 5, 5.1 (full indexers) without indexer_types metadata
|
||||
const bool is_pre_5_2 = hparams.n_ctx_train < 1048576;
|
||||
if (is_pre_5_2) {
|
||||
@@ -70,9 +66,7 @@ void llama_model_glm_dsa::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key_or_arr(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl, hparams.n_layer(), false);
|
||||
|
||||
switch (hparams.n_layer()) {
|
||||
case 78: // GGUF with NextN/MTP metadata: n_layer() excludes the nextn layer
|
||||
case 79:
|
||||
type = LLM_TYPE_744B_A40B; break;
|
||||
case 78: type = LLM_TYPE_744B_A40B; break;
|
||||
default: type = LLM_TYPE_UNKNOWN;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,10 +17,6 @@ void llama_model_glm4_moe::load_arch_hparams(llama_model_loader & ml) {
|
||||
hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;
|
||||
}
|
||||
|
||||
// NextN/MTP parameters
|
||||
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
|
||||
GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
|
||||
|
||||
switch (hparams.n_layer()) {
|
||||
case 46: type = LLM_TYPE_106B_A12B; break; // GLM-4.5-Air
|
||||
case 48: type = LLM_TYPE_102B_A12B; break; // Solar Open
|
||||
|
||||
@@ -4,10 +4,6 @@ void llama_model_glm4::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
|
||||
ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, false);
|
||||
|
||||
// NextN/MTP parameters (GLM-OCR)
|
||||
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
|
||||
GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
|
||||
|
||||
switch (hparams.n_layer()) {
|
||||
case 17: type = LLM_TYPE_1B; break; // GLM-OCR
|
||||
case 40: type = LLM_TYPE_9B; break;
|
||||
|
||||
@@ -13,10 +13,6 @@ void llama_model_hy_v3::load_arch_hparams(llama_model_loader & ml) {
|
||||
hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;
|
||||
}
|
||||
|
||||
// NextN/MTP (HY V3): extra decoder block(s) appended beyond the main stack
|
||||
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
|
||||
GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all");
|
||||
|
||||
switch (hparams.n_layer()) {
|
||||
case 48: type = LLM_TYPE_30B_A3B; break;
|
||||
default: type = LLM_TYPE_UNKNOWN;
|
||||
|
||||
@@ -94,7 +94,7 @@ void llama_model_kimi_k3::load_arch_tensors(llama_model_loader &) {
|
||||
layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", i), {n_embd, n_head}, 0);
|
||||
|
||||
// K3's A_log is a plain 1-D [n_head] tensor (kimi-linear's is padded)
|
||||
layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {n_head}, 0);
|
||||
layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, i), {n_head}, 0);
|
||||
layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {d_inner}, 0);
|
||||
|
||||
// K3 uses a single full-rank gate instead of kimi-linear's g_a/g_b pair
|
||||
|
||||
@@ -84,9 +84,9 @@ void llama_model_kimi_linear::load_arch_tensors(llama_model_loader &) {
|
||||
layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", i), {n_embd, n_head}, 0);
|
||||
|
||||
// A_log - Shape in GGUF: [1, num_heads, 1, 1] (4D) or [1, num_heads] (2D after quantization) Note: -exp(A_log) is applied in convert_hf_to_gguf.py
|
||||
layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {1, n_head, 1, 1}, TENSOR_NOT_REQUIRED);
|
||||
layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, i), {1, n_head, 1, 1}, TENSOR_NOT_REQUIRED);
|
||||
if (!layer.ssm_a) {
|
||||
layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {1, n_head}, 0);
|
||||
layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, i), {1, n_head}, 0);
|
||||
}
|
||||
|
||||
// dt_bias - shape [n_embd_head_k_kda * n_head] = [4096]
|
||||
|
||||
@@ -16,9 +16,6 @@ void llama_model_mimo2::load_arch_hparams(llama_model_loader & ml) {
|
||||
hparams.f_attn_value_scale = value_scale;
|
||||
}
|
||||
|
||||
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
|
||||
GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
|
||||
|
||||
switch (hparams.n_layer()) {
|
||||
case 48: type = LLM_TYPE_310B_A15B; break;
|
||||
default: type = LLM_TYPE_UNKNOWN;
|
||||
|
||||
@@ -7,10 +7,6 @@ void llama_model_nemotron_h::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
|
||||
ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);
|
||||
|
||||
// NextN/MTP: optional draft head appended as extra trailing block(s)
|
||||
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
|
||||
GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all");
|
||||
|
||||
// A layer is recurrent IFF the n_head_kv value is set to 0 and
|
||||
// the n_ff value is set to 0. Appended MTP blocks are dense (non-recurrent)
|
||||
for (uint32_t i = 0; i < hparams.n_layer_all; ++i) {
|
||||
|
||||
@@ -12,10 +12,6 @@ void llama_model_qwen35::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
|
||||
ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);
|
||||
|
||||
// NextN/MTP (Qwen3.5/3.6): extra decoder block appended beyond the main stack
|
||||
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
|
||||
GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
|
||||
|
||||
// Mark recurrent layers (linear attention layers). MTP layers are dense
|
||||
// attention-only and must be flagged non-recurrent.
|
||||
if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) {
|
||||
|
||||
@@ -15,10 +15,6 @@ void llama_model_qwen35moe::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
|
||||
ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);
|
||||
|
||||
// NextN/MTP (Qwen3.5/3.6): extra decoder block appended beyond the main stack
|
||||
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
|
||||
GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
|
||||
|
||||
// Mark recurrent layers (linear attention layers). MTP layers are dense
|
||||
// attention-only and must be flagged non-recurrent.
|
||||
if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) {
|
||||
|
||||
@@ -13,10 +13,6 @@ void llama_model_qwen3next::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
|
||||
ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);
|
||||
|
||||
// NextN/MTP: extra decoder block appended beyond the main stack
|
||||
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
|
||||
GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all");
|
||||
|
||||
// Mark recurrent layers (linear attention layers).
|
||||
if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) {
|
||||
uint32_t full_attn_interval = 4;
|
||||
|
||||
+100
-34
@@ -6,6 +6,23 @@
|
||||
#include <algorithm>
|
||||
#include <cinttypes>
|
||||
|
||||
// bad metadata must be catchable: GGML_ASSERT aborts the whole process
|
||||
static void qwen4exp_require_nonzero(const llama_model_loader & ml, llm_kv kid, uint32_t value) {
|
||||
if (value == 0) {
|
||||
throw std::runtime_error(format("%s must be greater than zero, got %u", ml.llm_kv(kid).c_str(), value));
|
||||
}
|
||||
}
|
||||
|
||||
// get_arr() copies a short array as-is, leaving a zero tail the n-gram hash silently drops
|
||||
static void qwen4exp_require_arr_len(llama_model_loader & ml, llm_kv kid, uint32_t n_min) {
|
||||
uint32_t n_arr = 0;
|
||||
ml.get_arr_n(kid, n_arr, true);
|
||||
if (n_arr < n_min) {
|
||||
throw std::runtime_error(format("%s has %u entries, but at least %u are required",
|
||||
ml.llm_kv(kid).c_str(), n_arr, n_min));
|
||||
}
|
||||
}
|
||||
|
||||
void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);
|
||||
ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);
|
||||
@@ -18,21 +35,30 @@ void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state);
|
||||
ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
|
||||
ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);
|
||||
GGML_ASSERT(hparams.ssm_d_conv > 0 && hparams.ssm_d_inner > 0 && hparams.ssm_d_state > 0 &&
|
||||
hparams.ssm_dt_rank > 0 && hparams.ssm_n_group > 0);
|
||||
qwen4exp_require_nonzero(ml, LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);
|
||||
qwen4exp_require_nonzero(ml, LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner);
|
||||
qwen4exp_require_nonzero(ml, LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state);
|
||||
qwen4exp_require_nonzero(ml, LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
|
||||
qwen4exp_require_nonzero(ml, LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);
|
||||
|
||||
// HC; low_rank is qwen4exp-specific, DeepSeek-V4 leaves it absent (full rank)
|
||||
ml.get_key(LLM_KV_HYPER_CONNECTION_COUNT, hparams.dsv4_hc_mult);
|
||||
ml.get_key(LLM_KV_HYPER_CONNECTION_LOW_RANK, hparams.hc_low_rank);
|
||||
GGML_ASSERT(hparams.dsv4_hc_mult > 0 && hparams.hc_low_rank > 0);
|
||||
// a count of 1 has nothing to mix: transformers configuration_qwen4_exp.py:196, vLLM
|
||||
// config.py:49 and SGLang configs/qwen4_exp.py:38 all raise on hc_count <= 1
|
||||
if (hparams.dsv4_hc_mult <= 1) {
|
||||
throw std::runtime_error(format("%s must be greater than one, got %u",
|
||||
ml.llm_kv(LLM_KV_HYPER_CONNECTION_COUNT).c_str(), hparams.dsv4_hc_mult));
|
||||
}
|
||||
qwen4exp_require_nonzero(ml, LLM_KV_HYPER_CONNECTION_LOW_RANK, hparams.hc_low_rank);
|
||||
hparams.n_embd_out_impl = hparams.dsv4_hc_mult * hparams.n_embd;
|
||||
|
||||
ml.get_key(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head);
|
||||
ml.get_key(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size);
|
||||
ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k);
|
||||
GGML_ASSERT(hparams.indexer_n_head > 0
|
||||
&& hparams.indexer_head_size > 0
|
||||
&& hparams.indexer_top_k > 0);
|
||||
qwen4exp_require_nonzero(ml, LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head);
|
||||
qwen4exp_require_nonzero(ml, LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size);
|
||||
qwen4exp_require_nonzero(ml, LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k);
|
||||
ml.get_key_or_arr(LLM_KV_ATTENTION_COMPRESS_RATIOS, hparams.dsv4_compress_ratios, hparams.n_layer_all, false);
|
||||
|
||||
// PLE n-gram hash embeddings; if the key group is absent every field stays zero
|
||||
@@ -44,7 +70,11 @@ void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) {
|
||||
if (n_ple > 0) {
|
||||
std::vector<uint32_t> ple_layers;
|
||||
ml.get_arr(LLM_KV_PLE_LAYERS, ple_layers);
|
||||
GGML_ASSERT(n_ple == 1 && "qwen4exp supports only one PLE layer");
|
||||
if (n_ple != 1) {
|
||||
// hparams holds one set of hash constants, so several PLE modules cannot be represented
|
||||
throw std::runtime_error(format("%s lists %u layers, but only one PLE layer is supported",
|
||||
ml.llm_kv(LLM_KV_PLE_LAYERS).c_str(), n_ple));
|
||||
}
|
||||
for (uint32_t il : ple_layers) {
|
||||
if (il >= hparams.n_layer_all) {
|
||||
throw std::runtime_error(format("PLE layer %u is out of range", il));
|
||||
@@ -59,7 +89,8 @@ void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) {
|
||||
// optional: files written before this key fall back to the EOS token
|
||||
ml.get_key(LLM_KV_PLE_IMAGE_TOKEN_ID, hparams.ple_image_token_id, false);
|
||||
ml.get_key(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, hparams.n_embd_per_layer);
|
||||
GGML_ASSERT(hparams.ple_conv_kernel > 0 && hparams.n_embd_per_layer > 0);
|
||||
qwen4exp_require_nonzero(ml, LLM_KV_PLE_CONV_KERNEL, hparams.ple_conv_kernel);
|
||||
qwen4exp_require_nonzero(ml, LLM_KV_EMBEDDING_LENGTH_PER_LAYER, hparams.n_embd_per_layer);
|
||||
|
||||
hparams.ple_n_heads = (hparams.ple_ngram_size - 1) * hparams.ple_heads_per_ngram;
|
||||
hparams.ple_head_dim = hparams.n_embd_per_layer;
|
||||
@@ -70,6 +101,10 @@ void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) {
|
||||
throw std::runtime_error(format("PLE head count %u is out of range", hparams.ple_n_heads));
|
||||
}
|
||||
|
||||
qwen4exp_require_arr_len(ml, LLM_KV_PLE_LAYER_MULTIPLIERS, hparams.ple_ngram_size);
|
||||
qwen4exp_require_arr_len(ml, LLM_KV_PLE_HEAD_OFFSETS, hparams.ple_n_heads);
|
||||
qwen4exp_require_arr_len(ml, LLM_KV_PLE_HEAD_VOCAB_SIZES, hparams.ple_n_heads);
|
||||
|
||||
ml.get_arr(LLM_KV_PLE_LAYER_MULTIPLIERS, hparams.ple_layer_multipliers);
|
||||
|
||||
// the file stores the head ranges as uint64, so read at that width and narrow to the int32 the gather uses
|
||||
@@ -93,12 +128,19 @@ void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) {
|
||||
if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) {
|
||||
uint32_t full_attn_interval = 4;
|
||||
ml.get_key(LLM_KV_FULL_ATTENTION_INTERVAL, full_attn_interval, false);
|
||||
GGML_ASSERT(full_attn_interval > 0);
|
||||
qwen4exp_require_nonzero(ml, LLM_KV_FULL_ATTENTION_INTERVAL, full_attn_interval);
|
||||
for (uint32_t i = 0; i < hparams.n_layer_all; ++i) {
|
||||
hparams.is_recr_impl[i] = (i < hparams.n_layer()) && ((i + 1) % full_attn_interval != 0);
|
||||
}
|
||||
}
|
||||
|
||||
// the PLE conv history is a row of the recurrent cache, which linear layers alone have
|
||||
for (uint32_t i = 0; i < hparams.n_layer_all; ++i) {
|
||||
if (hparams.is_ple(i) && !hparams.is_recr(i)) {
|
||||
throw std::runtime_error(format("PLE layer %u is not a linear attention layer", i));
|
||||
}
|
||||
}
|
||||
|
||||
switch (hparams.n_layer()) {
|
||||
case 48: type = LLM_TYPE_A3B; break;
|
||||
default: type = LLM_TYPE_UNKNOWN;
|
||||
@@ -124,18 +166,24 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {
|
||||
output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, TENSOR_DUPLICATED);
|
||||
}
|
||||
|
||||
// flat [ple_head_dim, n_rows] gather target; n_rows is padded, so read it back
|
||||
// flat [ple_head_dim, n_rows] gather target
|
||||
if (hparams.ple_n_heads > 0) {
|
||||
const std::string ple_name = tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight").str();
|
||||
const auto & ple_w = ml.require_weight(ple_name.c_str());
|
||||
const int64_t ple_rows = ple_w.tensor->ne[1];
|
||||
|
||||
// sanity check
|
||||
// the head ranges are what the gather indexes, so they set the minimum row count
|
||||
int64_t ple_rows = 0;
|
||||
for (uint32_t h = 0; h < hparams.ple_n_heads; ++h) {
|
||||
if ((int64_t) hparams.ple_head_offsets[h] + hparams.ple_head_vocab_sizes[h] > ple_rows) {
|
||||
throw std::runtime_error(format("PLE head %u range exceeds the %" PRId64 " table rows", h, ple_rows));
|
||||
}
|
||||
ple_rows = std::max(ple_rows, (int64_t) hparams.ple_head_offsets[h] + hparams.ple_head_vocab_sizes[h]);
|
||||
}
|
||||
|
||||
// the converter pads the table; a model synthesised from metadata has no tensor to ask
|
||||
const std::string ple_name = tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight").str();
|
||||
if (const auto * ple_w = ml.get_weight(ple_name.c_str())) {
|
||||
if (ple_w->tensor->ne[1] < ple_rows) {
|
||||
throw std::runtime_error(format("%s has %" PRId64 " rows, too few for the PLE head ranges (%" PRId64 ")",
|
||||
ple_name.c_str(), ple_w->tensor->ne[1], ple_rows));
|
||||
}
|
||||
ple_rows = ple_w->tensor->ne[1];
|
||||
}
|
||||
|
||||
per_layer_tok_embd = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight"),
|
||||
{ hparams.ple_head_dim, ple_rows }, TENSOR_READ_LAZY);
|
||||
}
|
||||
@@ -556,9 +604,12 @@ ggml_tensor * llama_model_qwen4exp::graph::build_qsa_top_k(
|
||||
pooled = ggml_scale(ctx0, pooled, 1.0f/(float) r);
|
||||
cb(pooled, "indexer_k_pooled", il);
|
||||
|
||||
// count blocks along ne1: rms_norm launches gridDim.y = ne2, capped at 65535, and 262144/4 = 65536
|
||||
pooled = ggml_reshape_3d(ctx0, pooled, idx_dim, n_blocks*n_stream, 1);
|
||||
pooled = build_norm(pooled, model.layers[il].index_k_norm, nullptr, LLM_NORM_RMS, il);
|
||||
|
||||
// rope wants [n_dims, n_head, n_tokens]: lay every stream's blocks flat, split after.
|
||||
pooled = ggml_reshape_3d(ctx0, pooled, idx_dim, 1, n_blocks*n_stream);
|
||||
pooled = build_norm(pooled, model.layers[il].index_k_norm, nullptr, LLM_NORM_RMS, il);
|
||||
pooled = ggml_rope_multi(ctx0, pooled, inp->blk_pos, nullptr,
|
||||
n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
|
||||
ext_factor, attn_factor, beta_fast, beta_slow);
|
||||
@@ -576,12 +627,19 @@ ggml_tensor * llama_model_qwen4exp::graph::build_qsa_top_k(
|
||||
// rectify each head dot product before the sum, as in the DeepSeek lightning indexer
|
||||
// mul_mat matches ne[2], so the queries of stream s only meet the blocks of stream s
|
||||
ggml_tensor * score = ggml_mul_mat(ctx0, pooled,
|
||||
ggml_reshape_3d(ctx0, ggml_cont(ctx0, q), idx_dim, n_idx_h*n_tps, n_stream));
|
||||
ggml_reshape_3d(ctx0, q, idx_dim, n_idx_h*n_tps, n_stream));
|
||||
score = ggml_reshape_4d(ctx0, score, n_blocks, n_idx_h, n_tps, n_stream);
|
||||
score = ggml_relu(ctx0, score);
|
||||
score = ggml_cont(ctx0, ggml_permute(ctx0, score, 1, 0, 2, 3));
|
||||
score = ggml_sum_rows(ctx0, score);
|
||||
score = ggml_reshape_3d(ctx0, score, n_blocks, n_tps, n_stream);
|
||||
|
||||
// the heads sit side by side on ne[1] and there are only a few of them
|
||||
ggml_tensor * summed = nullptr;
|
||||
for (int64_t h = 0; h < n_idx_h; ++h) {
|
||||
ggml_tensor * slice = ggml_view_3d(ctx0, score, n_blocks, n_tps, n_stream,
|
||||
score->nb[2], score->nb[3], h*score->nb[1]);
|
||||
summed = summed ? ggml_add(ctx0, summed, slice) : ggml_cont(ctx0, slice);
|
||||
}
|
||||
|
||||
score = summed;
|
||||
cb(score, "indexer_score", il);
|
||||
|
||||
// one value per block, so it is cheaper to bias here than after the cells are expanded
|
||||
@@ -686,7 +744,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_attn_qsa(
|
||||
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
|
||||
ggml_tensor * v = mctx_cur->get_v(ctx0, il);
|
||||
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, nullptr, kq_mask_top_k, nullptr, nullptr, kq_scale, il);
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, nullptr, kq_mask_top_k, nullptr, nullptr, 0, kq_scale, il);
|
||||
cb(cur, "kqv_out", il);
|
||||
|
||||
// the rotation is its own inverse, so undo it on the value side of the output
|
||||
@@ -1080,20 +1138,28 @@ ggml_tensor * llama_model_qwen4exp::graph::build_conv_state_at(
|
||||
|
||||
ggml_tensor * conv_input = ggml_concat(ctx0, state, ggml_transpose(ctx0, x), 0);
|
||||
|
||||
// keep the last state_cols columns for the next ubatch
|
||||
// [TAG_RECURRENT_ROLLBACK_SPLITS] keep the last state_cols columns once per rollback slot,
|
||||
// slot s ending s tokens earlier so a rollback of s tokens reads a history that never saw them
|
||||
const size_t row_size = ggml_row_size(conv_states_all->type, row_total);
|
||||
const uint32_t mem_size = mctx_cur->get_size();
|
||||
|
||||
ggml_tensor * tail = ggml_view_3d(ctx0, conv_input,
|
||||
state_cols, channels, n_seqs,
|
||||
conv_input->nb[1], conv_input->nb[2],
|
||||
ggml_row_size(conv_input->type, conv_input->ne[0] - state_cols));
|
||||
const int64_t n_slots = (int64_t) cparams.n_rs_seq + 1;
|
||||
|
||||
ggml_tensor * dst = ggml_view_2d(ctx0, conv_states_all,
|
||||
state_cols * channels, n_seqs,
|
||||
conv_states_all->nb[1],
|
||||
kv_head * row_size);
|
||||
for (int64_t slot = 0; slot < n_slots; ++slot) {
|
||||
const int64_t s_idx = std::max<int64_t>(0, conv_input->ne[0] - state_cols - slot);
|
||||
|
||||
ggml_build_forward_expand(gf, ggml_cpy(ctx0, ggml_cont(ctx0, tail), dst));
|
||||
ggml_tensor * tail = ggml_view_3d(ctx0, conv_input,
|
||||
state_cols, channels, n_seqs,
|
||||
conv_input->nb[1], conv_input->nb[2],
|
||||
ggml_row_size(conv_input->type, s_idx));
|
||||
|
||||
ggml_tensor * dst = ggml_view_2d(ctx0, conv_states_all,
|
||||
state_cols * channels, n_seqs,
|
||||
conv_states_all->nb[1],
|
||||
(slot * mem_size + kv_head) * row_size);
|
||||
|
||||
ggml_build_forward_expand(gf, ggml_cpy(ctx0, ggml_cont(ctx0, tail), dst));
|
||||
}
|
||||
|
||||
return conv_input;
|
||||
}
|
||||
|
||||
@@ -23,14 +23,10 @@ void llama_model_step35::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
|
||||
ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);
|
||||
|
||||
ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer());
|
||||
ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer_all);
|
||||
|
||||
ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, hparams.n_layer(), false);
|
||||
ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer(), false);
|
||||
|
||||
// NextN/MTP (Step3p5): extra decoder block appended beyond the main stack.
|
||||
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
|
||||
GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
|
||||
ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, hparams.n_layer_all, false);
|
||||
ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer_all, false);
|
||||
|
||||
switch (hparams.n_layer()) {
|
||||
case 45: type = LLM_TYPE_196B_A11B; break;
|
||||
|
||||
+26
-2
@@ -153,6 +153,9 @@
|
||||
#define TN_MM_MERGER_FC1 "mm.merger.fc1.%s" // minimax-m3 patch-merge MLP
|
||||
#define TN_MM_MERGER_FC2 "mm.merger.fc2.%s"
|
||||
#define TN_TOK_IMG_BREAK "v.token_embd.img_break" // pixtral
|
||||
#define TN_TOK_IMG_START "v.token_embd.img_start" // deepseek4v
|
||||
#define TN_TOK_IMG_END "v.token_embd.img_end" // deepseek4v
|
||||
#define TN_TOK_IMG_PAD "v.token_embd.img_pad" // deepseek4v
|
||||
#define TN_TOK_GLM_BOI "adapter.boi" // glm-edge (these embeddings are not in text model)
|
||||
#define TN_TOK_GLM_EOI "adapter.eoi" // glm-edge (these embeddings are not in text model)
|
||||
#define TN_DEEPSTACK_NORM "v.deepstack.%d.norm.%s" // qwen3vl deepstack
|
||||
@@ -296,8 +299,8 @@
|
||||
|
||||
// hunyuanvl (shared GGUF tensor names)
|
||||
#define TN_MM_PRE_NORM "mm.pre_norm.%s"
|
||||
#define TN_TOK_IMG_BEGIN "mm.image_begin"
|
||||
#define TN_TOK_IMG_END "mm.image_end"
|
||||
#define TN_MM_IMG_BEGIN "mm.image_begin" // note: legacy name, new models should use v.token_embd.*
|
||||
#define TN_MM_IMG_END "mm.image_end" // note: legacy name, new models should use v.token_embd.*
|
||||
|
||||
// deepseek-ocr
|
||||
#define TN_SAM_POS_EMBD "v.sam.pos_embd.%s"
|
||||
@@ -480,6 +483,7 @@ enum projector_type {
|
||||
PROJECTOR_TYPE_DOTS3NOTE_A,
|
||||
PROJECTOR_TYPE_DEEPSEEKOCR,
|
||||
PROJECTOR_TYPE_DEEPSEEKOCR2,
|
||||
PROJECTOR_TYPE_DEEPSEEK4V,
|
||||
PROJECTOR_TYPE_LFM2A,
|
||||
PROJECTOR_TYPE_GLM4V,
|
||||
PROJECTOR_TYPE_YOUTUVL,
|
||||
@@ -544,6 +548,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
|
||||
{ PROJECTOR_TYPE_DOTS3NOTE_A, "dots3note_a"},
|
||||
{ PROJECTOR_TYPE_DEEPSEEKOCR, "deepseekocr"},
|
||||
{ PROJECTOR_TYPE_DEEPSEEKOCR2, "deepseekocr2"},
|
||||
{ PROJECTOR_TYPE_DEEPSEEK4V, "deepseek4v"},
|
||||
{ PROJECTOR_TYPE_LFM2A, "lfm2a"},
|
||||
{ PROJECTOR_TYPE_GLM4V, "glm4v"},
|
||||
{ PROJECTOR_TYPE_YOUTUVL, "youtuvl"},
|
||||
@@ -655,6 +660,9 @@ struct clip_image_f32 {
|
||||
// appends a learned newline (or EOI) token after the image
|
||||
// no model uses it now (Granite4 Vision moved to anyres), kept for future models
|
||||
bool add_newline = false;
|
||||
// deepseek4v: number of leading IMAGE_PAD embeddings, aligns IMAGE_START to the LLM compressor ratio
|
||||
// depends on the chunk position, set at tokenize time (see mtmd_tokenizer::add_media)
|
||||
int32_t lead_pad = 0;
|
||||
|
||||
// llava-next "anyres" tiling, used by Granite4 Vision
|
||||
// the whole grid is encoded and assembled in a single graph
|
||||
@@ -771,6 +779,22 @@ static inline void clip_anyres_unpad(int cur_w, int cur_h, int orig_w, int orig_
|
||||
}
|
||||
}
|
||||
|
||||
// deepseek4v: layout of the LLM token block built from the aligner grid
|
||||
struct dsv4_block_layout {
|
||||
int rows; // grid rows, padded to an even count
|
||||
int row_len; // grid width + 1 newline
|
||||
int pad_last; // trailing pads
|
||||
int n_out; // total block size, including lead pads and the start/end sentinels
|
||||
};
|
||||
static inline dsv4_block_layout dsv4_get_block_layout(int n_llm_w, int n_llm_h, int lead_pad) {
|
||||
dsv4_block_layout bl;
|
||||
bl.rows = n_llm_h + (n_llm_h % 2);
|
||||
bl.row_len = n_llm_w + 1;
|
||||
bl.pad_last = (bl.rows / 2 * bl.row_len) % 2 * 2;
|
||||
bl.n_out = lead_pad + 1 + bl.rows * bl.row_len + bl.pad_last + 1;
|
||||
return bl;
|
||||
}
|
||||
|
||||
//
|
||||
// logging
|
||||
//
|
||||
|
||||
@@ -100,6 +100,10 @@ struct clip_hparams {
|
||||
std::unordered_set<int32_t> wa_layer_indexes; // explicit layer indexes that use full attention (for irregular patterns like YoutuVL)
|
||||
std::vector<int32_t> wa_pattern_mode; // mimovl: per-layer window-attention mode
|
||||
|
||||
// deepseek4v: resize solver caps the LLM token count of the aligner grid
|
||||
int32_t dsv4_max_n_token = 0;
|
||||
int32_t dsv4_max_wh_ratio = 0;
|
||||
|
||||
// deepseek-ocr (sam)
|
||||
int32_t sam_n_layer = 0;
|
||||
int32_t sam_n_head = 0;
|
||||
@@ -724,6 +728,11 @@ struct clip_model {
|
||||
|
||||
// pixtral, glm4v
|
||||
ggml_tensor * token_embd_img_break = nullptr;
|
||||
|
||||
// deepseek4v sentinel embeddings (image_newline is reused for IMAGE_NEW_LINE)
|
||||
ggml_tensor * token_embd_img_start = nullptr;
|
||||
ggml_tensor * token_embd_img_end = nullptr;
|
||||
ggml_tensor * token_embd_img_pad = nullptr;
|
||||
ggml_tensor * mm_patch_merger_w = nullptr;
|
||||
ggml_tensor * mm_patch_merger_b = nullptr;
|
||||
|
||||
|
||||
+120
-5
@@ -81,6 +81,7 @@
|
||||
#include "models/whisper-enc.cpp"
|
||||
#include "models/deepseekocr.cpp"
|
||||
#include "models/deepseekocr2.cpp"
|
||||
#include "models/deepseek4v.cpp"
|
||||
#include "models/mobilenetv5.cpp"
|
||||
#include "models/youtuvl.cpp"
|
||||
#include "models/yasa2.cpp"
|
||||
@@ -1095,6 +1096,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
|
||||
{
|
||||
builder = std::make_unique<clip_graph_kimik25>(ctx, img);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DEEPSEEK4V:
|
||||
{
|
||||
builder = std::make_unique<clip_graph_deepseek4v>(ctx, img);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_COGVLM:
|
||||
{
|
||||
builder = std::make_unique<clip_graph_cogvlm>(ctx, img);
|
||||
@@ -1666,6 +1671,31 @@ struct clip_model_loader {
|
||||
hparams.set_limit_image_tokens(2, 4096);
|
||||
}
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DEEPSEEK4V:
|
||||
{
|
||||
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
|
||||
hparams.image_pad_color = {127, 127, 127};
|
||||
hparams.rope_theta = 10000.0f;
|
||||
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge);
|
||||
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
|
||||
hparams.dsv4_max_n_token = 384;
|
||||
hparams.dsv4_max_wh_ratio = 8;
|
||||
const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge;
|
||||
// handle min/max token counts from CLI
|
||||
if (hparams.custom_image_min_tokens > 0) {
|
||||
hparams.image_min_pixels = hparams.custom_image_min_tokens * patch_area;
|
||||
}
|
||||
if (hparams.custom_image_max_tokens > 0) {
|
||||
// the cap is on the whole token block, keep some room for the resize solver
|
||||
hparams.dsv4_max_n_token = std::max(hparams.custom_image_max_tokens, 16);
|
||||
}
|
||||
hparams.image_max_pixels = hparams.dsv4_max_n_token * patch_area;
|
||||
// a small custom max token count also lowers the min-pixel upscale threshold
|
||||
hparams.image_min_pixels = std::min(hparams.image_min_pixels, hparams.image_max_pixels);
|
||||
// avoid OOM on warmup
|
||||
const int warmup_side = (int) std::sqrt((double) std::min(256, hparams.dsv4_max_n_token));
|
||||
hparams.set_warmup_n_tokens(warmup_side * warmup_side);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_GEMMA3:
|
||||
{
|
||||
// default value (used by all model sizes in gemma 3 family)
|
||||
@@ -2805,6 +2835,18 @@ struct clip_model_loader {
|
||||
model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
|
||||
model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DEEPSEEK4V:
|
||||
{
|
||||
model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
|
||||
model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"));
|
||||
model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
|
||||
model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
|
||||
// sentinel token embeddings written into the output block
|
||||
model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
|
||||
model.token_embd_img_start = get_tensor(TN_TOK_IMG_START);
|
||||
model.token_embd_img_end = get_tensor(TN_TOK_IMG_END);
|
||||
model.token_embd_img_pad = get_tensor(TN_TOK_IMG_PAD);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_PIXTRAL:
|
||||
{
|
||||
model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
|
||||
@@ -3079,9 +3121,9 @@ struct clip_model_loader {
|
||||
} break;
|
||||
case PROJECTOR_TYPE_QWEN3TTS_GEN:
|
||||
{
|
||||
// code_predictor
|
||||
model.gen_code_proj_in_w = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "weight"));
|
||||
model.gen_code_proj_in_b = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "bias"));
|
||||
// code_predictor, proj_in is absent when the talker and the predictor share the hidden size
|
||||
model.gen_code_proj_in_w = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "weight"), false);
|
||||
model.gen_code_proj_in_b = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "bias"), false);
|
||||
model.gen_code_embd_w = get_tensor(string_format(TN_A_GEN_CODE_EMBD, "weight"));
|
||||
model.gen_code_head_w = get_tensor(string_format(TN_A_GEN_CODE_HEAD, "weight"));
|
||||
model.gen_code_out_embd_w = get_tensor(string_format(TN_A_GEN_CODE_OUT_EMBD, "weight"));
|
||||
@@ -3252,8 +3294,8 @@ struct clip_model_loader {
|
||||
model.mm_model_proj_b = get_tensor(string_format(TN_MM_PROJECTOR, "bias"));
|
||||
model.mm_pre_norm_w = get_tensor(string_format(TN_MM_PRE_NORM, "weight"));
|
||||
model.mm_post_norm_w = get_tensor(string_format(TN_MM_POST_NORM, "weight"));
|
||||
model.mm_img_begin = get_tensor(TN_TOK_IMG_BEGIN);
|
||||
model.mm_img_end = get_tensor(TN_TOK_IMG_END);
|
||||
model.mm_img_begin = get_tensor(TN_MM_IMG_BEGIN);
|
||||
model.mm_img_end = get_tensor(TN_MM_IMG_END);
|
||||
model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
|
||||
model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR, false);
|
||||
} break;
|
||||
@@ -4241,6 +4283,13 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
|
||||
int y_patch = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size;
|
||||
n_patches = x_patch * y_patch;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DEEPSEEK4V:
|
||||
{
|
||||
const int out_patch_size = params.patch_size * params.n_merge;
|
||||
const int n_llm_w = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size;
|
||||
const int n_llm_h = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size;
|
||||
n_patches = dsv4_get_block_layout(n_llm_w, n_llm_h, img->lead_pad).n_out;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_PADDLEOCR:
|
||||
case PROJECTOR_TYPE_DOTS_OCR:
|
||||
case PROJECTOR_TYPE_DOTS3NOTE_V:
|
||||
@@ -5112,6 +5161,58 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
|
||||
}
|
||||
set_input_i32("pos_w", pos_data);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DEEPSEEK4V:
|
||||
{
|
||||
// set the 2D positions (mrope layout, only the first 2 channels are used)
|
||||
int n_patches_per_row = image_size_width / patch_size;
|
||||
std::vector<int32_t> positions(n_pos * 4, 0);
|
||||
for (int i = 0; i < n_pos; i++) {
|
||||
positions[i] = i / n_patches_per_row; // row
|
||||
positions[n_pos + i] = i % n_patches_per_row; // col
|
||||
}
|
||||
set_input_i32("positions", positions);
|
||||
|
||||
// token block layout index (see clip_graph_deepseek4v::build)
|
||||
// rows [0, n_grid) are the aligner output, the sentinels follow
|
||||
const int n_merge = hparams.n_merge;
|
||||
const int n_llm_w = CLIP_ALIGN(pos_w, n_merge) / n_merge;
|
||||
const int n_llm_h = CLIP_ALIGN(pos_h, n_merge) / n_merge;
|
||||
const int n_grid = n_llm_w * n_llm_h;
|
||||
const int idx_start = n_grid;
|
||||
const int idx_end = n_grid + 1;
|
||||
const int idx_newline = n_grid + 2;
|
||||
const int idx_pad = n_grid + 3;
|
||||
|
||||
const int lead_pad = imgs.entries[0].lead_pad;
|
||||
const auto bl = dsv4_get_block_layout(n_llm_w, n_llm_h, lead_pad);
|
||||
|
||||
std::vector<int32_t> idx;
|
||||
idx.reserve(bl.n_out);
|
||||
for (int i = 0; i < lead_pad; i++) {
|
||||
idx.push_back(idx_pad);
|
||||
}
|
||||
idx.push_back(idx_start);
|
||||
// pairs of adjacent rows are interleaved column-wise ("N-layout")
|
||||
// ref: build_image_block in inference/image_processor.py
|
||||
for (int t = 0; t < bl.rows * bl.row_len; t++) {
|
||||
const int g = t / (2 * bl.row_len);
|
||||
const int rem = t % (2 * bl.row_len);
|
||||
const int c = rem / 2; // column
|
||||
const int r = 2 * g + rem % 2; // row
|
||||
if (r >= n_llm_h) {
|
||||
idx.push_back(idx_pad);
|
||||
} else if (c == n_llm_w) {
|
||||
idx.push_back(idx_newline);
|
||||
} else {
|
||||
idx.push_back(r * n_llm_w + c);
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < bl.pad_last; i++) {
|
||||
idx.push_back(idx_pad);
|
||||
}
|
||||
idx.push_back(idx_end);
|
||||
set_input_i32("layout_idx", idx);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_GLM_EDGE:
|
||||
{
|
||||
// llava and other models
|
||||
@@ -5853,6 +5954,19 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
|
||||
LOG_INF("\n=== MTMD_DEBUG_EMBEDDINGS ===\n");
|
||||
LOG_INF("Shape: [%lld, %lld]\n", (long long)n_embd, (long long)n_tokens);
|
||||
|
||||
// TEMP debugging (parity validation), will be removed before merge
|
||||
// when the env var holds a path, dump the raw data: [int32 n_tokens][int32 n_embd][f32 data]
|
||||
const char * dump_path = std::getenv("MTMD_DEBUG_EMBEDDINGS");
|
||||
if (dump_path && strcmp(dump_path, "1") != 0) {
|
||||
FILE * f = fopen(dump_path, "wb");
|
||||
if (f) {
|
||||
const int32_t hdr[2] = { (int32_t)n_tokens, (int32_t)n_embd };
|
||||
fwrite(hdr, sizeof(hdr), 1, f);
|
||||
fwrite(emb_data.data(), sizeof(float), emb_data.size(), f);
|
||||
fclose(f);
|
||||
}
|
||||
}
|
||||
|
||||
// Print first few values of first token
|
||||
LOG_INF("Token 0 (first 16 values): ");
|
||||
for (int i = 0; i < std::min((int64_t)16, n_embd); i++) {
|
||||
@@ -5957,6 +6071,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
|
||||
case PROJECTOR_TYPE_PADDLEOCR:
|
||||
case PROJECTOR_TYPE_KIMIK25:
|
||||
case PROJECTOR_TYPE_YASA2:
|
||||
case PROJECTOR_TYPE_DEEPSEEK4V:
|
||||
return ctx->model.mm_2_w->ne[1];
|
||||
case PROJECTOR_TYPE_HUNYUANVL:
|
||||
return ctx->model.mm_model_proj->ne[1];
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
#include "models.h"
|
||||
|
||||
// DeepSeek-V4-Flash-Vision encoder (deepseek4v)
|
||||
//
|
||||
// native-resolution ViT (RMSNorm, SwiGLU, 2D RoPE, no CLS / learned pos-embd)
|
||||
// then the "aligner": 3x3 patch merge (torch.nn.functional.unfold) + 2-layer GELU MLP
|
||||
//
|
||||
// the graph outputs the complete LLM token block, built from the aligner output and 4 learned sentinel embeddings:
|
||||
//
|
||||
// [PAD]*lead_pad [START] <interleaved rows> [PAD]*pad_last [END]
|
||||
//
|
||||
// each aligner row ends with a NEWLINE, an odd row count is padded with a full row of PADs
|
||||
// pairs of adjacent rows are interleaved column-wise ("N-layout")
|
||||
// the mapping is precomputed on CPU as the "layout_idx" input (see set_input in clip.cpp)
|
||||
//
|
||||
// ref: inference/vision.py and inference/image_processor.py in the HF repo
|
||||
|
||||
ggml_cgraph * clip_graph_deepseek4v::build() {
|
||||
const int n_merge = hparams.n_merge;
|
||||
|
||||
// 2D input positions
|
||||
ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches * 4);
|
||||
ggml_set_name(positions, "positions");
|
||||
ggml_set_input(positions);
|
||||
|
||||
int sections[4] = {d_head/4, d_head/4, 0, 0};
|
||||
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
|
||||
return ggml_rope_multi(ctx0, cur, positions, nullptr,
|
||||
d_head/2, sections, GGML_ROPE_TYPE_VISION,
|
||||
0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
};
|
||||
|
||||
ggml_tensor * inp = build_inp();
|
||||
ggml_tensor * cur = build_vit(
|
||||
inp, n_patches,
|
||||
NORM_TYPE_RMS,
|
||||
hparams.ffn_op,
|
||||
nullptr, // no learned pos embd
|
||||
add_pos);
|
||||
cb(cur, "vit_out", -1);
|
||||
|
||||
// aligner patch merge: zero-pad the patch grid to a multiple of n_merge
|
||||
// then F.unfold == im2col with a dummy kernel (same trick as pixtral)
|
||||
{
|
||||
cur = ggml_reshape_3d(ctx0, cur, n_embd, n_patches_x, n_patches_y);
|
||||
cur = ggml_permute(ctx0, cur, 2, 0, 1, 3); // [x, y, n_embd]
|
||||
cur = ggml_cont(ctx0, cur);
|
||||
|
||||
const int pad_x = (n_merge - n_patches_x % n_merge) % n_merge;
|
||||
const int pad_y = (n_merge - n_patches_y % n_merge) % n_merge;
|
||||
if (pad_x || pad_y) {
|
||||
cur = ggml_pad(ctx0, cur, pad_x, pad_y, 0, 0);
|
||||
}
|
||||
|
||||
ggml_tensor * kernel = ggml_view_3d(ctx0, cur, n_merge, n_merge, cur->ne[2], 0, 0, 0);
|
||||
cur = ggml_im2col(ctx0, kernel, cur, n_merge, n_merge, 0, 0, 1, 1, true, inp->type);
|
||||
cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], cur->ne[1] * cur->ne[2]);
|
||||
|
||||
// aligner MLP (F.gelu in the reference == erf-based gelu)
|
||||
cur = build_ffn(cur,
|
||||
model.mm_1_w, model.mm_1_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_2_w, model.mm_2_b,
|
||||
FFN_GELU_ERF,
|
||||
-1);
|
||||
cb(cur, "aligner_out", -1);
|
||||
}
|
||||
|
||||
// assemble the token block: append the sentinel embeddings as extra rows
|
||||
// then reorder everything with the precomputed layout index
|
||||
{
|
||||
const int64_t n_embd_out = cur->ne[0];
|
||||
const int64_t n_grid = cur->ne[1]; // n_llm_w * n_llm_h
|
||||
|
||||
// rows n_grid + 0..3, keep in sync with the index computation in set_input
|
||||
ggml_tensor * sentinels[] = {
|
||||
model.token_embd_img_start,
|
||||
model.token_embd_img_end,
|
||||
model.image_newline,
|
||||
model.token_embd_img_pad,
|
||||
};
|
||||
for (ggml_tensor * tok : sentinels) {
|
||||
cur = ggml_concat(ctx0, cur, ggml_reshape_2d(ctx0, tok, n_embd_out, 1), 1);
|
||||
}
|
||||
|
||||
const int n_llm_w = CLIP_ALIGN(n_patches_x, n_merge) / n_merge;
|
||||
const int n_llm_h = CLIP_ALIGN(n_patches_y, n_merge) / n_merge;
|
||||
const int n_out = dsv4_get_block_layout(n_llm_w, n_llm_h, img.lead_pad).n_out;
|
||||
GGML_ASSERT(n_grid == n_llm_w * n_llm_h);
|
||||
|
||||
ggml_tensor * layout_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_out);
|
||||
ggml_set_name(layout_idx, "layout_idx");
|
||||
ggml_set_input(layout_idx);
|
||||
|
||||
cur = ggml_get_rows(ctx0, cur, layout_idx);
|
||||
}
|
||||
|
||||
// build the graph
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -34,6 +34,11 @@ struct clip_graph_pixtral : clip_graph {
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_deepseek4v : clip_graph {
|
||||
clip_graph_deepseek4v(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_qwen2vl : clip_graph {
|
||||
clip_graph_qwen2vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
@@ -1092,6 +1092,108 @@ clip_image_size mtmd_image_preprocessor_deepseekocr::find_closest_aspect_ratio(
|
||||
return best_ratio;
|
||||
}
|
||||
|
||||
//
|
||||
// DeepSeek-V4-Flash-Vision (deepseek4v)
|
||||
//
|
||||
// port of load_image / safe_resize / solve_resize_ratio / grid_tokens from inference/image_processor.py
|
||||
// the resize solver picks the largest target size (multiple of patch_size) whose LLM token block fits max_n_token
|
||||
//
|
||||
|
||||
// ref: grid_tokens()
|
||||
mtmd_image_preprocessor_deepseek4v::grid_info mtmd_image_preprocessor_deepseek4v::grid_tokens(int best_height, int best_width, int patch_size, int r) {
|
||||
grid_info g;
|
||||
g.n_llm_h = ((best_height / patch_size) + r - 1) / r;
|
||||
g.n_llm_w = ((best_width / patch_size) + r - 1) / r;
|
||||
g.n_tokens = dsv4_get_block_layout(g.n_llm_w, g.n_llm_h, 0).n_out;
|
||||
return g;
|
||||
}
|
||||
|
||||
// ref: solve_resize_ratio()
|
||||
void mtmd_image_preprocessor_deepseek4v::solve_resize_ratio(int height, int width, int p, int r, int max_n_token,
|
||||
int & best_height, int & best_width) {
|
||||
const double ratio = (double) height / width;
|
||||
const double max_w_f = std::sqrt((max_n_token - 2) / ratio + 0.25) - 0.5;
|
||||
const double max_h_f = max_w_f * ratio;
|
||||
if (max_w_f < 1.0) {
|
||||
const int max_w = 1;
|
||||
int max_h = (max_n_token - 2) / (max_w + 1);
|
||||
if (max_h % 2 == 1) {
|
||||
max_h -= 1;
|
||||
}
|
||||
best_width = max_w * p * r;
|
||||
best_height = max_h * p * r;
|
||||
} else if (max_h_f < 2.0) {
|
||||
const int max_h = 2;
|
||||
// guard tiny budgets; cannot be hit with the current lower bound on max_n_token
|
||||
const int max_w = std::max(((max_n_token - 2) / max_h) - 1, 2);
|
||||
best_width = max_w * p * r;
|
||||
best_height = max_h * p * r;
|
||||
} else {
|
||||
const int max_w_i = (int) std::floor(max_w_f);
|
||||
int max_h_i = (int) std::floor(max_h_f);
|
||||
if (max_h_i % 2 == 1) {
|
||||
max_h_i -= 1;
|
||||
}
|
||||
const double beta = std::min(
|
||||
(double) max_w_i * p * r / width,
|
||||
(double) max_h_i * p * r / height);
|
||||
best_width = (int) std::floor(width * beta / p) * p;
|
||||
best_height = (int) std::floor(height * beta / p) * p;
|
||||
}
|
||||
}
|
||||
|
||||
// ref: safe_resize()
|
||||
void mtmd_image_preprocessor_deepseek4v::safe_resize(int height, int width, int & best_height, int & best_width,
|
||||
int p, int r, int max_n_token) {
|
||||
max_n_token -= 4 - 1; // reserve room for the position-dependent lead pads (COMPRESS_PAD_TO - 1)
|
||||
grid_info g = grid_tokens(best_height, best_width, p, r);
|
||||
int budget = max_n_token;
|
||||
while (g.n_tokens > max_n_token) {
|
||||
solve_resize_ratio(height, width, p, r, budget, best_height, best_width);
|
||||
g = grid_tokens(best_height, best_width, p, r);
|
||||
budget -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
// ref: load_image()
|
||||
mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const clip_image_u8 & img) {
|
||||
mtmd_image_preproc_out out;
|
||||
|
||||
const int p = hparams.patch_size;
|
||||
const int r = hparams.n_merge;
|
||||
const int max_n_token = hparams.dsv4_max_n_token;
|
||||
const int max_wh = hparams.dsv4_max_wh_ratio;
|
||||
|
||||
const clip_image_size orig = img.get_size();
|
||||
int width = orig.width;
|
||||
int height = orig.height;
|
||||
if (max_wh > 0 && width > height * max_wh) {
|
||||
width = height * max_wh;
|
||||
}
|
||||
if (hparams.image_min_pixels > 0 && width * height > 0
|
||||
&& width * height < hparams.image_min_pixels) {
|
||||
const double up = std::sqrt((double) hparams.image_min_pixels / ((double) width * height));
|
||||
width = (int) (width * up);
|
||||
height = (int) (height * up);
|
||||
}
|
||||
int best_width = CLIP_ALIGN(width, p);
|
||||
int best_height = CLIP_ALIGN(height, p);
|
||||
safe_resize(height, width, best_height, best_width, p, r, max_n_token);
|
||||
|
||||
clip_image_u8 resized;
|
||||
if (max_wh > 0 && orig.width >= max_wh * orig.height) {
|
||||
// extreme aspect ratio: plain stretch resize, no padding
|
||||
img_tool::resize(img, resized, {best_width, best_height}, hparams.image_resize_algo, PAD_NONE);
|
||||
} else {
|
||||
// aspect-preserving resize + centered padding (PIL ImageOps.pad)
|
||||
img_tool::resize(img, resized, {best_width, best_height}, hparams.image_resize_algo,
|
||||
PAD_NEAREST, hparams.image_pad_color);
|
||||
}
|
||||
|
||||
out.append(hparams, resized);
|
||||
return out;
|
||||
}
|
||||
|
||||
mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) {
|
||||
mtmd_image_preproc_out output;
|
||||
int grid_w = 0;
|
||||
|
||||
@@ -129,6 +129,22 @@ struct mtmd_image_preprocessor_longest_edge : mtmd_image_preprocessor {
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
|
||||
};
|
||||
|
||||
// ref: inference/image_processor.py in the HF repo (DeepSeek-V4-Flash-Vision)
|
||||
struct mtmd_image_preprocessor_deepseek4v : mtmd_image_preprocessor {
|
||||
mtmd_image_preprocessor_deepseek4v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
|
||||
|
||||
private:
|
||||
struct grid_info {
|
||||
int n_llm_h;
|
||||
int n_llm_w;
|
||||
int n_tokens; // token count of the block (incl. newline/pad rows and start/end, excl. lead pads)
|
||||
};
|
||||
static grid_info grid_tokens(int best_height, int best_width, int patch_size, int r);
|
||||
static void solve_resize_ratio(int height, int width, int p, int r, int max_n_token, int & best_height, int & best_width);
|
||||
static void safe_resize(int height, int width, int & best_height, int & best_width, int p, int r, int max_n_token);
|
||||
};
|
||||
|
||||
// custom llava-uhd slicing logic for MiniCPM-V
|
||||
struct mtmd_image_preprocessor_minicpmv : mtmd_image_preprocessor_llava_uhd {
|
||||
using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd;
|
||||
|
||||
+20
-1
@@ -27,7 +27,7 @@
|
||||
#include <vector>
|
||||
|
||||
// remember to bump this if the serialization format changes
|
||||
#define MTMD_SERIALIZATION_VERSION 1
|
||||
#define MTMD_SERIALIZATION_VERSION 2
|
||||
|
||||
struct mtmd_serialization {
|
||||
// note: using 64-bit here for future-proofing
|
||||
@@ -105,12 +105,14 @@ void clip_image_f32::serialize(mtmd_serialization & ser) const {
|
||||
// note: buf is intentionally NOT serialized; the loaded clip_image_f32 will always be a placeholder
|
||||
ser.write(add_viewsep);
|
||||
ser.write(add_newline);
|
||||
ser.write(lead_pad);
|
||||
ser.write((int32_t)nx_);
|
||||
ser.write((int32_t)ny_);
|
||||
}
|
||||
void clip_image_f32::deserialize(mtmd_serialization & ser) {
|
||||
add_viewsep = ser.read<bool>();
|
||||
add_newline = ser.read<bool>();
|
||||
lead_pad = ser.read<int32_t>();
|
||||
nx_ = ser.read<int32_t>();
|
||||
ny_ = ser.read<int32_t>();
|
||||
buf.clear(); // always a placeholder after loading
|
||||
@@ -824,6 +826,11 @@ struct mtmd_context {
|
||||
img_end = "<|im_end|>";
|
||||
image_preproc = std::make_unique<mtmd_image_preprocessor_longest_edge>(ctx_v);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DEEPSEEK4V:
|
||||
{
|
||||
// no vocab tokens are added; the start/end/newline markers are learned embeddings emitted by the encoder
|
||||
image_preproc = std::make_unique<mtmd_image_preprocessor_deepseek4v>(ctx_v);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DOTS_OCR:
|
||||
case PROJECTOR_TYPE_DOTS3NOTE_V:
|
||||
{
|
||||
@@ -1451,6 +1458,18 @@ struct mtmd_tokenizer {
|
||||
return 2;
|
||||
}
|
||||
|
||||
if (ctx->proj_type_v() == PROJECTOR_TYPE_DEEPSEEK4V) {
|
||||
// the text model perceives input in blocks of N tokens (N = COMPRESS_PAD_TO = 4, same as the CSA compress ratio)
|
||||
// image need to be aligned to block size, while adding IMAGE_PAD embeddings to the beginning
|
||||
// TODO @ngxson : maybe refactor this in the future
|
||||
constexpr int32_t align = 4;
|
||||
size_t n_past = 0;
|
||||
for (const auto & e : cur.entries) {
|
||||
n_past += mtmd_input_chunk_get_n_tokens(&e);
|
||||
}
|
||||
preproc_out.entries[0].lead_pad = align - 1 - (int32_t)(n_past % align);
|
||||
}
|
||||
|
||||
size_t n_tokens = 0;
|
||||
for (auto & e : preproc_out.entries) {
|
||||
n_tokens += clip_n_output_tokens(ctx->ctx_v, &e);
|
||||
|
||||
Reference in New Issue
Block a user