Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-04 12:01:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

metal: enable Metal 4.0 tensor API on M5+/A19+ (#27461)

Browse Source 
* metal : request Metal 4.0 language version for the tensor API

* metal : load the tensor API kernels from a separate metallib

* tests : add external-metallib tensor API regression test

* metal : fix metallib build order for the tensor API kernels
This commit is contained in:
James Francis
2026-09-01 03:02:42 -06:00
committed by GitHub
parent 234a6ebaa0
commit d5d993a093
3 changed files with 124 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-metal/CMakeLists.txt
+20 -1
View File
@@ -163,19 +163,37 @@ else()
)
endforeach()
# the tensor API kernels go in a separate metallib, loaded only where supported
set(AIR_MM_TENSOR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/mul_mm_tensor.air")
add_custom_command(
OUTPUT ${AIR_MM_TENSOR}
COMMAND xcrun -sdk macosx metal ${XC_FLAGS} -DGGML_METAL_HAS_TENSOR -I ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels/mul_mm.metal -o ${AIR_MM_TENSOR}
DEPENDS kernels/mul_mm.metal kernels/common.h kernels/dequantize.h ${METALLIB_COMMON} ggml-metal-impl.h
COMMENT "Compiling kernels/mul_mm.metal (tensor API)"
VERBATIM
)
add_custom_command(
OUTPUT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib
COMMAND xcrun -sdk macosx metallib ${AIR_MM_TENSOR} -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib
DEPENDS ${AIR_MM_TENSOR}
COMMENT "Linking tensor API Metal kernels into ggml-tensor.metallib"
)
add_custom_command(
OUTPUT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
COMMAND xcrun -sdk macosx metallib ${AIR_FILES} -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h
COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal-impl.h
COMMAND rm -rf ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels
DEPENDS ${AIR_FILES}
DEPENDS ${AIR_FILES} ${AIR_MM_TENSOR}
COMMENT "Linking Metal kernels into default.metallib"
)
add_custom_target(
ggml-metal-lib ALL
DEPENDS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib
)
endif() # GGML_METAL_EMBED_LIBRARY
@@ -188,6 +206,7 @@ if (NOT GGML_METAL_EMBED_LIBRARY)
install(
FILES ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib
DESTINATION ${CMAKE_INSTALL_BINDIR}
)
endif()
ggml/src/ggml-metal/ggml-metal-device.m
+96 -33
View File
@@ -27,6 +27,9 @@
static const NSInteger MTLGPUFamilyMetal3_GGML = 5001;
static const NSInteger MTLGPUFamilyMetal4_GGML = 5002;
// MTLLanguageVersion4_0 is not present in older SDKs
static const NSUInteger MTLLanguageVersion4_0_GGML = 4 << 16;
#if !GGML_METAL_EMBED_LIBRARY
// Here to assist with NSBundle Path Hack
@interface GGMLMetalClass : NSObject
@@ -154,6 +157,9 @@ struct ggml_metal_library {
// nil in single_library mode (everything resolves to objs[0]).
NSMutableDictionary<NSString *, NSNumber *> * fn_to_lib;
// kernels from a second metallib, resolved ahead of the combined library
NSSet<NSString *> * override_fns;
ggml_metal_device_t dev;
ggml_metal_pipelines_t pipelines; // cache of compiled pipelines
@@ -174,6 +180,18 @@ static void ggml_metal_library_build_index(ggml_metal_library_t lib) {
}
}
// note: defined below, after struct ggml_metal_device
static void ggml_metal_device_disable_tensor(ggml_metal_device_t dev);
// the tensor API headers are exposed to the shader compiler only at Metal language version 4.0
static void ggml_metal_compile_options_set_lang(MTLCompileOptions * options, bool has_tensor) {
if (!has_tensor) {
return;
}
options.languageVersion = (MTLLanguageVersion) MTLLanguageVersion4_0_GGML;
}
// Parse a `#include "name"` line. Returns the quoted name in *include_name on
// success. Whitespace-tolerant; ignores `#include <...>` (system headers).
static bool ggml_metal_library_parse_quoted_include(NSString * line, NSString ** include_name) {
@@ -313,6 +331,7 @@ static bool ggml_metal_library_compile_all(
@autoreleasepool {
MTLCompileOptions * options = [MTLCompileOptions new];
options.preprocessorMacros = prep;
ggml_metal_compile_options_set_lang(options, ggml_metal_device_get_props(res->dev)->has_tensor);
lib = [device newLibraryWithSource:src options:options error:&error];
@@ -369,6 +388,46 @@ static bool ggml_metal_library_compile_all(
return ok;
}
// look for <name>.metallib as a bundle resource, then next to the running binary
static NSString * ggml_metal_find_metallib(NSBundle * bundle, NSString * name) {
NSError * error = nil;
NSString * path_lib = [bundle pathForResource:name ofType:@"metallib"];
if (path_lib == nil) {
// Try to find the resource in the directory where the current binary located.
NSString * bin_cur = [[NSProcessInfo processInfo] arguments][0];
NSString * bin_dir = [bin_cur stringByDeletingLastPathComponent];
NSString * path_lib_default = [NSString pathWithComponents:@[bin_dir, [name stringByAppendingPathExtension:@"metallib"]]];
if ([[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {
GGML_LOG_INFO("%s: found '%s'\n", __func__, [path_lib_default UTF8String]);
NSDictionary * atts = [[NSFileManager defaultManager] attributesOfItemAtPath:path_lib_default error:&error];
if (atts && atts[NSFileType] == NSFileTypeSymbolicLink) {
// Optionally, if this is a symlink, try to resolve it.
path_lib_default = [[NSFileManager defaultManager] destinationOfSymbolicLinkAtPath:path_lib_default error:&error];
if (path_lib_default && [path_lib_default length] > 0 && ![[path_lib_default substringToIndex:1] isEqualToString:@"/"]) {
// It is a relative path, adding the binary directory as directory prefix.
path_lib_default = [NSString pathWithComponents:@[bin_dir, path_lib_default]];
}
if (!path_lib_default || ![[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {
// Link to the resource could not be resolved.
path_lib_default = nil;
} else {
GGML_LOG_INFO("%s: symlink resolved '%s'\n", __func__, [path_lib_default UTF8String]);
}
}
} else {
// The resource couldn't be found in the binary's directory.
path_lib_default = nil;
}
path_lib = path_lib_default;
}
return path_lib;
}
ggml_metal_library_t ggml_metal_library_init(ggml_metal_device_t dev) {
id<MTLDevice> device = ggml_metal_device_get_obj(dev);
@@ -432,38 +491,7 @@ ggml_metal_library_t ggml_metal_library_init(ggml_metal_device_t dev) {
const int64_t t_start = ggml_time_us();
NSError * error = nil;
NSString * path_lib = [bundle pathForResource:@"default" ofType:@"metallib"];
if (path_lib == nil) {
// Try to find the resource in the directory where the current binary located.
NSString * bin_cur = [[NSProcessInfo processInfo] arguments][0];
NSString * bin_dir = [bin_cur stringByDeletingLastPathComponent];
NSString * path_lib_default = [NSString pathWithComponents:@[bin_dir, @"default.metallib"]];
if ([[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {
GGML_LOG_INFO("%s: found '%s'\n", __func__, [path_lib_default UTF8String]);
NSDictionary * atts = [[NSFileManager defaultManager] attributesOfItemAtPath:path_lib_default error:&error];
if (atts && atts[NSFileType] == NSFileTypeSymbolicLink) {
// Optionally, if this is a symlink, try to resolve it.
path_lib_default = [[NSFileManager defaultManager] destinationOfSymbolicLinkAtPath:path_lib_default error:&error];
if (path_lib_default && [path_lib_default length] > 0 && ![[path_lib_default substringToIndex:1] isEqualToString:@"/"]) {
// It is a relative path, adding the binary directory as directory prefix.
path_lib_default = [NSString pathWithComponents:@[bin_dir, path_lib_default]];
}
if (!path_lib_default || ![[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {
// Link to the resource could not be resolved.
path_lib_default = nil;
} else {
GGML_LOG_INFO("%s: symlink resolved '%s'\n", __func__, [path_lib_default UTF8String]);
}
}
} else {
// The resource couldn't be found in the binary's directory.
path_lib_default = nil;
}
path_lib = path_lib_default;
}
NSString * path_lib = ggml_metal_find_metallib(bundle, @"default");
if (path_lib != nil) {
// pre-compiled library found: a single combined default.metallib
@@ -478,6 +506,30 @@ ggml_metal_library_t ggml_metal_library_init(ggml_metal_device_t dev) {
return NULL;
}
// the tensor API kernels are built into a separate metallib
if (ggml_metal_device_get_props(dev)->has_tensor) {
NSString * path_mm = ggml_metal_find_metallib(bundle, @"ggml-tensor");
id<MTLLibrary> lib_mm = nil;
if (path_mm != nil) {
lib_mm = [device newLibraryWithURL:[NSURL fileURLWithPath:path_mm] error:&error];
if (!lib_mm && error) {
GGML_LOG_ERROR("%s: %s\n", __func__, [[error description] UTF8String]);
}
}
if (lib_mm) {
GGML_LOG_INFO("%s: loaded '%s'\n", __func__, [path_mm UTF8String]);
res->objs[GGML_METAL_LIB_MUL_MM] = [lib_mm retain];
res->override_fns = [[NSSet setWithArray:[lib_mm functionNames]] retain];
} else {
GGML_LOG_INFO("%s: ggml-tensor.metallib not found - disabling the tensor API\n", __func__);
ggml_metal_device_disable_tensor(dev);
}
}
GGML_LOG_INFO("%s: loaded in %.3f sec\n", __func__, (ggml_time_us() - t_start) / 1e6);
return res;
}
@@ -557,6 +609,7 @@ ggml_metal_library_t ggml_metal_library_init_from_source(ggml_metal_device_t dev
MTLCompileOptions * options = [MTLCompileOptions new];
options.preprocessorMacros = prep;
ggml_metal_compile_options_set_lang(options, ggml_metal_device_get_props(dev)->has_tensor);
library = [device newLibraryWithSource:src options:options error:&error];
if (error) {
@@ -615,6 +668,10 @@ void ggml_metal_library_free(ggml_metal_library_t lib) {
[lib->fn_to_lib release];
}
if (lib->override_fns) {
[lib->override_fns release];
}
ggml_metal_pipelines_free(lib->pipelines);
[lib->lock release];
@@ -676,7 +733,9 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_compile_pipeline(ggml_
// route to the library that actually defines this kernel; fn_to_lib is
// built from -[MTLLibrary functionNames] so it's always in sync
int lib_idx = 0;
if (!lib->single_library) {
if (lib->override_fns && [lib->override_fns containsObject:base_func]) {
lib_idx = GGML_METAL_LIB_MUL_MM;
} else if (!lib->single_library) {
NSNumber * idx = lib->fn_to_lib[base_func];
if (!idx) {
[lib->lock unlock];
@@ -1862,6 +1921,10 @@ const struct ggml_metal_device_props * ggml_metal_device_get_props(ggml_metal_de
return &dev->props;
}
static void ggml_metal_device_disable_tensor(ggml_metal_device_t dev) {
dev->props.has_tensor = false;
}
//
// device buffers
//
tests/CMakeLists.txt
+8
View File
@@ -299,6 +299,14 @@ if (NOT LLAMA_SANITIZE_ADDRESS AND NOT GGML_SCHED_NO_REALLOC)
endif()
llama_build_and_test(test-backend-ops.cpp)
# the tensor API kernels come from a separate metallib - check they produce correct results
# ref: https://github.com/ggml-org/llama.cpp/issues/27473
if (GGML_METAL AND NOT GGML_METAL_EMBED_LIBRARY)
llama_test(test-backend-ops NAME test-backend-ops-metallib-tensor
ARGS test -b MTL0 -o MUL_MAT -p type_a=q6_K)
set_tests_properties(test-backend-ops-metallib-tensor PROPERTIES ENVIRONMENT GGML_METAL_TENSOR_ENABLE=1)
endif()
llama_build_and_test(test-model-load-cancel.cpp LABEL "model")
llama_build_and_test(test-autorelease.cpp LABEL "model")
llama_build_and_test(test-backend-sampler.cpp LABEL "model")