Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-20 01:31:31 +02:00
Code Issues Packages Projects Releases Wiki Activity

vulkan: drop D2D, make the ring the default, remove the OFF toggle

Browse Source 
- Remove the GGML_VK_COMM_D2D peer-buffer path entirely (helpers,
  VK_KHR_external_memory_fd / VK_EXT_external_memory_dma_buf detection+enable,
  comm fields, init validation, ensure() branch). On AMD it regresses badly:
  the dmabuf import lands in GTT (PCIe P2P not established on stock kernels),
  so peer reads stall the GPU (pp/tg down 40-90%, power collapses). Not viable
  without box access to validate true VRAM P2P; revisit later.
- Make the O(n) ring the default large-tensor AllReduce. The old all-to-all
  pipeline is now opt-in via GGML_VK_COMM_PIPELINE (was: ring opt-in via
  GGML_VK_COMM_RING).
- Remove the GGML_VK_COMM_OFF toggle (forced the meta-backend butterfly on
  Vulkan); the custom comm is always better. The generic butterfly fallback in
  the meta backend stays -- it is the shared fallback for CUDA/SYCL and for
  Vulkan configs without a usable custom comm (e.g. MoltenVK).

Verified byte-identical greedy across ring / pipeline / proxy / fp32 on 4x A16.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ApKCQ32VLqUW4Kus6tUvBL
This commit is contained in:
Piotr Wilkin
2026-06-28 20:20:26 +02:00
parent 5a829a9ac9
commit bd1d3189b2
1 changed files with 8 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-vulkan/ggml-vulkan.cpp
+8 -166
View File
@@ -679,8 +679,6 @@ struct vk_device_struct {
uint64_t min_imported_host_pointer_alignment;
bool external_memory_host {};
bool external_semaphore {};
bool external_memory_fd {};
bool external_memory_dma_buf {};
bool fp16;
bool bf16;
bool pipeline_robustness;
@@ -5745,10 +5743,6 @@ static vk_device ggml_vk_get_device(size_t idx) {
device->external_memory_host = true;
} else if (strcmp("VK_KHR_external_semaphore_fd", properties.extensionName) == 0) {
device->external_semaphore = true;
} else if (strcmp("VK_KHR_external_memory_fd", properties.extensionName) == 0) {
device->external_memory_fd = true;
} else if (strcmp("VK_EXT_external_memory_dma_buf", properties.extensionName) == 0) {
device->external_memory_dma_buf = true;
#if defined(VK_EXT_shader_64bit_indexing)
} else if (strcmp("VK_EXT_shader_64bit_indexing", properties.extensionName) == 0) {
device->shader_64b_indexing = true;
@@ -6080,13 +6074,6 @@ static vk_device ggml_vk_get_device(size_t idx) {
device_extensions.push_back("VK_KHR_external_semaphore");
device_extensions.push_back("VK_KHR_external_semaphore_fd");
}
if (device->external_memory_fd) {
device_extensions.push_back("VK_KHR_external_memory");
device_extensions.push_back("VK_KHR_external_memory_fd");
}
if (device->external_memory_dma_buf) {
device_extensions.push_back("VK_EXT_external_memory_dma_buf");
}
#if defined(VK_EXT_shader_64bit_indexing)
VkPhysicalDeviceShader64BitIndexingFeaturesEXT shader_64bit_indexing_features {};
@@ -17861,10 +17848,6 @@ struct ggml_backend_vk_comm_context {
std::vector<uint64_t> pool_max_val;
PFN_vkGetSemaphoreFdKHR pGetSemFd = nullptr;
PFN_vkImportSemaphoreFdKHR pImportSemFd = nullptr;
bool d2d = false;
PFN_vkGetMemoryFdKHR pGetMemFd = nullptr;
PFN_vkGetMemoryFdPropertiesKHR pGetMemFdProps = nullptr;
std::vector<vk_buffer> d2d_own;
std::vector<void*> host_ptr;
std::vector<std::vector<vk_buffer>> host_buf;
std::vector<ggml_backend_buffer_t> tmp_buffer;
@@ -17949,96 +17932,6 @@ static vk::Semaphore ggml_vk_import_timeline(ggml_backend_vk_comm_context * comm
return dst;
}
static vk_buffer ggml_vk_comm_create_exportable(vk_device & device, size_t size) {
vk_buffer buf = std::make_shared<vk_buffer_struct>();
buf->size = size;
buf->device = device;
buf->ptr = nullptr;
vk::ExternalMemoryBufferCreateInfo ext_bci{ vk::ExternalMemoryHandleTypeFlagBits::eDmaBufEXT };
vk::BufferCreateInfo bci{ {}, size,
vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst, vk::SharingMode::eExclusive };
bci.pNext = &ext_bci;
buf->buffer = device->device.createBuffer(bci);
vk::MemoryRequirements mem_req = device->device.getBufferMemoryRequirements(buf->buffer);
vk::PhysicalDeviceMemoryProperties mem_props = device->physical_device.getMemoryProperties();
uint32_t mtype = UINT32_MAX;
for (uint32_t t = 0; t < mem_props.memoryTypeCount; t++) {
if (!(mem_req.memoryTypeBits & (1u << t))) { continue; }
if (mem_props.memoryTypes[t].propertyFlags & vk::MemoryPropertyFlagBits::eDeviceLocal) { mtype = t; break; }
}
if (mtype == UINT32_MAX) { GGML_LOG_WARN("ggml_vulkan: D2D export: no device-local memory type\n"); return {}; }
vk::ExportMemoryAllocateInfo emai{ vk::ExternalMemoryHandleTypeFlagBits::eDmaBufEXT };
vk::MemoryAllocateInfo mai{ mem_req.size, mtype };
mai.pNext = &emai;
try {
buf->device_memory = device->device.allocateMemory(mai);
} catch (const vk::SystemError & e) {
GGML_LOG_WARN("ggml_vulkan: D2D export: allocateMemory(exportable) failed: %s\n", e.what());
return {};
}
buf->memory_property_flags = mem_props.memoryTypes[mtype].propertyFlags;
device->device.bindBufferMemory(buf->buffer, buf->device_memory, 0);
return buf;
}
static vk_buffer ggml_vk_comm_import_buffer(ggml_backend_vk_comm_context * comm, vk_device & dst_dev,
vk_device & src_dev, vk_buffer & src_buf, size_t size) {
int fd = -1;
VkMemoryGetFdInfoKHR gfi{};
gfi.sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR;
gfi.memory = (VkDeviceMemory) src_buf->device_memory;
gfi.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT;
if (comm->pGetMemFd((VkDevice) src_dev->device, &gfi, &fd) != VK_SUCCESS || fd < 0) {
GGML_LOG_WARN("ggml_vulkan: D2D import: vkGetMemoryFdKHR failed (fd=%d)\n", fd);
return {};
}
vk_buffer buf = std::make_shared<vk_buffer_struct>();
buf->size = size;
buf->device = dst_dev;
buf->ptr = nullptr;
vk::ExternalMemoryBufferCreateInfo ext_bci{ vk::ExternalMemoryHandleTypeFlagBits::eDmaBufEXT };
vk::BufferCreateInfo bci{ {}, size,
vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst, vk::SharingMode::eExclusive };
bci.pNext = &ext_bci;
buf->buffer = dst_dev->device.createBuffer(bci);
vk::MemoryRequirements mem_req = dst_dev->device.getBufferMemoryRequirements(buf->buffer);
VkMemoryFdPropertiesKHR fdProps{};
fdProps.sType = VK_STRUCTURE_TYPE_MEMORY_FD_PROPERTIES_KHR;
comm->pGetMemFdProps((VkDevice) dst_dev->device, VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, fd, &fdProps);
vk::PhysicalDeviceMemoryProperties mem_props = dst_dev->physical_device.getMemoryProperties();
uint32_t mtype = UINT32_MAX;
for (uint32_t t = 0; t < mem_props.memoryTypeCount; t++) {
if (!(mem_req.memoryTypeBits & (1u << t))) { continue; }
if (!(fdProps.memoryTypeBits & (1u << t))) { continue; }
mtype = t;
if (mem_props.memoryTypes[t].propertyFlags & vk::MemoryPropertyFlagBits::eDeviceLocal) { break; }
}
if (mtype == UINT32_MAX) {
GGML_LOG_WARN("ggml_vulkan: D2D import: no compatible memory type (fdBits=0x%x reqBits=0x%x)\n",
fdProps.memoryTypeBits, mem_req.memoryTypeBits);
return {};
}
VkImportMemoryFdInfoKHR imfi{};
imfi.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR;
imfi.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT;
imfi.fd = fd;
vk::MemoryAllocateInfo mai{ mem_req.size, mtype };
mai.pNext = &imfi;
try {
buf->device_memory = dst_dev->device.allocateMemory(mai);
} catch (const vk::SystemError & e) {
GGML_LOG_WARN("ggml_vulkan: D2D import: allocateMemory(import) failed: %s\n", e.what());
return {};
}
buf->memory_property_flags = mem_props.memoryTypes[mtype].propertyFlags;
dst_dev->device.bindBufferMemory(buf->buffer, buf->device_memory, 0);
return buf;
}
static bool ggml_vk_comm_opaque_fd_supported(ggml_backend_vk_comm_context * comm) {
const size_t n = comm->device.size();
uint8_t driver_uuid[VK_UUID_SIZE];
@@ -18087,9 +17980,7 @@ static void * ggml_backend_vk_comm_init(ggml_backend_t * backends, size_t n_back
comm->host_buf[k].resize(n_backends);
}
comm->use_f16 = (getenv("GGML_VK_COMM_FP32") == nullptr);
comm->ring = (getenv("GGML_VK_COMM_RING") != nullptr);
comm->d2d = (getenv("GGML_VK_COMM_D2D") != nullptr);
comm->d2d_own.resize(n_backends, nullptr);
comm->ring = (getenv("GGML_VK_COMM_PIPELINE") == nullptr);
comm->up16_buffer.resize(n_backends, nullptr);
comm->dn16_buffer.resize(n_backends, nullptr);
comm->up16_tensor.resize(n_backends, nullptr);
@@ -18103,18 +17994,6 @@ static void * ggml_backend_vk_comm_init(ggml_backend_t * backends, size_t n_back
comm->pImportSemFd = (PFN_vkImportSemaphoreFdKHR) comm->device[0]->device.getProcAddr("vkImportSemaphoreFdKHR");
comm->fast = comm->pGetSemFd && comm->pImportSemFd;
}
if (comm->fast && comm->d2d) {
bool dmabuf = true;
for (size_t i = 0; i < n_backends; i++) {
dmabuf = dmabuf && comm->device[i]->external_memory_fd && comm->device[i]->external_memory_dma_buf;
}
comm->pGetMemFd = (PFN_vkGetMemoryFdKHR) comm->device[0]->device.getProcAddr("vkGetMemoryFdKHR");
comm->pGetMemFdProps = (PFN_vkGetMemoryFdPropertiesKHR) comm->device[0]->device.getProcAddr("vkGetMemoryFdPropertiesKHR");
comm->d2d = dmabuf && comm->pGetMemFd && comm->pGetMemFdProps;
if (!comm->d2d) {
GGML_LOG_INFO("ggml_vulkan: comm D2D requested but VK_EXT_external_memory_dma_buf unavailable; using host staging\n");
}
}
if (comm->fast) {
comm->prog.resize(n_backends);
comm->peer_prog.assign(n_backends, std::vector<vk::Semaphore>(n_backends));
@@ -18271,7 +18150,6 @@ static bool ggml_backend_vk_comm_ensure(ggml_backend_vk_comm_context * comm, siz
for (size_t i = 0; i < n; i++) {
comm->host_buf[k][i].reset();
}
comm->d2d_own[k].reset();
ggml_vk_comm_aligned_free(comm->host_ptr[k]);
comm->host_ptr[k] = nullptr;
}
@@ -18291,47 +18169,16 @@ static bool ggml_backend_vk_comm_ensure(ggml_backend_vk_comm_context * comm, siz
}
const size_t slotcap = (comm->ring ? 4 : 2) * newcap;
if (comm->d2d) {
bool ok = true;
for (size_t k = 0; k < n && ok; k++) {
comm->d2d_own[k] = ggml_vk_comm_create_exportable(comm->device[k], slotcap);
if (!comm->d2d_own[k]) { ok = false; break; }
comm->host_buf[k][k] = comm->d2d_own[k];
for (size_t i = 0; i < n; i++) {
if (i == k) {
continue;
}
comm->host_buf[k][i] = ggml_vk_comm_import_buffer(comm, comm->device[i], comm->device[k],
comm->d2d_own[k], slotcap);
if (!comm->host_buf[k][i]) { ok = false; break; }
}
for (size_t k = 0; k < n; k++) {
comm->host_ptr[k] = ggml_vk_comm_aligned_alloc(comm->align, slotcap);
if (!comm->host_ptr[k]) {
return false;
}
if (ok) {
GGML_LOG_INFO("ggml_vulkan: comm D2D peer buffers active (%zu devices, %zu KiB/slot over PCIe P2P)\n",
n, slotcap / 1024);
} else {
for (size_t k = 0; k < n; k++) {
for (size_t i = 0; i < n; i++) {
comm->host_buf[k][i].reset();
}
comm->d2d_own[k].reset();
}
comm->d2d = false;
GGML_LOG_WARN("ggml_vulkan: comm D2D peer import unsupported here; falling back to host staging\n");
}
}
if (!comm->d2d) {
for (size_t k = 0; k < n; k++) {
comm->host_ptr[k] = ggml_vk_comm_aligned_alloc(comm->align, slotcap);
if (!comm->host_ptr[k]) {
for (size_t i = 0; i < n; i++) {
comm->host_buf[k][i] = ggml_vk_buffer_from_host_ptr(comm->device[i], comm->host_ptr[k], slotcap);
if (!comm->host_buf[k][i]) {
return false;
}
for (size_t i = 0; i < n; i++) {
comm->host_buf[k][i] = ggml_vk_buffer_from_host_ptr(comm->device[i], comm->host_ptr[k], slotcap);
if (!comm->host_buf[k][i]) {
return false;
}
}
}
}
for (size_t i = 0; i < n; i++) {
@@ -18718,11 +18565,6 @@ static bool ggml_backend_vk_comm_allreduce_tensor(void * comm_ctx, ggml_tensor *
ggml_backend_vk_comm_context * comm = static_cast<ggml_backend_vk_comm_context *>(comm_ctx);
const size_t n = comm->backends.size();
static const bool off = getenv("GGML_VK_COMM_OFF") != nullptr;
if (off) {
return false;
}
if (!comm->fast || tensors[0]->type != GGML_TYPE_F32) {
return false;
}