Public/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-11 15:29:03 +02:00
Code Issues Packages Projects Releases Wiki Activity

Completely revamp how we do object management with the vulkan backend and

Browse Source 
stop using so many static objects so we can tear down and bring up vulkan
on new devices in the same runtime.
This commit is contained in:
Adam Treat
2023-09-12 13:04:55 -04:00
parent 5b2d8236a7
commit 0412ec287c
7 changed files with 175 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kompute/src/Manager.cpp
+30 -11
View File
@@ -88,15 +88,14 @@ Manager::destroy()
this->mManagedSequences.clear();
}
if (this->mManageResources && this->mManagedAlgorithms.size()) {
if (this->mManageResources && !this->mManagedAlgorithmsMap.empty()) {
KP_LOG_DEBUG("Kompute Manager explicitly freeing algorithms");
for (const std::weak_ptr<Algorithm>& weakAlgorithm :
this->mManagedAlgorithms) {
if (std::shared_ptr<Algorithm> algorithm = weakAlgorithm.lock()) {
for (const auto& kv : this->mManagedAlgorithmsMap) {
if (std::shared_ptr<Algorithm> algorithm = kv.second) {
algorithm->destroy();
}
}
this->mManagedAlgorithms.clear();
this->mManagedAlgorithmsMap.clear();
}
if (this->mManageResources && this->mManagedTensors.size()) {
@@ -109,6 +108,18 @@ Manager::destroy()
this->mManagedTensors.clear();
}
if (this->mPipelineCache) {
KP_LOG_DEBUG("Kompute Manager Destroying pipeline cache");
if (!this->mPipelineCache) {
KP_LOG_WARN("Kompute Manager Error requested to destroy "
"pipeline cache but it is null");
}
this->mDevice->destroy(
*this->mPipelineCache,
(vk::Optional<const vk::AllocationCallbacks>)nullptr);
this->mPipelineCache = nullptr;
}
if (this->mFreeDevice) {
KP_LOG_INFO("Destroying device");
this->mDevice->destroy(
@@ -269,12 +280,14 @@ Manager::clear()
end(this->mManagedTensors),
[](std::weak_ptr<Tensor> t) { return t.expired(); }),
end(this->mManagedTensors));
this->mManagedAlgorithms.erase(
std::remove_if(
begin(this->mManagedAlgorithms),
end(this->mManagedAlgorithms),
[](std::weak_ptr<Algorithm> t) { return t.expired(); }),
end(this->mManagedAlgorithms));
for (auto it = this->mManagedAlgorithmsMap.begin();
it != this->mManagedAlgorithmsMap.end();) {
if (it->second) {
it = this->mManagedAlgorithmsMap.erase(it);
} else {
++it;
}
}
this->mManagedSequences.erase(
std::remove_if(begin(this->mManagedSequences),
end(this->mManagedSequences),
@@ -452,6 +465,12 @@ Manager::createDevice(const std::vector<uint32_t>& familyQueueIndices,
}
KP_LOG_DEBUG("Kompute Manager compute queue obtained");
mPipelineCache = std::make_shared<vk::PipelineCache>();
vk::PipelineCacheCreateInfo pipelineCacheInfo =
vk::PipelineCacheCreateInfo();
this->mDevice->createPipelineCache(
&pipelineCacheInfo, nullptr, mPipelineCache.get());
}
std::shared_ptr<Sequence>