[vulkan] Add lazy memory tracking for ResourceMemAllocatorVk
While investigating high GPU memory usage for Graphite/Dawn/Vulkan
we found that lazily allocated memory in both Ganesh/Vulkan
and Graphite/Dawn/Vulkan are being tracked as part of total and used
vulkan memory allocations. Since presumably Graphite has more memory
that is lazily allocated it is better to track this separately.
Hopefully these memory allocations never happen, like on tiling GPUs,
but if an allocation is needed then hopefully it will be transient and
only exists for the duration of the draw.
This change tracks lazy allocated memory for dawn/vulkan separately.
For this we update AllocatorMemoryInfo and AllocationInfo for reporting
and tracking in ResourceMemoryAllocatorVk.
Bug: chromium:407730048
Change-Id: Idda25b1bc0d66ac2ab40e7f6545327bf079cbac8
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/243655
Reviewed-by: Kyle Charbonneau <kylechar@google.com>
Commit-Queue: Saifuddin Hitawala <hitawala@chromium.org>
Reviewed-by: Corentin Wallez <cwallez@chromium.org>
diff --git a/include/dawn/native/DawnNative.h b/include/dawn/native/DawnNative.h
index 28f6c125..9866248 100644
--- a/include/dawn/native/DawnNative.h
+++ b/include/dawn/native/DawnNative.h
@@ -340,6 +340,8 @@
struct DAWN_NATIVE_EXPORT AllocatorMemoryInfo {
uint64_t totalUsedMemory = 0;
uint64_t totalAllocatedMemory = 0;
+ uint64_t totalLazyAllocatedMemory = 0;
+ uint64_t totalLazyUsedMemory = 0;
};
DAWN_NATIVE_EXPORT AllocatorMemoryInfo GetAllocatorMemoryInfo(WGPUDevice device);
diff --git a/src/dawn/native/BuddyMemoryAllocator.cpp b/src/dawn/native/BuddyMemoryAllocator.cpp
index 26f0465..409166e 100644
--- a/src/dawn/native/BuddyMemoryAllocator.cpp
+++ b/src/dawn/native/BuddyMemoryAllocator.cpp
@@ -55,7 +55,8 @@
}
ResultOrError<ResourceMemoryAllocation> BuddyMemoryAllocator::Allocate(uint64_t allocationSize,
- uint64_t alignment) {
+ uint64_t alignment,
+ bool isLazyMemoryType) {
ResourceMemoryAllocation invalidAllocation = ResourceMemoryAllocation{};
if (allocationSize == 0) {
@@ -96,6 +97,7 @@
info.mBlockOffset = blockOffset;
info.mRequestedSize = originalAllocationSize;
info.mMethod = AllocationMethod::kSubAllocated;
+ info.mIsLazyAllocated = isLazyMemoryType;
// Allocation offset is always local to the memory.
const uint64_t memoryOffset = blockOffset % mMemoryBlockSize;
diff --git a/src/dawn/native/BuddyMemoryAllocator.h b/src/dawn/native/BuddyMemoryAllocator.h
index 96ace51..b5878d4 100644
--- a/src/dawn/native/BuddyMemoryAllocator.h
+++ b/src/dawn/native/BuddyMemoryAllocator.h
@@ -58,7 +58,9 @@
ResourceHeapAllocator* heapAllocator);
~BuddyMemoryAllocator();
- ResultOrError<ResourceMemoryAllocation> Allocate(uint64_t allocationSize, uint64_t alignment);
+ ResultOrError<ResourceMemoryAllocation> Allocate(uint64_t allocationSize,
+ uint64_t alignment,
+ bool isLazyMemoryType);
void Deallocate(const ResourceMemoryAllocation& allocation);
uint64_t GetMemoryBlockSize() const;
diff --git a/src/dawn/native/ResourceMemoryAllocation.h b/src/dawn/native/ResourceMemoryAllocation.h
index b373c8e..fa8c932 100644
--- a/src/dawn/native/ResourceMemoryAllocation.h
+++ b/src/dawn/native/ResourceMemoryAllocation.h
@@ -62,6 +62,9 @@
// Represents the requested memory allocation size (without padding) by the allocator.
uint64_t mRequestedSize = 0;
+
+ // Tracks whether the memory allocation contains VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT.
+ bool mIsLazyAllocated = false;
};
// Handle into a resource heap pool.
diff --git a/src/dawn/native/d3d12/ResourceAllocatorManagerD3D12.cpp b/src/dawn/native/d3d12/ResourceAllocatorManagerD3D12.cpp
index f993674..71b5b0a 100644
--- a/src/dawn/native/d3d12/ResourceAllocatorManagerD3D12.cpp
+++ b/src/dawn/native/d3d12/ResourceAllocatorManagerD3D12.cpp
@@ -467,7 +467,8 @@
ResourceMemoryAllocation allocation;
DAWN_TRY_ASSIGN(allocation,
- allocator->Allocate(resourceInfo.SizeInBytes, resourceInfo.Alignment));
+ allocator->Allocate(resourceInfo.SizeInBytes, resourceInfo.Alignment,
+ /*isLazyMemoryType=*/false));
if (allocation.GetInfo().mMethod == AllocationMethod::kInvalid) {
return ResourceHeapAllocation{}; // invalid
}
diff --git a/src/dawn/native/vulkan/DeviceVk.cpp b/src/dawn/native/vulkan/DeviceVk.cpp
index 8c5d02c..299a4e4 100644
--- a/src/dawn/native/vulkan/DeviceVk.cpp
+++ b/src/dawn/native/vulkan/DeviceVk.cpp
@@ -1036,6 +1036,8 @@
AllocatorMemoryInfo info = {};
info.totalAllocatedMemory = GetResourceMemoryAllocator()->GetTotalAllocatedMemory();
info.totalUsedMemory = GetResourceMemoryAllocator()->GetTotalUsedMemory();
+ info.totalLazyAllocatedMemory = GetResourceMemoryAllocator()->GetTotalLazyAllocatedMemory();
+ info.totalLazyUsedMemory = GetResourceMemoryAllocator()->GetTotalLazyUsedMemory();
return info;
}
diff --git a/src/dawn/native/vulkan/ResourceMemoryAllocatorVk.cpp b/src/dawn/native/vulkan/ResourceMemoryAllocatorVk.cpp
index 31de727..d13d492 100644
--- a/src/dawn/native/vulkan/ResourceMemoryAllocatorVk.cpp
+++ b/src/dawn/native/vulkan/ResourceMemoryAllocatorVk.cpp
@@ -97,17 +97,18 @@
// SingleTypeAllocator is a combination of a BuddyMemoryAllocator and its client and can
// service suballocation requests, but for a single Vulkan memory type.
-
class ResourceMemoryAllocator::SingleTypeAllocator : public ResourceHeapAllocator {
public:
SingleTypeAllocator(Device* device,
size_t memoryTypeIndex,
+ bool isLazyMemoryType,
VkDeviceSize maxHeapSize,
VkDeviceSize heapBlockSize,
ResourceMemoryAllocator* memoryAllocator)
: mDevice(device),
mResourceMemoryAllocator(memoryAllocator),
mMemoryTypeIndex(memoryTypeIndex),
+ mIsLazyMemoryType(isLazyMemoryType),
mMaxHeapSize(maxHeapSize),
mPooledMemoryAllocator(this),
mBuddySystem(
@@ -121,11 +122,13 @@
}
~SingleTypeAllocator() override = default;
+ bool IsLazyMemoryType() const { return mIsLazyMemoryType; }
+
// Frees any heaps that are unused and waiting to be recycled by the pool allocator.
void FreeRecycledMemory() { mPooledMemoryAllocator.FreeRecycledAllocations(); }
ResultOrError<ResourceMemoryAllocation> AllocateMemory(uint64_t size, uint64_t alignment) {
- return mBuddySystem.Allocate(size, alignment);
+ return mBuddySystem.Allocate(size, alignment, mIsLazyMemoryType);
}
void DeallocateMemory(const ResourceMemoryAllocation& allocation) {
@@ -133,7 +136,6 @@
}
// Implementation of the MemoryAllocator interface to be a client of BuddyMemoryAllocator
-
ResultOrError<std::unique_ptr<ResourceHeapBase>> AllocateResourceHeap(uint64_t size) override {
if (size > mMaxHeapSize) {
return DAWN_OUT_OF_MEMORY_ERROR("Allocation size too large");
@@ -154,23 +156,47 @@
"vkAllocateMemory"));
DAWN_ASSERT(allocatedMemory != VK_NULL_HANDLE);
- mResourceMemoryAllocator->RecordHeapAllocation(size);
+ mResourceMemoryAllocator->RecordHeapAllocation(size, mIsLazyMemoryType);
return {std::make_unique<ResourceHeap>(allocatedMemory, mMemoryTypeIndex, size)};
}
void DeallocateResourceHeap(std::unique_ptr<ResourceHeapBase> allocation) override {
- mResourceMemoryAllocator->DeallocateResourceHeap(ToBackend(allocation.get()));
+ mResourceMemoryAllocator->DeallocateResourceHeap(ToBackend(allocation.get()),
+ mIsLazyMemoryType);
}
private:
raw_ptr<Device> mDevice;
raw_ptr<ResourceMemoryAllocator> mResourceMemoryAllocator;
size_t mMemoryTypeIndex;
+ const bool mIsLazyMemoryType;
VkDeviceSize mMaxHeapSize;
PooledResourceMemoryAllocator mPooledMemoryAllocator;
BuddyMemoryAllocator mBuddySystem;
};
+void ResourceMemoryAllocator::AllocationSizeTracker::Increment(VkDeviceSize incrementSize) {
+ mTotalSize += incrementSize;
+}
+
+void ResourceMemoryAllocator::AllocationSizeTracker::Decrement(ExecutionSerial currentSerial,
+ VkDeviceSize decrementSize) {
+ DAWN_ASSERT(mTotalSize >= decrementSize);
+ mMemoryToDecrement[currentSerial] += decrementSize;
+}
+
+void ResourceMemoryAllocator::AllocationSizeTracker::Tick(ExecutionSerial completedSerial) {
+ auto it = mMemoryToDecrement.begin();
+ while (it != mMemoryToDecrement.end() && it->first <= completedSerial) {
+ // Update tracking for allocation/used memory that will be deallocated.
+ DAWN_ASSERT(mTotalSize >= it->second);
+ mTotalSize -= it->second;
+ it++;
+ }
+ // Erase the map serials up to the completed serial.
+ mMemoryToDecrement.erase(mMemoryToDecrement.begin(), it);
+}
+
VkDeviceSize ResourceMemoryAllocator::GetHeapBlockSize(const DawnDeviceAllocatorControl* control) {
static constexpr VkDeviceSize kDefaultHeapBlockSize = 8ull * 1024ull * 1024ull; // 8MiB
VkDeviceSize heapBlockSize = kDefaultHeapBlockSize;
@@ -188,8 +214,12 @@
mAllocatorsPerType.reserve(info.memoryTypes.size());
for (size_t i = 0; i < info.memoryTypes.size(); i++) {
+ const auto& memoryType = info.memoryTypes[i];
+ bool isLazyMemoryType =
+ (memoryType.propertyFlags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0u;
mAllocatorsPerType.emplace_back(std::make_unique<SingleTypeAllocator>(
- mDevice, i, info.memoryHeaps[info.memoryTypes[i].heapIndex].size, heapBlockSize, this));
+ mDevice, i, isLazyMemoryType, info.memoryHeaps[memoryType.heapIndex].size,
+ heapBlockSize, this));
}
}
@@ -201,6 +231,7 @@
bool forceDisableSubAllocation) {
// The Vulkan spec guarantees at least one memory type is valid.
int memoryType = FindBestTypeIndex(requirements, kind);
+ bool isLazyMemoryType = mAllocatorsPerType[memoryType]->IsLazyMemoryType();
DAWN_ASSERT(memoryType >= 0);
VkDeviceSize size = requirements.size;
@@ -238,7 +269,8 @@
DAWN_TRY_ASSIGN(subAllocation, mAllocatorsPerType[memoryType]->AllocateMemory(
requirements.size, alignment));
if (subAllocation.GetInfo().mMethod != AllocationMethod::kInvalid) {
- mTotalUsedMemory += requirements.size;
+ mUsedMemory.Increment(requirements.size);
+ mLazyUsedMemory.Increment(isLazyMemoryType ? requirements.size : 0);
return subAllocation;
}
}
@@ -257,16 +289,20 @@
{ mAllocatorsPerType[memoryType]->DeallocateResourceHeap(std::move(resourceHeap)); });
}
- mTotalUsedMemory += size;
+ mUsedMemory.Increment(size);
+ mLazyUsedMemory.Increment(isLazyMemoryType ? size : 0);
+
AllocationInfo info;
info.mMethod = AllocationMethod::kDirect;
info.mRequestedSize = size;
+ info.mIsLazyAllocated = isLazyMemoryType;
return ResourceMemoryAllocation(info, /*offset*/ 0, resourceHeap.release(),
static_cast<uint8_t*>(mappedPointer));
}
void ResourceMemoryAllocator::Deallocate(ResourceMemoryAllocation* allocation) {
- switch (allocation->GetInfo().mMethod) {
+ AllocationInfo info = allocation->GetInfo();
+ switch (info.mMethod) {
// Some memory allocation can never be initialized, for example when wrapping
// swapchain VkImages with a Texture.
case AllocationMethod::kInvalid:
@@ -276,13 +312,14 @@
// deleter will make sure the resources are freed before the memory.
case AllocationMethod::kDirect: {
ResourceHeap* heap = ToBackend(allocation->GetResourceHeap());
- // Track the direct allocation that will be deallocated for both allocated and used
- // memory sizes.
- DAWN_ASSERT(mTotalUsedMemory >= allocation->GetInfo().mRequestedSize);
- mUsedMemoryToDecrement[mDevice->GetFencedDeleter()->GetCurrentDeletionSerial()] +=
- allocation->GetInfo().mRequestedSize;
+ auto currentDeletionSerial = mDevice->GetFencedDeleter()->GetCurrentDeletionSerial();
+ // Track the direct allocation that will be deallocated used memory sizes.
+ mUsedMemory.Decrement(currentDeletionSerial, info.mRequestedSize);
+ if (info.mIsLazyAllocated) {
+ mLazyUsedMemory.Decrement(currentDeletionSerial, info.mRequestedSize);
+ }
allocation->Invalidate();
- DeallocateResourceHeap(heap);
+ DeallocateResourceHeap(heap, info.mIsLazyAllocated);
delete heap;
break;
}
@@ -296,8 +333,10 @@
mDevice->GetFencedDeleter()->GetCurrentDeletionSerial();
mSubAllocationsToDelete.Enqueue(*allocation, deletionSerial);
// Track suballocation that will be deallocated for used memory sizes.
- DAWN_ASSERT(mTotalUsedMemory >= allocation->GetInfo().mRequestedSize);
- mUsedMemoryToDecrement[deletionSerial] += allocation->GetInfo().mRequestedSize;
+ mUsedMemory.Decrement(deletionSerial, info.mRequestedSize);
+ if (info.mIsLazyAllocated) {
+ mLazyUsedMemory.Decrement(deletionSerial, info.mRequestedSize);
+ }
break;
}
@@ -322,14 +361,21 @@
return lastSerial;
}
-void ResourceMemoryAllocator::RecordHeapAllocation(VkDeviceSize size) {
- mTotalAllocatedMemory += size;
+void ResourceMemoryAllocator::RecordHeapAllocation(VkDeviceSize size, bool isLazyMemoryType) {
+ mAllocatedMemory.Increment(size);
+ mLazyAllocatedMemory.Increment(isLazyMemoryType ? size : 0);
}
-void ResourceMemoryAllocator::DeallocateResourceHeap(ResourceHeap* heap) {
- DAWN_ASSERT(mTotalAllocatedMemory >= heap->GetSize());
+void ResourceMemoryAllocator::DeallocateResourceHeap(ResourceHeap* heap, bool isLazyMemoryType) {
+ VkDeviceSize heapSize = heap->GetSize();
MutexProtected<FencedDeleter>& fencedDeleter = mDevice->GetFencedDeleter();
- mAllocatedMemoryToDecrement[fencedDeleter->GetCurrentDeletionSerial()] += heap->GetSize();
+ auto currentDeletionSerial = fencedDeleter->GetCurrentDeletionSerial();
+
+ // Track heap that will be deallocated for allocated memory sizes.
+ mAllocatedMemory.Decrement(currentDeletionSerial, heapSize);
+ if (isLazyMemoryType) {
+ mLazyAllocatedMemory.Decrement(currentDeletionSerial, heapSize);
+ }
fencedDeleter->DeleteWhenUnused(heap->GetMemory());
}
@@ -342,25 +388,11 @@
}
mSubAllocationsToDelete.ClearUpTo(completedSerial);
- auto it = mUsedMemoryToDecrement.begin();
- while (it != mUsedMemoryToDecrement.end() && it->first <= completedSerial) {
- // Track the direct allocation memory as used memory that will be deallocated.
- DAWN_ASSERT(mTotalUsedMemory >= it->second);
- mTotalUsedMemory -= it->second;
- it++;
- }
- // Erase the map serials up to the completed serial.
- mUsedMemoryToDecrement.erase(mUsedMemoryToDecrement.begin(), it);
-
- it = mAllocatedMemoryToDecrement.begin();
- while (it != mAllocatedMemoryToDecrement.end() && it->first <= completedSerial) {
- // Track the direct allocation memory as used memory that will be deallocated.
- DAWN_ASSERT(mTotalAllocatedMemory >= it->second);
- mTotalAllocatedMemory -= it->second;
- it++;
- }
- // Erase the map serials up to the completed serial.
- mAllocatedMemoryToDecrement.erase(mAllocatedMemoryToDecrement.begin(), it);
+ // Update the allocation sizes after completed serials.
+ mAllocatedMemory.Tick(completedSerial);
+ mUsedMemory.Tick(completedSerial);
+ mLazyAllocatedMemory.Tick(completedSerial);
+ mLazyUsedMemory.Tick(completedSerial);
}
int ResourceMemoryAllocator::FindBestTypeIndex(VkMemoryRequirements requirements, MemoryKind kind) {
@@ -447,11 +479,19 @@
}
uint64_t ResourceMemoryAllocator::GetTotalUsedMemory() const {
- return mTotalUsedMemory;
+ return mUsedMemory.Size();
}
uint64_t ResourceMemoryAllocator::GetTotalAllocatedMemory() const {
- return mTotalAllocatedMemory;
+ return mAllocatedMemory.Size();
+}
+
+uint64_t ResourceMemoryAllocator::GetTotalLazyAllocatedMemory() const {
+ return mLazyAllocatedMemory.Size();
+}
+
+uint64_t ResourceMemoryAllocator::GetTotalLazyUsedMemory() const {
+ return mLazyUsedMemory.Size();
}
} // namespace dawn::native::vulkan
diff --git a/src/dawn/native/vulkan/ResourceMemoryAllocatorVk.h b/src/dawn/native/vulkan/ResourceMemoryAllocatorVk.h
index d80087e..691f16f 100644
--- a/src/dawn/native/vulkan/ResourceMemoryAllocatorVk.h
+++ b/src/dawn/native/vulkan/ResourceMemoryAllocatorVk.h
@@ -83,14 +83,36 @@
int FindBestTypeIndex(VkMemoryRequirements requirements, MemoryKind kind);
+ // Reports the total vulkan allocated and vulkan used memories.
uint64_t GetTotalUsedMemory() const;
uint64_t GetTotalAllocatedMemory() const;
+ // Reports the total lazy allocated and used vulkan memory.
+ uint64_t GetTotalLazyAllocatedMemory() const;
+ uint64_t GetTotalLazyUsedMemory() const;
protected:
- void RecordHeapAllocation(VkDeviceSize size);
- void DeallocateResourceHeap(ResourceHeap* heap);
+ void RecordHeapAllocation(VkDeviceSize size, bool isLazyMemoryType);
+ void DeallocateResourceHeap(ResourceHeap* heap, bool isLazyMemoryType);
private:
+ // Wrapper for tracking the allocation sizes to be decremented up to a completed ExecutionSerial
+ // and reporting total allocation/used sizes.
+ class AllocationSizeTracker {
+ public:
+ // Increment the total size for tracking.
+ void Increment(VkDeviceSize incrementSize);
+ // Track the size to be decremented on Tick.
+ void Decrement(ExecutionSerial currentSerial, VkDeviceSize decrementSize);
+ // Update the total size after completed serials.
+ void Tick(ExecutionSerial completedSerial);
+
+ VkDeviceSize Size() const { return mTotalSize; }
+
+ private:
+ std::map<ExecutionSerial, VkDeviceSize> mMemoryToDecrement;
+ VkDeviceSize mTotalSize = 0;
+ };
+
raw_ptr<Device> mDevice;
const VkDeviceSize mMaxSizeForSuballocation;
@@ -98,11 +120,10 @@
std::vector<std::unique_ptr<SingleTypeAllocator>> mAllocatorsPerType;
SerialQueue<ExecutionSerial, ResourceMemoryAllocation> mSubAllocationsToDelete;
- std::map<ExecutionSerial, VkDeviceSize> mUsedMemoryToDecrement;
- std::map<ExecutionSerial, VkDeviceSize> mAllocatedMemoryToDecrement;
-
- VkDeviceSize mTotalAllocatedMemory = 0;
- VkDeviceSize mTotalUsedMemory = 0;
+ AllocationSizeTracker mAllocatedMemory;
+ AllocationSizeTracker mUsedMemory;
+ AllocationSizeTracker mLazyAllocatedMemory;
+ AllocationSizeTracker mLazyUsedMemory;
};
} // namespace dawn::native::vulkan
diff --git a/src/dawn/tests/unittests/BuddyMemoryAllocatorTests.cpp b/src/dawn/tests/unittests/BuddyMemoryAllocatorTests.cpp
index 5fb0f49..617d774 100644
--- a/src/dawn/tests/unittests/BuddyMemoryAllocatorTests.cpp
+++ b/src/dawn/tests/unittests/BuddyMemoryAllocatorTests.cpp
@@ -57,7 +57,7 @@
ResourceMemoryAllocation Allocate(uint64_t allocationSize, uint64_t alignment = 1) {
ResultOrError<ResourceMemoryAllocation> result =
- mAllocator.Allocate(allocationSize, alignment);
+ mAllocator.Allocate(allocationSize, alignment, /*isLazyMemoryType=*/false);
return (result.IsSuccess()) ? result.AcquireSuccess() : ResourceMemoryAllocation{};
}
