| // Copyright 2019 The Dawn Authors |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| #include "dawn/native/d3d12/ResourceAllocatorManagerD3D12.h" |
| |
| #include "dawn/native/d3d12/D3D12Error.h" |
| #include "dawn/native/d3d12/DeviceD3D12.h" |
| #include "dawn/native/d3d12/HeapAllocatorD3D12.h" |
| #include "dawn/native/d3d12/HeapD3D12.h" |
| #include "dawn/native/d3d12/ResidencyManagerD3D12.h" |
| #include "dawn/native/d3d12/UtilsD3D12.h" |
| |
| namespace dawn::native::d3d12 { |
| namespace { |
| MemorySegment GetMemorySegment(Device* device, D3D12_HEAP_TYPE heapType) { |
| if (device->GetDeviceInfo().isUMA) { |
| return MemorySegment::Local; |
| } |
| |
| D3D12_HEAP_PROPERTIES heapProperties = |
| device->GetD3D12Device()->GetCustomHeapProperties(0, heapType); |
| |
| if (heapProperties.MemoryPoolPreference == D3D12_MEMORY_POOL_L1) { |
| return MemorySegment::Local; |
| } |
| |
| return MemorySegment::NonLocal; |
| } |
| |
| D3D12_HEAP_TYPE GetD3D12HeapType(ResourceHeapKind resourceHeapKind) { |
| switch (resourceHeapKind) { |
| case Readback_OnlyBuffers: |
| case Readback_AllBuffersAndTextures: |
| return D3D12_HEAP_TYPE_READBACK; |
| case Default_AllBuffersAndTextures: |
| case Default_OnlyBuffers: |
| case Default_OnlyNonRenderableOrDepthTextures: |
| case Default_OnlyRenderableOrDepthTextures: |
| return D3D12_HEAP_TYPE_DEFAULT; |
| case Upload_OnlyBuffers: |
| case Upload_AllBuffersAndTextures: |
| return D3D12_HEAP_TYPE_UPLOAD; |
| case EnumCount: |
| UNREACHABLE(); |
| } |
| } |
| |
| D3D12_HEAP_FLAGS GetD3D12HeapFlags(ResourceHeapKind resourceHeapKind) { |
| switch (resourceHeapKind) { |
| case Default_AllBuffersAndTextures: |
| case Readback_AllBuffersAndTextures: |
| case Upload_AllBuffersAndTextures: |
| return D3D12_HEAP_FLAG_ALLOW_ALL_BUFFERS_AND_TEXTURES; |
| case Default_OnlyBuffers: |
| case Readback_OnlyBuffers: |
| case Upload_OnlyBuffers: |
| return D3D12_HEAP_FLAG_ALLOW_ONLY_BUFFERS; |
| case Default_OnlyNonRenderableOrDepthTextures: |
| return D3D12_HEAP_FLAG_ALLOW_ONLY_NON_RT_DS_TEXTURES; |
| case Default_OnlyRenderableOrDepthTextures: |
| return D3D12_HEAP_FLAG_ALLOW_ONLY_RT_DS_TEXTURES; |
| case EnumCount: |
| UNREACHABLE(); |
| } |
| } |
| |
| ResourceHeapKind GetResourceHeapKind(D3D12_RESOURCE_DIMENSION dimension, |
| D3D12_HEAP_TYPE heapType, |
| D3D12_RESOURCE_FLAGS flags, |
| uint32_t resourceHeapTier) { |
| if (resourceHeapTier >= 2) { |
| switch (heapType) { |
| case D3D12_HEAP_TYPE_UPLOAD: |
| return Upload_AllBuffersAndTextures; |
| case D3D12_HEAP_TYPE_DEFAULT: |
| return Default_AllBuffersAndTextures; |
| case D3D12_HEAP_TYPE_READBACK: |
| return Readback_AllBuffersAndTextures; |
| default: |
| UNREACHABLE(); |
| } |
| } |
| |
| switch (dimension) { |
| case D3D12_RESOURCE_DIMENSION_BUFFER: { |
| switch (heapType) { |
| case D3D12_HEAP_TYPE_UPLOAD: |
| return Upload_OnlyBuffers; |
| case D3D12_HEAP_TYPE_DEFAULT: |
| return Default_OnlyBuffers; |
| case D3D12_HEAP_TYPE_READBACK: |
| return Readback_OnlyBuffers; |
| default: |
| UNREACHABLE(); |
| } |
| break; |
| } |
| case D3D12_RESOURCE_DIMENSION_TEXTURE1D: |
| case D3D12_RESOURCE_DIMENSION_TEXTURE2D: |
| case D3D12_RESOURCE_DIMENSION_TEXTURE3D: { |
| switch (heapType) { |
| case D3D12_HEAP_TYPE_DEFAULT: { |
| if ((flags & D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL) || |
| (flags & D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET)) { |
| return Default_OnlyRenderableOrDepthTextures; |
| } |
| return Default_OnlyNonRenderableOrDepthTextures; |
| } |
| |
| default: |
| UNREACHABLE(); |
| } |
| break; |
| } |
| default: |
| UNREACHABLE(); |
| } |
| } |
| |
| uint64_t GetResourcePlacementAlignment(ResourceHeapKind resourceHeapKind, |
| uint32_t sampleCount, |
| uint64_t requestedAlignment) { |
| switch (resourceHeapKind) { |
| // Small resources can take advantage of smaller alignments. For example, |
| // if the most detailed mip can fit under 64KB, 4KB alignments can be used. |
| // Must be non-depth or without render-target to use small resource alignment. |
| // This also applies to MSAA textures (4MB => 64KB). |
| // |
| // Note: Only known to be used for small textures; however, MSDN suggests |
| // it could be extended for more cases. If so, this could default to always |
| // attempt small resource placement. |
| // https://docs.microsoft.com/en-us/windows/win32/api/d3d12/ns-d3d12-d3d12_resource_desc |
| case Default_OnlyNonRenderableOrDepthTextures: |
| return (sampleCount > 1) ? D3D12_SMALL_MSAA_RESOURCE_PLACEMENT_ALIGNMENT |
| : D3D12_SMALL_RESOURCE_PLACEMENT_ALIGNMENT; |
| default: |
| return requestedAlignment; |
| } |
| } |
| |
| bool IsClearValueOptimizable(const D3D12_RESOURCE_DESC& resourceDescriptor) { |
| // Optimized clear color cannot be set on buffers, non-render-target/depth-stencil |
| // textures, or typeless resources |
| // https://docs.microsoft.com/en-us/windows/win32/api/d3d12/nf-d3d12-id3d12device-createcommittedresource |
| // https://docs.microsoft.com/en-us/windows/win32/api/d3d12/nf-d3d12-id3d12device-createplacedresource |
| return !IsTypeless(resourceDescriptor.Format) && |
| resourceDescriptor.Dimension != D3D12_RESOURCE_DIMENSION_BUFFER && |
| (resourceDescriptor.Flags & (D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET | |
| D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL)) != 0; |
| } |
| |
| } // namespace |
| |
| ResourceAllocatorManager::ResourceAllocatorManager(Device* device) : mDevice(device) { |
| mResourceHeapTier = (mDevice->IsToggleEnabled(Toggle::UseD3D12ResourceHeapTier2)) |
| ? mDevice->GetDeviceInfo().resourceHeapTier |
| : 1; |
| |
| for (uint32_t i = 0; i < ResourceHeapKind::EnumCount; i++) { |
| const ResourceHeapKind resourceHeapKind = static_cast<ResourceHeapKind>(i); |
| mHeapAllocators[i] = std::make_unique<HeapAllocator>( |
| mDevice, GetD3D12HeapType(resourceHeapKind), GetD3D12HeapFlags(resourceHeapKind), |
| GetMemorySegment(device, GetD3D12HeapType(resourceHeapKind))); |
| mPooledHeapAllocators[i] = |
| std::make_unique<PooledResourceMemoryAllocator>(mHeapAllocators[i].get()); |
| mSubAllocatedResourceAllocators[i] = std::make_unique<BuddyMemoryAllocator>( |
| kMaxHeapSize, kMinHeapSize, mPooledHeapAllocators[i].get()); |
| } |
| } |
| |
| ResultOrError<ResourceHeapAllocation> ResourceAllocatorManager::AllocateMemory( |
| D3D12_HEAP_TYPE heapType, |
| const D3D12_RESOURCE_DESC& resourceDescriptor, |
| D3D12_RESOURCE_STATES initialUsage) { |
| // In order to suppress a warning in the D3D12 debug layer, we need to specify an |
| // optimized clear value. As there are no negative consequences when picking a mismatched |
| // clear value, we use zero as the optimized clear value. This also enables fast clears on |
| // some architectures. |
| D3D12_CLEAR_VALUE zero{}; |
| D3D12_CLEAR_VALUE* optimizedClearValue = nullptr; |
| if (IsClearValueOptimizable(resourceDescriptor)) { |
| zero.Format = resourceDescriptor.Format; |
| optimizedClearValue = &zero; |
| } |
| |
| // TODO(crbug.com/dawn/849): Conditionally disable sub-allocation. |
| // For very large resources, there is no benefit to suballocate. |
| // For very small resources, it is inefficent to suballocate given the min. heap |
| // size could be much larger then the resource allocation. |
| // Attempt to satisfy the request using sub-allocation (placed resource in a heap). |
| ResourceHeapAllocation subAllocation; |
| DAWN_TRY_ASSIGN(subAllocation, CreatePlacedResource(heapType, resourceDescriptor, |
| optimizedClearValue, initialUsage)); |
| if (subAllocation.GetInfo().mMethod != AllocationMethod::kInvalid) { |
| return std::move(subAllocation); |
| } |
| |
| // If sub-allocation fails, fall-back to direct allocation (committed resource). |
| ResourceHeapAllocation directAllocation; |
| DAWN_TRY_ASSIGN(directAllocation, |
| CreateCommittedResource(heapType, resourceDescriptor, optimizedClearValue, |
| initialUsage)); |
| if (directAllocation.GetInfo().mMethod != AllocationMethod::kInvalid) { |
| return std::move(directAllocation); |
| } |
| |
| // If direct allocation fails, the system is probably out of memory. |
| return DAWN_OUT_OF_MEMORY_ERROR("Allocation failed"); |
| } |
| |
| void ResourceAllocatorManager::Tick(ExecutionSerial completedSerial) { |
| for (ResourceHeapAllocation& allocation : |
| mAllocationsToDelete.IterateUpTo(completedSerial)) { |
| if (allocation.GetInfo().mMethod == AllocationMethod::kSubAllocated) { |
| FreeMemory(allocation); |
| } |
| } |
| mAllocationsToDelete.ClearUpTo(completedSerial); |
| } |
| |
| void ResourceAllocatorManager::DeallocateMemory(ResourceHeapAllocation& allocation) { |
| if (allocation.GetInfo().mMethod == AllocationMethod::kInvalid) { |
| return; |
| } |
| |
| mAllocationsToDelete.Enqueue(allocation, mDevice->GetPendingCommandSerial()); |
| |
| // Directly allocated ResourceHeapAllocations are created with a heap object that must be |
| // manually deleted upon deallocation. See ResourceAllocatorManager::CreateCommittedResource |
| // for more information. |
| if (allocation.GetInfo().mMethod == AllocationMethod::kDirect) { |
| delete allocation.GetResourceHeap(); |
| } |
| |
| // Invalidate the allocation immediately in case one accidentally |
| // calls DeallocateMemory again using the same allocation. |
| allocation.Invalidate(); |
| |
| ASSERT(allocation.GetD3D12Resource() == nullptr); |
| } |
| |
| void ResourceAllocatorManager::FreeMemory(ResourceHeapAllocation& allocation) { |
| ASSERT(allocation.GetInfo().mMethod == AllocationMethod::kSubAllocated); |
| |
| D3D12_HEAP_PROPERTIES heapProp; |
| allocation.GetD3D12Resource()->GetHeapProperties(&heapProp, nullptr); |
| |
| const D3D12_RESOURCE_DESC resourceDescriptor = allocation.GetD3D12Resource()->GetDesc(); |
| |
| const size_t resourceHeapKindIndex = |
| GetResourceHeapKind(resourceDescriptor.Dimension, heapProp.Type, |
| resourceDescriptor.Flags, mResourceHeapTier); |
| |
| mSubAllocatedResourceAllocators[resourceHeapKindIndex]->Deallocate(allocation); |
| } |
| |
| ResultOrError<ResourceHeapAllocation> ResourceAllocatorManager::CreatePlacedResource( |
| D3D12_HEAP_TYPE heapType, |
| const D3D12_RESOURCE_DESC& requestedResourceDescriptor, |
| const D3D12_CLEAR_VALUE* optimizedClearValue, |
| D3D12_RESOURCE_STATES initialUsage) { |
| const ResourceHeapKind resourceHeapKind = |
| GetResourceHeapKind(requestedResourceDescriptor.Dimension, heapType, |
| requestedResourceDescriptor.Flags, mResourceHeapTier); |
| |
| D3D12_RESOURCE_DESC resourceDescriptor = requestedResourceDescriptor; |
| resourceDescriptor.Alignment = GetResourcePlacementAlignment( |
| resourceHeapKind, requestedResourceDescriptor.SampleDesc.Count, |
| requestedResourceDescriptor.Alignment); |
| |
| // TODO(bryan.bernhart): Figure out how to compute the alignment without calling this |
| // twice. |
| D3D12_RESOURCE_ALLOCATION_INFO resourceInfo = |
| mDevice->GetD3D12Device()->GetResourceAllocationInfo(0, 1, &resourceDescriptor); |
| |
| // If the requested resource alignment was rejected, let D3D tell us what the |
| // required alignment is for this resource. |
| if (resourceDescriptor.Alignment != 0 && |
| resourceDescriptor.Alignment != resourceInfo.Alignment) { |
| resourceDescriptor.Alignment = 0; |
| resourceInfo = |
| mDevice->GetD3D12Device()->GetResourceAllocationInfo(0, 1, &resourceDescriptor); |
| } |
| |
| // If d3d tells us the resource size is invalid, treat the error as OOM. |
| // Otherwise, creating the resource could cause a device loss (too large). |
| // This is because NextPowerOfTwo(UINT64_MAX) overflows and proceeds to |
| // incorrectly allocate a mismatched size. |
| if (resourceInfo.SizeInBytes == 0 || |
| resourceInfo.SizeInBytes == std::numeric_limits<uint64_t>::max()) { |
| return DAWN_OUT_OF_MEMORY_ERROR(absl::StrFormat( |
| "Resource allocation size (%u) was invalid.", resourceInfo.SizeInBytes)); |
| } |
| |
| BuddyMemoryAllocator* allocator = |
| mSubAllocatedResourceAllocators[static_cast<size_t>(resourceHeapKind)].get(); |
| |
| ResourceMemoryAllocation allocation; |
| DAWN_TRY_ASSIGN(allocation, |
| allocator->Allocate(resourceInfo.SizeInBytes, resourceInfo.Alignment)); |
| if (allocation.GetInfo().mMethod == AllocationMethod::kInvalid) { |
| return ResourceHeapAllocation{}; // invalid |
| } |
| |
| Heap* heap = ToBackend(allocation.GetResourceHeap()); |
| |
| // Before calling CreatePlacedResource, we must ensure the target heap is resident. |
| // CreatePlacedResource will fail if it is not. |
| DAWN_TRY(mDevice->GetResidencyManager()->LockAllocation(heap)); |
| |
| // With placed resources, a single heap can be reused. |
| // The resource placed at an offset is only reclaimed |
| // upon Tick or after the last command list using the resource has completed |
| // on the GPU. This means the same physical memory is not reused |
| // within the same command-list and does not require additional synchronization (aliasing |
| // barrier). |
| // https://docs.microsoft.com/en-us/windows/win32/api/d3d12/nf-d3d12-id3d12device-createplacedresource |
| ComPtr<ID3D12Resource> placedResource; |
| DAWN_TRY(CheckOutOfMemoryHRESULT( |
| mDevice->GetD3D12Device()->CreatePlacedResource( |
| heap->GetD3D12Heap(), allocation.GetOffset(), &resourceDescriptor, initialUsage, |
| optimizedClearValue, IID_PPV_ARGS(&placedResource)), |
| "ID3D12Device::CreatePlacedResource")); |
| |
| // After CreatePlacedResource has finished, the heap can be unlocked from residency. This |
| // will insert it into the residency LRU. |
| mDevice->GetResidencyManager()->UnlockAllocation(heap); |
| |
| return ResourceHeapAllocation{allocation.GetInfo(), allocation.GetOffset(), |
| std::move(placedResource), heap}; |
| } |
| |
| ResultOrError<ResourceHeapAllocation> ResourceAllocatorManager::CreateCommittedResource( |
| D3D12_HEAP_TYPE heapType, |
| const D3D12_RESOURCE_DESC& resourceDescriptor, |
| const D3D12_CLEAR_VALUE* optimizedClearValue, |
| D3D12_RESOURCE_STATES initialUsage) { |
| D3D12_HEAP_PROPERTIES heapProperties; |
| heapProperties.Type = heapType; |
| heapProperties.CPUPageProperty = D3D12_CPU_PAGE_PROPERTY_UNKNOWN; |
| heapProperties.MemoryPoolPreference = D3D12_MEMORY_POOL_UNKNOWN; |
| heapProperties.CreationNodeMask = 0; |
| heapProperties.VisibleNodeMask = 0; |
| |
| // If d3d tells us the resource size is invalid, treat the error as OOM. |
| // Otherwise, creating the resource could cause a device loss (too large). |
| // This is because NextPowerOfTwo(UINT64_MAX) overflows and proceeds to |
| // incorrectly allocate a mismatched size. |
| D3D12_RESOURCE_ALLOCATION_INFO resourceInfo = |
| mDevice->GetD3D12Device()->GetResourceAllocationInfo(0, 1, &resourceDescriptor); |
| if (resourceInfo.SizeInBytes == 0 || |
| resourceInfo.SizeInBytes == std::numeric_limits<uint64_t>::max()) { |
| return DAWN_OUT_OF_MEMORY_ERROR("Resource allocation size was invalid."); |
| } |
| |
| if (resourceInfo.SizeInBytes > kMaxHeapSize) { |
| return ResourceHeapAllocation{}; // Invalid |
| } |
| |
| // CreateCommittedResource will implicitly make the created resource resident. We must |
| // ensure enough free memory exists before allocating to avoid an out-of-memory error when |
| // overcommitted. |
| DAWN_TRY(mDevice->GetResidencyManager()->EnsureCanAllocate( |
| resourceInfo.SizeInBytes, GetMemorySegment(mDevice, heapType))); |
| |
| // Note: Heap flags are inferred by the resource descriptor and do not need to be explicitly |
| // provided to CreateCommittedResource. |
| ComPtr<ID3D12Resource> committedResource; |
| DAWN_TRY(CheckOutOfMemoryHRESULT( |
| mDevice->GetD3D12Device()->CreateCommittedResource( |
| &heapProperties, D3D12_HEAP_FLAG_NONE, &resourceDescriptor, initialUsage, |
| optimizedClearValue, IID_PPV_ARGS(&committedResource)), |
| "ID3D12Device::CreateCommittedResource")); |
| |
| // When using CreateCommittedResource, D3D12 creates an implicit heap that contains the |
| // resource allocation. Because Dawn's memory residency management occurs at the resource |
| // heap granularity, every directly allocated ResourceHeapAllocation also stores a Heap |
| // object. This object is created manually, and must be deleted manually upon deallocation |
| // of the committed resource. |
| Heap* heap = new Heap(committedResource, GetMemorySegment(mDevice, heapType), |
| resourceInfo.SizeInBytes); |
| |
| // Calling CreateCommittedResource implicitly calls MakeResident on the resource. We must |
| // track this to avoid calling MakeResident a second time. |
| mDevice->GetResidencyManager()->TrackResidentAllocation(heap); |
| |
| AllocationInfo info; |
| info.mMethod = AllocationMethod::kDirect; |
| |
| return ResourceHeapAllocation{info, |
| /*offset*/ 0, std::move(committedResource), heap}; |
| } |
| |
| void ResourceAllocatorManager::DestroyPool() { |
| for (auto& alloc : mPooledHeapAllocators) { |
| alloc->DestroyPool(); |
| } |
| } |
| |
| } // namespace dawn::native::d3d12 |