blob: 237ee54a1c39d943e3c669337320e549a502800e [file] [log] [blame]
// Copyright 2020 The Dawn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "dawn_native/d3d12/ResidencyManagerD3D12.h"
#include "dawn_native/d3d12/AdapterD3D12.h"
#include "dawn_native/d3d12/D3D12Error.h"
#include "dawn_native/d3d12/DeviceD3D12.h"
#include "dawn_native/d3d12/Forward.h"
#include "dawn_native/d3d12/HeapD3D12.h"
#include "dawn_native/d3d12/d3d12_platform.h"
namespace dawn_native { namespace d3d12 {
ResidencyManager::ResidencyManager(Device* device)
: mDevice(device),
mResidencyManagementEnabled(
device->IsToggleEnabled(Toggle::UseD3D12ResidencyManagement)) {
UpdateVideoMemoryInfo();
}
// Increments number of locks on a heap to ensure the heap remains resident.
MaybeError ResidencyManager::LockHeap(Heap* heap) {
if (!mResidencyManagementEnabled) {
return {};
}
// Depending on device architecture, the heap may not need tracked.
if (!ShouldTrackHeap(heap)) {
return {};
}
// If the heap isn't already resident, make it resident.
if (!heap->IsInResidencyLRUCache() && !heap->IsResidencyLocked()) {
DAWN_TRY(EnsureCanMakeResident(heap->GetSize()));
ID3D12Pageable* pageable = heap->GetD3D12Pageable().Get();
DAWN_TRY(CheckHRESULT(mDevice->GetD3D12Device()->MakeResident(1, &pageable),
"Making a scheduled-to-be-used resource resident in "
"device local memory"));
}
// Since we can't evict the heap, it's unnecessary to track the heap in the LRU Cache.
if (heap->IsInResidencyLRUCache()) {
heap->RemoveFromList();
}
heap->IncrementResidencyLock();
return {};
}
// Decrements number of locks on a heap. When the number of locks becomes zero, the heap is
// inserted into the LRU cache and becomes eligible for eviction.
void ResidencyManager::UnlockHeap(Heap* heap) {
if (!mResidencyManagementEnabled) {
return;
}
// Depending on device architecture, the heap may not need tracked.
if (!ShouldTrackHeap(heap)) {
return;
}
ASSERT(heap->IsResidencyLocked());
ASSERT(!heap->IsInResidencyLRUCache());
heap->DecrementResidencyLock();
// When all locks have been removed, the resource remains resident and becomes tracked in
// the LRU.
if (!heap->IsResidencyLocked()) {
mLRUCache.Append(heap);
}
}
// Allows an application component external to Dawn to cap Dawn's residency budget to prevent
// competition for device local memory. Returns the amount of memory reserved, which may be less
// that the requested reservation when under pressure.
uint64_t ResidencyManager::SetExternalMemoryReservation(uint64_t requestedReservationSize) {
mVideoMemoryInfo.externalRequest = requestedReservationSize;
UpdateVideoMemoryInfo();
return mVideoMemoryInfo.externalReservation;
}
void ResidencyManager::UpdateVideoMemoryInfo() {
if (!mResidencyManagementEnabled) {
return;
}
DXGI_QUERY_VIDEO_MEMORY_INFO queryVideoMemoryInfo;
ToBackend(mDevice->GetAdapter())
->GetHardwareAdapter()
->QueryVideoMemoryInfo(0, DXGI_MEMORY_SEGMENT_GROUP_LOCAL, &queryVideoMemoryInfo);
// The video memory budget provided by QueryVideoMemoryInfo is defined by the operating
// system, and may be lower than expected in certain scenarios. Under memory pressure, we
// cap the external reservation to half the available budget, which prevents the external
// component from consuming a disproportionate share of memory and ensures that Dawn can
// continue to make forward progress. Note the choice to halve memory is arbitrarily chosen
// and subject to future experimentation.
mVideoMemoryInfo.externalReservation =
std::min(queryVideoMemoryInfo.Budget / 2, mVideoMemoryInfo.externalRequest);
mVideoMemoryInfo.dawnUsage =
queryVideoMemoryInfo.CurrentUsage - mVideoMemoryInfo.externalReservation;
// If we're restricting the budget for testing, leave the budget as is.
if (mRestrictBudgetForTesting) {
return;
}
// We cap Dawn's budget to 95% of the provided budget. Leaving some budget unused
// decreases fluctuations in the operating-system-defined budget, which improves stability
// for both Dawn and other applications on the system. Note the value of 95% is arbitrarily
// chosen and subject to future experimentation.
static constexpr float kBudgetCap = 0.95;
mVideoMemoryInfo.dawnBudget =
(queryVideoMemoryInfo.Budget - mVideoMemoryInfo.externalReservation) * kBudgetCap;
}
// Removes from the LRU and returns the least recently used heap when possible. Returns nullptr
// when nothing further can be evicted.
ResultOrError<Heap*> ResidencyManager::RemoveSingleEntryFromLRU() {
ASSERT(!mLRUCache.empty());
Heap* heap = mLRUCache.head()->value();
Serial lastSubmissionSerial = heap->GetLastSubmission();
// If the next candidate for eviction was inserted into the LRU during the current serial,
// it is because more memory is being used in a single command list than is available.
// In this scenario, we cannot make any more resources resident and thrashing must occur.
if (lastSubmissionSerial == mDevice->GetPendingCommandSerial()) {
return nullptr;
}
// We must ensure that any previous use of a resource has completed before the resource can
// be evicted.
if (lastSubmissionSerial > mDevice->GetCompletedCommandSerial()) {
DAWN_TRY(mDevice->WaitForSerial(lastSubmissionSerial));
}
heap->RemoveFromList();
return heap;
}
// Any time we need to make something resident in local memory, we must check that we have
// enough free memory to make the new object resident while also staying within our budget.
// If there isn't enough memory, we should evict until there is.
MaybeError ResidencyManager::EnsureCanMakeResident(uint64_t sizeToMakeResident) {
if (!mResidencyManagementEnabled) {
return {};
}
UpdateVideoMemoryInfo();
uint64_t memoryUsageAfterMakeResident = sizeToMakeResident + mVideoMemoryInfo.dawnUsage;
// Return when we can call MakeResident and remain under budget.
if (memoryUsageAfterMakeResident < mVideoMemoryInfo.dawnBudget) {
return {};
}
std::vector<ID3D12Pageable*> resourcesToEvict;
uint64_t sizeNeededToBeUnderBudget =
memoryUsageAfterMakeResident - mVideoMemoryInfo.dawnBudget;
uint64_t sizeEvicted = 0;
while (sizeEvicted < sizeNeededToBeUnderBudget) {
Heap* heap;
DAWN_TRY_ASSIGN(heap, RemoveSingleEntryFromLRU());
// If no heap was returned, then nothing more can be evicted.
if (heap == nullptr) {
break;
}
sizeEvicted += heap->GetSize();
resourcesToEvict.push_back(heap->GetD3D12Pageable().Get());
}
if (resourcesToEvict.size() > 0) {
DAWN_TRY(CheckHRESULT(
mDevice->GetD3D12Device()->Evict(resourcesToEvict.size(), resourcesToEvict.data()),
"Evicting resident heaps to free device local memory"));
}
return {};
}
// Ensure that we are only tracking heaps that exist in DXGI_MEMORY_SEGMENT_LOCAL.
bool ResidencyManager::ShouldTrackHeap(Heap* heap) const {
D3D12_HEAP_PROPERTIES heapProperties =
mDevice->GetD3D12Device()->GetCustomHeapProperties(0, heap->GetD3D12HeapType());
if (mDevice->GetDeviceInfo().isUMA) {
// On UMA devices, MEMORY_POOL_L0 corresponds to MEMORY_SEGMENT_LOCAL, so we must track
// heaps in MEMORY_POOL_L0. For UMA, all heaps types exist in MEMORY_POOL_L0.
return heapProperties.MemoryPoolPreference == D3D12_MEMORY_POOL_L0;
}
// On non-UMA devices, MEMORY_POOL_L1 corresponds to MEMORY_SEGMENT_LOCAL, so only track the
// heap if it is in MEMORY_POOL_L1. For non-UMA, DEFAULT heaps exist in MEMORY_POOL_L1,
// while READBACK and UPLOAD heaps exist in MEMORY_POOL_L0.
return heapProperties.MemoryPoolPreference == D3D12_MEMORY_POOL_L1;
}
// Given a list of heaps that are pending usage, this function will estimate memory needed,
// evict resources until enough space is available, then make resident any heaps scheduled for
// usage.
MaybeError ResidencyManager::EnsureHeapsAreResident(Heap** heaps, size_t heapCount) {
if (!mResidencyManagementEnabled) {
return {};
}
std::vector<ID3D12Pageable*> heapsToMakeResident;
uint64_t sizeToMakeResident = 0;
Serial pendingCommandSerial = mDevice->GetPendingCommandSerial();
for (size_t i = 0; i < heapCount; i++) {
Heap* heap = heaps[i];
// Depending on device architecture, the heap may not need tracked.
if (!ShouldTrackHeap(heap)) {
continue;
}
// Heaps that are locked resident are not tracked in the LRU cache.
if (heap->IsResidencyLocked()) {
continue;
}
if (heap->IsInResidencyLRUCache()) {
// If the heap is already in the LRU, we must remove it and append again below to
// update its position in the LRU.
heap->RemoveFromList();
} else {
heapsToMakeResident.push_back(heap->GetD3D12Pageable().Get());
sizeToMakeResident += heap->GetSize();
}
// If we submit a command list to the GPU, we must ensure that heaps referenced by that
// command list stay resident at least until that command list has finished execution.
// Setting this serial unnecessarily can leave the LRU in a state where nothing is
// eligible for eviction, even though some evictions may be possible.
heap->SetLastSubmission(pendingCommandSerial);
mLRUCache.Append(heap);
}
if (heapsToMakeResident.size() != 0) {
DAWN_TRY(EnsureCanMakeResident(sizeToMakeResident));
// Note that MakeResident is a synchronous function and can add a significant
// overhead to command recording. In the future, it may be possible to decrease this
// overhead by using MakeResident on a secondary thread, or by instead making use of
// the EnqueueMakeResident function (which is not available on all Windows 10
// platforms).
DAWN_TRY(CheckHRESULT(mDevice->GetD3D12Device()->MakeResident(
heapsToMakeResident.size(), heapsToMakeResident.data()),
"Making scheduled-to-be-used resources resident in "
"device local memory"));
}
return {};
}
// When a new heap is allocated, the heap will be made resident upon creation. We must track
// when this happens to avoid calling MakeResident a second time.
void ResidencyManager::TrackResidentAllocation(Heap* heap) {
if (!mResidencyManagementEnabled) {
return;
}
// Depending on device architecture and heap type, the heap may not need tracked.
if (!ShouldTrackHeap(heap)) {
return;
}
mLRUCache.Append(heap);
}
// Places an artifical cap on Dawn's budget so we can test in a predictable manner. If used,
// this function must be called before any resources have been created.
void ResidencyManager::RestrictBudgetForTesting(uint64_t artificialBudgetCap) {
ASSERT(mLRUCache.empty());
ASSERT(!mRestrictBudgetForTesting);
mRestrictBudgetForTesting = true;
UpdateVideoMemoryInfo();
// Dawn has a non-zero memory usage even before any resources have been created, and this
// value can vary depending on the environment Dawn is running in. By adding this in
// addition to the artificial budget cap, we can create a predictable and reproducible
// budget for testing.
mVideoMemoryInfo.dawnBudget = mVideoMemoryInfo.dawnUsage + artificialBudgetCap;
}
}} // namespace dawn_native::d3d12