Brandon Jones | 1f05968 | 2020-03-17 13:47:57 +0000 | [diff] [blame] | 1 | // Copyright 2020 The Dawn Authors |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
| 15 | #include "dawn_native/d3d12/ResidencyManagerD3D12.h" |
| 16 | |
| 17 | #include "dawn_native/d3d12/AdapterD3D12.h" |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 18 | #include "dawn_native/d3d12/D3D12Error.h" |
Brandon Jones | 1f05968 | 2020-03-17 13:47:57 +0000 | [diff] [blame] | 19 | #include "dawn_native/d3d12/DeviceD3D12.h" |
| 20 | #include "dawn_native/d3d12/Forward.h" |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 21 | #include "dawn_native/d3d12/HeapD3D12.h" |
| 22 | |
Brandon Jones | 1f05968 | 2020-03-17 13:47:57 +0000 | [diff] [blame] | 23 | namespace dawn_native { namespace d3d12 { |
| 24 | |
| 25 | ResidencyManager::ResidencyManager(Device* device) |
| 26 | : mDevice(device), |
| 27 | mResidencyManagementEnabled( |
| 28 | device->IsToggleEnabled(Toggle::UseD3D12ResidencyManagement)) { |
| 29 | UpdateVideoMemoryInfo(); |
| 30 | } |
| 31 | |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 32 | // Increments number of locks on a heap to ensure the heap remains resident. |
Brandon Jones | e7b30fd | 2020-05-19 21:45:13 +0000 | [diff] [blame] | 33 | MaybeError ResidencyManager::LockAllocation(Pageable* pageable) { |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 34 | if (!mResidencyManagementEnabled) { |
| 35 | return {}; |
| 36 | } |
| 37 | |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 38 | // If the heap isn't already resident, make it resident. |
Brandon Jones | e7b30fd | 2020-05-19 21:45:13 +0000 | [diff] [blame] | 39 | if (!pageable->IsInResidencyLRUCache() && !pageable->IsResidencyLocked()) { |
Brandon Jones | e7b30fd | 2020-05-19 21:45:13 +0000 | [diff] [blame] | 40 | ID3D12Pageable* d3d12Pageable = pageable->GetD3D12Pageable(); |
Brandon Jones | 0395ca9 | 2020-06-03 17:04:45 +0000 | [diff] [blame] | 41 | uint64_t size = pageable->GetSize(); |
| 42 | |
| 43 | DAWN_TRY(MakeAllocationsResident(GetMemorySegmentInfo(pageable->GetMemorySegment()), |
| 44 | size, 1, &d3d12Pageable)); |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 45 | } |
| 46 | |
| 47 | // Since we can't evict the heap, it's unnecessary to track the heap in the LRU Cache. |
Brandon Jones | e7b30fd | 2020-05-19 21:45:13 +0000 | [diff] [blame] | 48 | if (pageable->IsInResidencyLRUCache()) { |
| 49 | pageable->RemoveFromList(); |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 50 | } |
| 51 | |
Brandon Jones | e7b30fd | 2020-05-19 21:45:13 +0000 | [diff] [blame] | 52 | pageable->IncrementResidencyLock(); |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 53 | |
| 54 | return {}; |
| 55 | } |
| 56 | |
| 57 | // Decrements number of locks on a heap. When the number of locks becomes zero, the heap is |
| 58 | // inserted into the LRU cache and becomes eligible for eviction. |
Brandon Jones | e7b30fd | 2020-05-19 21:45:13 +0000 | [diff] [blame] | 59 | void ResidencyManager::UnlockAllocation(Pageable* pageable) { |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 60 | if (!mResidencyManagementEnabled) { |
| 61 | return; |
| 62 | } |
| 63 | |
Brandon Jones | e7b30fd | 2020-05-19 21:45:13 +0000 | [diff] [blame] | 64 | ASSERT(pageable->IsResidencyLocked()); |
| 65 | ASSERT(!pageable->IsInResidencyLRUCache()); |
| 66 | pageable->DecrementResidencyLock(); |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 67 | |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 68 | // If another lock still exists on the heap, nothing further should be done. |
Brandon Jones | e7b30fd | 2020-05-19 21:45:13 +0000 | [diff] [blame] | 69 | if (pageable->IsResidencyLocked()) { |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 70 | return; |
| 71 | } |
| 72 | |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 73 | // When all locks have been removed, the resource remains resident and becomes tracked in |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 74 | // the corresponding LRU. |
Brandon Jones | e7b30fd | 2020-05-19 21:45:13 +0000 | [diff] [blame] | 75 | TrackResidentAllocation(pageable); |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 76 | } |
| 77 | |
| 78 | // Returns the appropriate MemorySegmentInfo for a given MemorySegment. |
| 79 | ResidencyManager::MemorySegmentInfo* ResidencyManager::GetMemorySegmentInfo( |
| 80 | MemorySegment memorySegment) { |
| 81 | switch (memorySegment) { |
| 82 | case MemorySegment::Local: |
| 83 | return &mVideoMemoryInfo.local; |
| 84 | case MemorySegment::NonLocal: |
| 85 | ASSERT(!mDevice->GetDeviceInfo().isUMA); |
| 86 | return &mVideoMemoryInfo.nonLocal; |
| 87 | default: |
| 88 | UNREACHABLE(); |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 89 | } |
| 90 | } |
| 91 | |
Brandon Jones | f56f190 | 2020-04-22 21:57:00 +0000 | [diff] [blame] | 92 | // Allows an application component external to Dawn to cap Dawn's residency budgets to prevent |
| 93 | // competition for device memory. Returns the amount of memory reserved, which may be less |
Brandon Jones | 1f05968 | 2020-03-17 13:47:57 +0000 | [diff] [blame] | 94 | // that the requested reservation when under pressure. |
Brandon Jones | f56f190 | 2020-04-22 21:57:00 +0000 | [diff] [blame] | 95 | uint64_t ResidencyManager::SetExternalMemoryReservation(MemorySegment segment, |
| 96 | uint64_t requestedReservationSize) { |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 97 | MemorySegmentInfo* segmentInfo = GetMemorySegmentInfo(segment); |
Brandon Jones | f56f190 | 2020-04-22 21:57:00 +0000 | [diff] [blame] | 98 | |
| 99 | segmentInfo->externalRequest = requestedReservationSize; |
| 100 | |
| 101 | UpdateMemorySegmentInfo(segmentInfo); |
| 102 | |
| 103 | return segmentInfo->externalReservation; |
Brandon Jones | 1f05968 | 2020-03-17 13:47:57 +0000 | [diff] [blame] | 104 | } |
| 105 | |
| 106 | void ResidencyManager::UpdateVideoMemoryInfo() { |
Brandon Jones | f56f190 | 2020-04-22 21:57:00 +0000 | [diff] [blame] | 107 | UpdateMemorySegmentInfo(&mVideoMemoryInfo.local); |
| 108 | if (!mDevice->GetDeviceInfo().isUMA) { |
| 109 | UpdateMemorySegmentInfo(&mVideoMemoryInfo.nonLocal); |
Brandon Jones | 1f05968 | 2020-03-17 13:47:57 +0000 | [diff] [blame] | 110 | } |
Brandon Jones | f56f190 | 2020-04-22 21:57:00 +0000 | [diff] [blame] | 111 | } |
Brandon Jones | 1f05968 | 2020-03-17 13:47:57 +0000 | [diff] [blame] | 112 | |
Brandon Jones | f56f190 | 2020-04-22 21:57:00 +0000 | [diff] [blame] | 113 | void ResidencyManager::UpdateMemorySegmentInfo(MemorySegmentInfo* segmentInfo) { |
Brandon Jones | 1f05968 | 2020-03-17 13:47:57 +0000 | [diff] [blame] | 114 | DXGI_QUERY_VIDEO_MEMORY_INFO queryVideoMemoryInfo; |
Brandon Jones | f56f190 | 2020-04-22 21:57:00 +0000 | [diff] [blame] | 115 | |
Brandon Jones | 1f05968 | 2020-03-17 13:47:57 +0000 | [diff] [blame] | 116 | ToBackend(mDevice->GetAdapter()) |
| 117 | ->GetHardwareAdapter() |
Brandon Jones | f56f190 | 2020-04-22 21:57:00 +0000 | [diff] [blame] | 118 | ->QueryVideoMemoryInfo(0, segmentInfo->dxgiSegment, &queryVideoMemoryInfo); |
Brandon Jones | 1f05968 | 2020-03-17 13:47:57 +0000 | [diff] [blame] | 119 | |
| 120 | // The video memory budget provided by QueryVideoMemoryInfo is defined by the operating |
| 121 | // system, and may be lower than expected in certain scenarios. Under memory pressure, we |
| 122 | // cap the external reservation to half the available budget, which prevents the external |
| 123 | // component from consuming a disproportionate share of memory and ensures that Dawn can |
| 124 | // continue to make forward progress. Note the choice to halve memory is arbitrarily chosen |
| 125 | // and subject to future experimentation. |
Brandon Jones | f56f190 | 2020-04-22 21:57:00 +0000 | [diff] [blame] | 126 | segmentInfo->externalReservation = |
| 127 | std::min(queryVideoMemoryInfo.Budget / 2, segmentInfo->externalRequest); |
Brandon Jones | ab2c84f | 2020-04-09 23:31:02 +0000 | [diff] [blame] | 128 | |
Brandon Jones | f56f190 | 2020-04-22 21:57:00 +0000 | [diff] [blame] | 129 | segmentInfo->usage = queryVideoMemoryInfo.CurrentUsage - segmentInfo->externalReservation; |
Brandon Jones | ab2c84f | 2020-04-09 23:31:02 +0000 | [diff] [blame] | 130 | |
| 131 | // If we're restricting the budget for testing, leave the budget as is. |
| 132 | if (mRestrictBudgetForTesting) { |
| 133 | return; |
| 134 | } |
Brandon Jones | 1f05968 | 2020-03-17 13:47:57 +0000 | [diff] [blame] | 135 | |
| 136 | // We cap Dawn's budget to 95% of the provided budget. Leaving some budget unused |
| 137 | // decreases fluctuations in the operating-system-defined budget, which improves stability |
| 138 | // for both Dawn and other applications on the system. Note the value of 95% is arbitrarily |
| 139 | // chosen and subject to future experimentation. |
| 140 | static constexpr float kBudgetCap = 0.95; |
Brandon Jones | f56f190 | 2020-04-22 21:57:00 +0000 | [diff] [blame] | 141 | segmentInfo->budget = |
| 142 | (queryVideoMemoryInfo.Budget - segmentInfo->externalReservation) * kBudgetCap; |
Brandon Jones | 1f05968 | 2020-03-17 13:47:57 +0000 | [diff] [blame] | 143 | } |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 144 | |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 145 | // Removes a heap from the LRU and returns the least recently used heap when possible. Returns |
| 146 | // nullptr when nothing further can be evicted. |
Brandon Jones | 9b54466 | 2020-05-19 10:15:12 +0000 | [diff] [blame] | 147 | ResultOrError<Pageable*> ResidencyManager::RemoveSingleEntryFromLRU( |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 148 | MemorySegmentInfo* memorySegment) { |
Brandon Jones | e370ec6 | 2020-05-26 19:40:23 +0000 | [diff] [blame] | 149 | // If the LRU is empty, return nullptr to allow execution to continue. Note that fully |
| 150 | // emptying the LRU is undesirable, because it can mean either 1) the LRU is not accurately |
| 151 | // accounting for Dawn's GPU allocations, or 2) a component external to Dawn is using all of |
| 152 | // the process budget and starving Dawn, which will cause thrash. |
| 153 | if (memorySegment->lruCache.empty()) { |
| 154 | return nullptr; |
| 155 | } |
| 156 | |
Brandon Jones | 9b54466 | 2020-05-19 10:15:12 +0000 | [diff] [blame] | 157 | Pageable* pageable = memorySegment->lruCache.head()->value(); |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 158 | |
Corentin Wallez | 62139fc | 2020-09-28 19:35:14 +0000 | [diff] [blame] | 159 | ExecutionSerial lastSubmissionSerial = pageable->GetLastSubmission(); |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 160 | |
| 161 | // If the next candidate for eviction was inserted into the LRU during the current serial, |
| 162 | // it is because more memory is being used in a single command list than is available. |
| 163 | // In this scenario, we cannot make any more resources resident and thrashing must occur. |
| 164 | if (lastSubmissionSerial == mDevice->GetPendingCommandSerial()) { |
| 165 | return nullptr; |
| 166 | } |
| 167 | |
| 168 | // We must ensure that any previous use of a resource has completed before the resource can |
| 169 | // be evicted. |
| 170 | if (lastSubmissionSerial > mDevice->GetCompletedCommandSerial()) { |
| 171 | DAWN_TRY(mDevice->WaitForSerial(lastSubmissionSerial)); |
| 172 | } |
| 173 | |
Brandon Jones | 9b54466 | 2020-05-19 10:15:12 +0000 | [diff] [blame] | 174 | pageable->RemoveFromList(); |
| 175 | return pageable; |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 176 | } |
| 177 | |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 178 | MaybeError ResidencyManager::EnsureCanAllocate(uint64_t allocationSize, |
| 179 | MemorySegment memorySegment) { |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 180 | if (!mResidencyManagementEnabled) { |
| 181 | return {}; |
| 182 | } |
| 183 | |
Brandon Jones | 0395ca9 | 2020-06-03 17:04:45 +0000 | [diff] [blame] | 184 | uint64_t bytesEvicted; |
| 185 | DAWN_TRY_ASSIGN(bytesEvicted, |
| 186 | EnsureCanMakeResident(allocationSize, GetMemorySegmentInfo(memorySegment))); |
Peter Kasting | 7e50a7f | 2021-07-27 18:42:59 +0000 | [diff] [blame] | 187 | DAWN_UNUSED(bytesEvicted); |
Brandon Jones | 0395ca9 | 2020-06-03 17:04:45 +0000 | [diff] [blame] | 188 | |
| 189 | return {}; |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 190 | } |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 191 | |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 192 | // Any time we need to make something resident, we must check that we have enough free memory to |
| 193 | // make the new object resident while also staying within budget. If there isn't enough |
Brandon Jones | 0395ca9 | 2020-06-03 17:04:45 +0000 | [diff] [blame] | 194 | // memory, we should evict until there is. Returns the number of bytes evicted. |
| 195 | ResultOrError<uint64_t> ResidencyManager::EnsureCanMakeResident( |
| 196 | uint64_t sizeToMakeResident, |
| 197 | MemorySegmentInfo* memorySegment) { |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 198 | ASSERT(mResidencyManagementEnabled); |
| 199 | |
| 200 | UpdateMemorySegmentInfo(memorySegment); |
| 201 | |
| 202 | uint64_t memoryUsageAfterMakeResident = sizeToMakeResident + memorySegment->usage; |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 203 | |
| 204 | // Return when we can call MakeResident and remain under budget. |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 205 | if (memoryUsageAfterMakeResident < memorySegment->budget) { |
Brandon Jones | 0395ca9 | 2020-06-03 17:04:45 +0000 | [diff] [blame] | 206 | return 0; |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 207 | } |
| 208 | |
| 209 | std::vector<ID3D12Pageable*> resourcesToEvict; |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 210 | uint64_t sizeNeededToBeUnderBudget = memoryUsageAfterMakeResident - memorySegment->budget; |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 211 | uint64_t sizeEvicted = 0; |
Brandon Jones | 82ae680 | 2020-04-16 23:52:23 +0000 | [diff] [blame] | 212 | while (sizeEvicted < sizeNeededToBeUnderBudget) { |
Brandon Jones | 9b54466 | 2020-05-19 10:15:12 +0000 | [diff] [blame] | 213 | Pageable* pageable; |
| 214 | DAWN_TRY_ASSIGN(pageable, RemoveSingleEntryFromLRU(memorySegment)); |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 215 | |
| 216 | // If no heap was returned, then nothing more can be evicted. |
Brandon Jones | 9b54466 | 2020-05-19 10:15:12 +0000 | [diff] [blame] | 217 | if (pageable == nullptr) { |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 218 | break; |
| 219 | } |
| 220 | |
Brandon Jones | 9b54466 | 2020-05-19 10:15:12 +0000 | [diff] [blame] | 221 | sizeEvicted += pageable->GetSize(); |
| 222 | resourcesToEvict.push_back(pageable->GetD3D12Pageable()); |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 223 | } |
| 224 | |
| 225 | if (resourcesToEvict.size() > 0) { |
| 226 | DAWN_TRY(CheckHRESULT( |
| 227 | mDevice->GetD3D12Device()->Evict(resourcesToEvict.size(), resourcesToEvict.data()), |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 228 | "Evicting resident heaps to free memory")); |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 229 | } |
| 230 | |
Brandon Jones | 0395ca9 | 2020-06-03 17:04:45 +0000 | [diff] [blame] | 231 | return sizeEvicted; |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 232 | } |
| 233 | |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 234 | // Given a list of heaps that are pending usage, this function will estimate memory needed, |
| 235 | // evict resources until enough space is available, then make resident any heaps scheduled for |
| 236 | // usage. |
| 237 | MaybeError ResidencyManager::EnsureHeapsAreResident(Heap** heaps, size_t heapCount) { |
| 238 | if (!mResidencyManagementEnabled) { |
| 239 | return {}; |
| 240 | } |
| 241 | |
Brandon Jones | 0395ca9 | 2020-06-03 17:04:45 +0000 | [diff] [blame] | 242 | std::vector<ID3D12Pageable*> localHeapsToMakeResident; |
| 243 | std::vector<ID3D12Pageable*> nonLocalHeapsToMakeResident; |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 244 | uint64_t localSizeToMakeResident = 0; |
| 245 | uint64_t nonLocalSizeToMakeResident = 0; |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 246 | |
Corentin Wallez | 62139fc | 2020-09-28 19:35:14 +0000 | [diff] [blame] | 247 | ExecutionSerial pendingCommandSerial = mDevice->GetPendingCommandSerial(); |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 248 | for (size_t i = 0; i < heapCount; i++) { |
| 249 | Heap* heap = heaps[i]; |
| 250 | |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 251 | // Heaps that are locked resident are not tracked in the LRU cache. |
| 252 | if (heap->IsResidencyLocked()) { |
| 253 | continue; |
| 254 | } |
| 255 | |
| 256 | if (heap->IsInResidencyLRUCache()) { |
| 257 | // If the heap is already in the LRU, we must remove it and append again below to |
| 258 | // update its position in the LRU. |
| 259 | heap->RemoveFromList(); |
| 260 | } else { |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 261 | if (heap->GetMemorySegment() == MemorySegment::Local) { |
| 262 | localSizeToMakeResident += heap->GetSize(); |
Brandon Jones | 0395ca9 | 2020-06-03 17:04:45 +0000 | [diff] [blame] | 263 | localHeapsToMakeResident.push_back(heap->GetD3D12Pageable()); |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 264 | } else { |
| 265 | nonLocalSizeToMakeResident += heap->GetSize(); |
Brandon Jones | 0395ca9 | 2020-06-03 17:04:45 +0000 | [diff] [blame] | 266 | nonLocalHeapsToMakeResident.push_back(heap->GetD3D12Pageable()); |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 267 | } |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 268 | } |
| 269 | |
Brandon Jones | 7be1d84 | 2020-04-09 23:28:22 +0000 | [diff] [blame] | 270 | // If we submit a command list to the GPU, we must ensure that heaps referenced by that |
| 271 | // command list stay resident at least until that command list has finished execution. |
| 272 | // Setting this serial unnecessarily can leave the LRU in a state where nothing is |
| 273 | // eligible for eviction, even though some evictions may be possible. |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 274 | heap->SetLastSubmission(pendingCommandSerial); |
Brandon Jones | 7be1d84 | 2020-04-09 23:28:22 +0000 | [diff] [blame] | 275 | |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 276 | // Insert the heap into the appropriate LRU. |
| 277 | TrackResidentAllocation(heap); |
| 278 | } |
| 279 | |
| 280 | if (localSizeToMakeResident > 0) { |
Brandon Jones | 0395ca9 | 2020-06-03 17:04:45 +0000 | [diff] [blame] | 281 | return MakeAllocationsResident(&mVideoMemoryInfo.local, localSizeToMakeResident, |
| 282 | localHeapsToMakeResident.size(), |
| 283 | localHeapsToMakeResident.data()); |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 284 | } |
| 285 | |
| 286 | if (nonLocalSizeToMakeResident > 0) { |
| 287 | ASSERT(!mDevice->GetDeviceInfo().isUMA); |
Brandon Jones | 0395ca9 | 2020-06-03 17:04:45 +0000 | [diff] [blame] | 288 | return MakeAllocationsResident(&mVideoMemoryInfo.nonLocal, nonLocalSizeToMakeResident, |
| 289 | nonLocalHeapsToMakeResident.size(), |
| 290 | nonLocalHeapsToMakeResident.data()); |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 291 | } |
| 292 | |
Brandon Jones | 0395ca9 | 2020-06-03 17:04:45 +0000 | [diff] [blame] | 293 | return {}; |
| 294 | } |
| 295 | |
| 296 | MaybeError ResidencyManager::MakeAllocationsResident(MemorySegmentInfo* segment, |
| 297 | uint64_t sizeToMakeResident, |
| 298 | uint64_t numberOfObjectsToMakeResident, |
| 299 | ID3D12Pageable** allocations) { |
| 300 | uint64_t bytesEvicted; |
| 301 | DAWN_TRY_ASSIGN(bytesEvicted, EnsureCanMakeResident(sizeToMakeResident, segment)); |
Peter Kasting | 7e50a7f | 2021-07-27 18:42:59 +0000 | [diff] [blame] | 302 | DAWN_UNUSED(bytesEvicted); |
Brandon Jones | 0395ca9 | 2020-06-03 17:04:45 +0000 | [diff] [blame] | 303 | |
| 304 | // Note that MakeResident is a synchronous function and can add a significant |
| 305 | // overhead to command recording. In the future, it may be possible to decrease this |
| 306 | // overhead by using MakeResident on a secondary thread, or by instead making use of |
| 307 | // the EnqueueMakeResident function (which is not available on all Windows 10 |
| 308 | // platforms). |
| 309 | HRESULT hr = |
| 310 | mDevice->GetD3D12Device()->MakeResident(numberOfObjectsToMakeResident, allocations); |
| 311 | |
| 312 | // A MakeResident call can fail if there's not enough available memory. This |
| 313 | // could occur when there's significant fragmentation or if the allocation size |
| 314 | // estimates are incorrect. We may be able to continue execution by evicting some |
| 315 | // more memory and calling MakeResident again. |
| 316 | while (FAILED(hr)) { |
| 317 | constexpr uint32_t kAdditonalSizeToEvict = 50000000; // 50MB |
| 318 | |
| 319 | uint64_t sizeEvicted = 0; |
| 320 | |
| 321 | DAWN_TRY_ASSIGN(sizeEvicted, EnsureCanMakeResident(kAdditonalSizeToEvict, segment)); |
| 322 | |
| 323 | // If nothing can be evicted after MakeResident has failed, we cannot continue |
| 324 | // execution and must throw a fatal error. |
| 325 | if (sizeEvicted == 0) { |
| 326 | return DAWN_OUT_OF_MEMORY_ERROR( |
| 327 | "MakeResident has failed due to excessive video memory usage."); |
| 328 | } |
| 329 | |
| 330 | hr = |
| 331 | mDevice->GetD3D12Device()->MakeResident(numberOfObjectsToMakeResident, allocations); |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 332 | } |
| 333 | |
| 334 | return {}; |
| 335 | } |
| 336 | |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 337 | // Inserts a heap at the bottom of the LRU. The passed heap must be resident or scheduled to |
Brandon Jones | 9435068 | 2020-06-12 08:17:41 +0000 | [diff] [blame] | 338 | // become resident within the current serial. Failing to call this function when an allocation |
| 339 | // is implicitly made resident will cause the residency manager to view the allocation as |
| 340 | // non-resident and call MakeResident - which will make D3D12's internal residency refcount on |
| 341 | // the allocation out of sync with Dawn. |
Brandon Jones | 9b54466 | 2020-05-19 10:15:12 +0000 | [diff] [blame] | 342 | void ResidencyManager::TrackResidentAllocation(Pageable* pageable) { |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 343 | if (!mResidencyManagementEnabled) { |
| 344 | return; |
| 345 | } |
| 346 | |
Brandon Jones | 9b54466 | 2020-05-19 10:15:12 +0000 | [diff] [blame] | 347 | ASSERT(pageable->IsInList() == false); |
| 348 | GetMemorySegmentInfo(pageable->GetMemorySegment())->lruCache.Append(pageable); |
Brandon Jones | 7982cc0 | 2020-03-31 15:31:56 +0000 | [diff] [blame] | 349 | } |
Brandon Jones | ab2c84f | 2020-04-09 23:31:02 +0000 | [diff] [blame] | 350 | |
| 351 | // Places an artifical cap on Dawn's budget so we can test in a predictable manner. If used, |
| 352 | // this function must be called before any resources have been created. |
| 353 | void ResidencyManager::RestrictBudgetForTesting(uint64_t artificialBudgetCap) { |
Brandon Jones | 635239f | 2020-04-23 21:50:32 +0000 | [diff] [blame] | 354 | ASSERT(mVideoMemoryInfo.nonLocal.lruCache.empty()); |
Brandon Jones | ab2c84f | 2020-04-09 23:31:02 +0000 | [diff] [blame] | 355 | ASSERT(!mRestrictBudgetForTesting); |
| 356 | |
| 357 | mRestrictBudgetForTesting = true; |
| 358 | UpdateVideoMemoryInfo(); |
| 359 | |
| 360 | // Dawn has a non-zero memory usage even before any resources have been created, and this |
| 361 | // value can vary depending on the environment Dawn is running in. By adding this in |
| 362 | // addition to the artificial budget cap, we can create a predictable and reproducible |
| 363 | // budget for testing. |
Brandon Jones | f56f190 | 2020-04-22 21:57:00 +0000 | [diff] [blame] | 364 | mVideoMemoryInfo.local.budget = mVideoMemoryInfo.local.usage + artificialBudgetCap; |
| 365 | if (!mDevice->GetDeviceInfo().isUMA) { |
| 366 | mVideoMemoryInfo.nonLocal.budget = |
| 367 | mVideoMemoryInfo.nonLocal.usage + artificialBudgetCap; |
| 368 | } |
Brandon Jones | ab2c84f | 2020-04-09 23:31:02 +0000 | [diff] [blame] | 369 | } |
| 370 | |
Kai Ninomiya | 6328356 | 2020-07-10 18:19:38 +0000 | [diff] [blame] | 371 | }} // namespace dawn_native::d3d12 |