blob: ffc1e1b85a4e3b4a375c77a6eccb873562cf1b49 [file] [log] [blame]
Brandon Jones1f059682020-03-17 13:47:57 +00001// Copyright 2020 The Dawn Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#include "dawn_native/d3d12/ResidencyManagerD3D12.h"
16
17#include "dawn_native/d3d12/AdapterD3D12.h"
Brandon Jones7982cc02020-03-31 15:31:56 +000018#include "dawn_native/d3d12/D3D12Error.h"
Brandon Jones1f059682020-03-17 13:47:57 +000019#include "dawn_native/d3d12/DeviceD3D12.h"
20#include "dawn_native/d3d12/Forward.h"
Brandon Jones7982cc02020-03-31 15:31:56 +000021#include "dawn_native/d3d12/HeapD3D12.h"
22
Brandon Jones1f059682020-03-17 13:47:57 +000023namespace dawn_native { namespace d3d12 {
24
25 ResidencyManager::ResidencyManager(Device* device)
26 : mDevice(device),
27 mResidencyManagementEnabled(
28 device->IsToggleEnabled(Toggle::UseD3D12ResidencyManagement)) {
29 UpdateVideoMemoryInfo();
30 }
31
Brandon Jones7982cc02020-03-31 15:31:56 +000032 // Increments number of locks on a heap to ensure the heap remains resident.
Brandon Jonese7b30fd2020-05-19 21:45:13 +000033 MaybeError ResidencyManager::LockAllocation(Pageable* pageable) {
Brandon Jones7982cc02020-03-31 15:31:56 +000034 if (!mResidencyManagementEnabled) {
35 return {};
36 }
37
Brandon Jones7982cc02020-03-31 15:31:56 +000038 // If the heap isn't already resident, make it resident.
Brandon Jonese7b30fd2020-05-19 21:45:13 +000039 if (!pageable->IsInResidencyLRUCache() && !pageable->IsResidencyLocked()) {
Brandon Jonese7b30fd2020-05-19 21:45:13 +000040 ID3D12Pageable* d3d12Pageable = pageable->GetD3D12Pageable();
Brandon Jones0395ca92020-06-03 17:04:45 +000041 uint64_t size = pageable->GetSize();
42
43 DAWN_TRY(MakeAllocationsResident(GetMemorySegmentInfo(pageable->GetMemorySegment()),
44 size, 1, &d3d12Pageable));
Brandon Jones7982cc02020-03-31 15:31:56 +000045 }
46
47 // Since we can't evict the heap, it's unnecessary to track the heap in the LRU Cache.
Brandon Jonese7b30fd2020-05-19 21:45:13 +000048 if (pageable->IsInResidencyLRUCache()) {
49 pageable->RemoveFromList();
Brandon Jones7982cc02020-03-31 15:31:56 +000050 }
51
Brandon Jonese7b30fd2020-05-19 21:45:13 +000052 pageable->IncrementResidencyLock();
Brandon Jones7982cc02020-03-31 15:31:56 +000053
54 return {};
55 }
56
57 // Decrements number of locks on a heap. When the number of locks becomes zero, the heap is
58 // inserted into the LRU cache and becomes eligible for eviction.
Brandon Jonese7b30fd2020-05-19 21:45:13 +000059 void ResidencyManager::UnlockAllocation(Pageable* pageable) {
Brandon Jones7982cc02020-03-31 15:31:56 +000060 if (!mResidencyManagementEnabled) {
61 return;
62 }
63
Brandon Jonese7b30fd2020-05-19 21:45:13 +000064 ASSERT(pageable->IsResidencyLocked());
65 ASSERT(!pageable->IsInResidencyLRUCache());
66 pageable->DecrementResidencyLock();
Brandon Jones7982cc02020-03-31 15:31:56 +000067
Brandon Jones635239f2020-04-23 21:50:32 +000068 // If another lock still exists on the heap, nothing further should be done.
Brandon Jonese7b30fd2020-05-19 21:45:13 +000069 if (pageable->IsResidencyLocked()) {
Brandon Jones635239f2020-04-23 21:50:32 +000070 return;
71 }
72
Brandon Jones7982cc02020-03-31 15:31:56 +000073 // When all locks have been removed, the resource remains resident and becomes tracked in
Brandon Jones635239f2020-04-23 21:50:32 +000074 // the corresponding LRU.
Brandon Jonese7b30fd2020-05-19 21:45:13 +000075 TrackResidentAllocation(pageable);
Brandon Jones635239f2020-04-23 21:50:32 +000076 }
77
78 // Returns the appropriate MemorySegmentInfo for a given MemorySegment.
79 ResidencyManager::MemorySegmentInfo* ResidencyManager::GetMemorySegmentInfo(
80 MemorySegment memorySegment) {
81 switch (memorySegment) {
82 case MemorySegment::Local:
83 return &mVideoMemoryInfo.local;
84 case MemorySegment::NonLocal:
85 ASSERT(!mDevice->GetDeviceInfo().isUMA);
86 return &mVideoMemoryInfo.nonLocal;
87 default:
88 UNREACHABLE();
Brandon Jones7982cc02020-03-31 15:31:56 +000089 }
90 }
91
Brandon Jonesf56f1902020-04-22 21:57:00 +000092 // Allows an application component external to Dawn to cap Dawn's residency budgets to prevent
93 // competition for device memory. Returns the amount of memory reserved, which may be less
Brandon Jones1f059682020-03-17 13:47:57 +000094 // that the requested reservation when under pressure.
Brandon Jonesf56f1902020-04-22 21:57:00 +000095 uint64_t ResidencyManager::SetExternalMemoryReservation(MemorySegment segment,
96 uint64_t requestedReservationSize) {
Brandon Jones635239f2020-04-23 21:50:32 +000097 MemorySegmentInfo* segmentInfo = GetMemorySegmentInfo(segment);
Brandon Jonesf56f1902020-04-22 21:57:00 +000098
99 segmentInfo->externalRequest = requestedReservationSize;
100
101 UpdateMemorySegmentInfo(segmentInfo);
102
103 return segmentInfo->externalReservation;
Brandon Jones1f059682020-03-17 13:47:57 +0000104 }
105
106 void ResidencyManager::UpdateVideoMemoryInfo() {
Brandon Jonesf56f1902020-04-22 21:57:00 +0000107 UpdateMemorySegmentInfo(&mVideoMemoryInfo.local);
108 if (!mDevice->GetDeviceInfo().isUMA) {
109 UpdateMemorySegmentInfo(&mVideoMemoryInfo.nonLocal);
Brandon Jones1f059682020-03-17 13:47:57 +0000110 }
Brandon Jonesf56f1902020-04-22 21:57:00 +0000111 }
Brandon Jones1f059682020-03-17 13:47:57 +0000112
Brandon Jonesf56f1902020-04-22 21:57:00 +0000113 void ResidencyManager::UpdateMemorySegmentInfo(MemorySegmentInfo* segmentInfo) {
Brandon Jones1f059682020-03-17 13:47:57 +0000114 DXGI_QUERY_VIDEO_MEMORY_INFO queryVideoMemoryInfo;
Brandon Jonesf56f1902020-04-22 21:57:00 +0000115
Brandon Jones1f059682020-03-17 13:47:57 +0000116 ToBackend(mDevice->GetAdapter())
117 ->GetHardwareAdapter()
Brandon Jonesf56f1902020-04-22 21:57:00 +0000118 ->QueryVideoMemoryInfo(0, segmentInfo->dxgiSegment, &queryVideoMemoryInfo);
Brandon Jones1f059682020-03-17 13:47:57 +0000119
120 // The video memory budget provided by QueryVideoMemoryInfo is defined by the operating
121 // system, and may be lower than expected in certain scenarios. Under memory pressure, we
122 // cap the external reservation to half the available budget, which prevents the external
123 // component from consuming a disproportionate share of memory and ensures that Dawn can
124 // continue to make forward progress. Note the choice to halve memory is arbitrarily chosen
125 // and subject to future experimentation.
Brandon Jonesf56f1902020-04-22 21:57:00 +0000126 segmentInfo->externalReservation =
127 std::min(queryVideoMemoryInfo.Budget / 2, segmentInfo->externalRequest);
Brandon Jonesab2c84f2020-04-09 23:31:02 +0000128
Brandon Jonesf56f1902020-04-22 21:57:00 +0000129 segmentInfo->usage = queryVideoMemoryInfo.CurrentUsage - segmentInfo->externalReservation;
Brandon Jonesab2c84f2020-04-09 23:31:02 +0000130
131 // If we're restricting the budget for testing, leave the budget as is.
132 if (mRestrictBudgetForTesting) {
133 return;
134 }
Brandon Jones1f059682020-03-17 13:47:57 +0000135
136 // We cap Dawn's budget to 95% of the provided budget. Leaving some budget unused
137 // decreases fluctuations in the operating-system-defined budget, which improves stability
138 // for both Dawn and other applications on the system. Note the value of 95% is arbitrarily
139 // chosen and subject to future experimentation.
140 static constexpr float kBudgetCap = 0.95;
Brandon Jonesf56f1902020-04-22 21:57:00 +0000141 segmentInfo->budget =
142 (queryVideoMemoryInfo.Budget - segmentInfo->externalReservation) * kBudgetCap;
Brandon Jones1f059682020-03-17 13:47:57 +0000143 }
Brandon Jones7982cc02020-03-31 15:31:56 +0000144
Brandon Jones635239f2020-04-23 21:50:32 +0000145 // Removes a heap from the LRU and returns the least recently used heap when possible. Returns
146 // nullptr when nothing further can be evicted.
Brandon Jones9b544662020-05-19 10:15:12 +0000147 ResultOrError<Pageable*> ResidencyManager::RemoveSingleEntryFromLRU(
Brandon Jones635239f2020-04-23 21:50:32 +0000148 MemorySegmentInfo* memorySegment) {
Brandon Jonese370ec62020-05-26 19:40:23 +0000149 // If the LRU is empty, return nullptr to allow execution to continue. Note that fully
150 // emptying the LRU is undesirable, because it can mean either 1) the LRU is not accurately
151 // accounting for Dawn's GPU allocations, or 2) a component external to Dawn is using all of
152 // the process budget and starving Dawn, which will cause thrash.
153 if (memorySegment->lruCache.empty()) {
154 return nullptr;
155 }
156
Brandon Jones9b544662020-05-19 10:15:12 +0000157 Pageable* pageable = memorySegment->lruCache.head()->value();
Brandon Jones635239f2020-04-23 21:50:32 +0000158
Corentin Wallez62139fc2020-09-28 19:35:14 +0000159 ExecutionSerial lastSubmissionSerial = pageable->GetLastSubmission();
Brandon Jones7982cc02020-03-31 15:31:56 +0000160
161 // If the next candidate for eviction was inserted into the LRU during the current serial,
162 // it is because more memory is being used in a single command list than is available.
163 // In this scenario, we cannot make any more resources resident and thrashing must occur.
164 if (lastSubmissionSerial == mDevice->GetPendingCommandSerial()) {
165 return nullptr;
166 }
167
168 // We must ensure that any previous use of a resource has completed before the resource can
169 // be evicted.
170 if (lastSubmissionSerial > mDevice->GetCompletedCommandSerial()) {
171 DAWN_TRY(mDevice->WaitForSerial(lastSubmissionSerial));
172 }
173
Brandon Jones9b544662020-05-19 10:15:12 +0000174 pageable->RemoveFromList();
175 return pageable;
Brandon Jones7982cc02020-03-31 15:31:56 +0000176 }
177
Brandon Jones635239f2020-04-23 21:50:32 +0000178 MaybeError ResidencyManager::EnsureCanAllocate(uint64_t allocationSize,
179 MemorySegment memorySegment) {
Brandon Jones7982cc02020-03-31 15:31:56 +0000180 if (!mResidencyManagementEnabled) {
181 return {};
182 }
183
Brandon Jones0395ca92020-06-03 17:04:45 +0000184 uint64_t bytesEvicted;
185 DAWN_TRY_ASSIGN(bytesEvicted,
186 EnsureCanMakeResident(allocationSize, GetMemorySegmentInfo(memorySegment)));
Peter Kasting7e50a7f2021-07-27 18:42:59 +0000187 DAWN_UNUSED(bytesEvicted);
Brandon Jones0395ca92020-06-03 17:04:45 +0000188
189 return {};
Brandon Jones635239f2020-04-23 21:50:32 +0000190 }
Brandon Jones7982cc02020-03-31 15:31:56 +0000191
Brandon Jones635239f2020-04-23 21:50:32 +0000192 // Any time we need to make something resident, we must check that we have enough free memory to
193 // make the new object resident while also staying within budget. If there isn't enough
Brandon Jones0395ca92020-06-03 17:04:45 +0000194 // memory, we should evict until there is. Returns the number of bytes evicted.
195 ResultOrError<uint64_t> ResidencyManager::EnsureCanMakeResident(
196 uint64_t sizeToMakeResident,
197 MemorySegmentInfo* memorySegment) {
Brandon Jones635239f2020-04-23 21:50:32 +0000198 ASSERT(mResidencyManagementEnabled);
199
200 UpdateMemorySegmentInfo(memorySegment);
201
202 uint64_t memoryUsageAfterMakeResident = sizeToMakeResident + memorySegment->usage;
Brandon Jones7982cc02020-03-31 15:31:56 +0000203
204 // Return when we can call MakeResident and remain under budget.
Brandon Jones635239f2020-04-23 21:50:32 +0000205 if (memoryUsageAfterMakeResident < memorySegment->budget) {
Brandon Jones0395ca92020-06-03 17:04:45 +0000206 return 0;
Brandon Jones7982cc02020-03-31 15:31:56 +0000207 }
208
209 std::vector<ID3D12Pageable*> resourcesToEvict;
Brandon Jones635239f2020-04-23 21:50:32 +0000210 uint64_t sizeNeededToBeUnderBudget = memoryUsageAfterMakeResident - memorySegment->budget;
Brandon Jones7982cc02020-03-31 15:31:56 +0000211 uint64_t sizeEvicted = 0;
Brandon Jones82ae6802020-04-16 23:52:23 +0000212 while (sizeEvicted < sizeNeededToBeUnderBudget) {
Brandon Jones9b544662020-05-19 10:15:12 +0000213 Pageable* pageable;
214 DAWN_TRY_ASSIGN(pageable, RemoveSingleEntryFromLRU(memorySegment));
Brandon Jones7982cc02020-03-31 15:31:56 +0000215
216 // If no heap was returned, then nothing more can be evicted.
Brandon Jones9b544662020-05-19 10:15:12 +0000217 if (pageable == nullptr) {
Brandon Jones7982cc02020-03-31 15:31:56 +0000218 break;
219 }
220
Brandon Jones9b544662020-05-19 10:15:12 +0000221 sizeEvicted += pageable->GetSize();
222 resourcesToEvict.push_back(pageable->GetD3D12Pageable());
Brandon Jones7982cc02020-03-31 15:31:56 +0000223 }
224
225 if (resourcesToEvict.size() > 0) {
226 DAWN_TRY(CheckHRESULT(
227 mDevice->GetD3D12Device()->Evict(resourcesToEvict.size(), resourcesToEvict.data()),
Brandon Jones635239f2020-04-23 21:50:32 +0000228 "Evicting resident heaps to free memory"));
Brandon Jones7982cc02020-03-31 15:31:56 +0000229 }
230
Brandon Jones0395ca92020-06-03 17:04:45 +0000231 return sizeEvicted;
Brandon Jones7982cc02020-03-31 15:31:56 +0000232 }
233
Brandon Jones7982cc02020-03-31 15:31:56 +0000234 // Given a list of heaps that are pending usage, this function will estimate memory needed,
235 // evict resources until enough space is available, then make resident any heaps scheduled for
236 // usage.
237 MaybeError ResidencyManager::EnsureHeapsAreResident(Heap** heaps, size_t heapCount) {
238 if (!mResidencyManagementEnabled) {
239 return {};
240 }
241
Brandon Jones0395ca92020-06-03 17:04:45 +0000242 std::vector<ID3D12Pageable*> localHeapsToMakeResident;
243 std::vector<ID3D12Pageable*> nonLocalHeapsToMakeResident;
Brandon Jones635239f2020-04-23 21:50:32 +0000244 uint64_t localSizeToMakeResident = 0;
245 uint64_t nonLocalSizeToMakeResident = 0;
Brandon Jones7982cc02020-03-31 15:31:56 +0000246
Corentin Wallez62139fc2020-09-28 19:35:14 +0000247 ExecutionSerial pendingCommandSerial = mDevice->GetPendingCommandSerial();
Brandon Jones7982cc02020-03-31 15:31:56 +0000248 for (size_t i = 0; i < heapCount; i++) {
249 Heap* heap = heaps[i];
250
Brandon Jones7982cc02020-03-31 15:31:56 +0000251 // Heaps that are locked resident are not tracked in the LRU cache.
252 if (heap->IsResidencyLocked()) {
253 continue;
254 }
255
256 if (heap->IsInResidencyLRUCache()) {
257 // If the heap is already in the LRU, we must remove it and append again below to
258 // update its position in the LRU.
259 heap->RemoveFromList();
260 } else {
Brandon Jones635239f2020-04-23 21:50:32 +0000261 if (heap->GetMemorySegment() == MemorySegment::Local) {
262 localSizeToMakeResident += heap->GetSize();
Brandon Jones0395ca92020-06-03 17:04:45 +0000263 localHeapsToMakeResident.push_back(heap->GetD3D12Pageable());
Brandon Jones635239f2020-04-23 21:50:32 +0000264 } else {
265 nonLocalSizeToMakeResident += heap->GetSize();
Brandon Jones0395ca92020-06-03 17:04:45 +0000266 nonLocalHeapsToMakeResident.push_back(heap->GetD3D12Pageable());
Brandon Jones635239f2020-04-23 21:50:32 +0000267 }
Brandon Jones7982cc02020-03-31 15:31:56 +0000268 }
269
Brandon Jones7be1d842020-04-09 23:28:22 +0000270 // If we submit a command list to the GPU, we must ensure that heaps referenced by that
271 // command list stay resident at least until that command list has finished execution.
272 // Setting this serial unnecessarily can leave the LRU in a state where nothing is
273 // eligible for eviction, even though some evictions may be possible.
Brandon Jones7982cc02020-03-31 15:31:56 +0000274 heap->SetLastSubmission(pendingCommandSerial);
Brandon Jones7be1d842020-04-09 23:28:22 +0000275
Brandon Jones635239f2020-04-23 21:50:32 +0000276 // Insert the heap into the appropriate LRU.
277 TrackResidentAllocation(heap);
278 }
279
280 if (localSizeToMakeResident > 0) {
Brandon Jones0395ca92020-06-03 17:04:45 +0000281 return MakeAllocationsResident(&mVideoMemoryInfo.local, localSizeToMakeResident,
282 localHeapsToMakeResident.size(),
283 localHeapsToMakeResident.data());
Brandon Jones635239f2020-04-23 21:50:32 +0000284 }
285
286 if (nonLocalSizeToMakeResident > 0) {
287 ASSERT(!mDevice->GetDeviceInfo().isUMA);
Brandon Jones0395ca92020-06-03 17:04:45 +0000288 return MakeAllocationsResident(&mVideoMemoryInfo.nonLocal, nonLocalSizeToMakeResident,
289 nonLocalHeapsToMakeResident.size(),
290 nonLocalHeapsToMakeResident.data());
Brandon Jones7982cc02020-03-31 15:31:56 +0000291 }
292
Brandon Jones0395ca92020-06-03 17:04:45 +0000293 return {};
294 }
295
296 MaybeError ResidencyManager::MakeAllocationsResident(MemorySegmentInfo* segment,
297 uint64_t sizeToMakeResident,
298 uint64_t numberOfObjectsToMakeResident,
299 ID3D12Pageable** allocations) {
300 uint64_t bytesEvicted;
301 DAWN_TRY_ASSIGN(bytesEvicted, EnsureCanMakeResident(sizeToMakeResident, segment));
Peter Kasting7e50a7f2021-07-27 18:42:59 +0000302 DAWN_UNUSED(bytesEvicted);
Brandon Jones0395ca92020-06-03 17:04:45 +0000303
304 // Note that MakeResident is a synchronous function and can add a significant
305 // overhead to command recording. In the future, it may be possible to decrease this
306 // overhead by using MakeResident on a secondary thread, or by instead making use of
307 // the EnqueueMakeResident function (which is not available on all Windows 10
308 // platforms).
309 HRESULT hr =
310 mDevice->GetD3D12Device()->MakeResident(numberOfObjectsToMakeResident, allocations);
311
312 // A MakeResident call can fail if there's not enough available memory. This
313 // could occur when there's significant fragmentation or if the allocation size
314 // estimates are incorrect. We may be able to continue execution by evicting some
315 // more memory and calling MakeResident again.
316 while (FAILED(hr)) {
317 constexpr uint32_t kAdditonalSizeToEvict = 50000000; // 50MB
318
319 uint64_t sizeEvicted = 0;
320
321 DAWN_TRY_ASSIGN(sizeEvicted, EnsureCanMakeResident(kAdditonalSizeToEvict, segment));
322
323 // If nothing can be evicted after MakeResident has failed, we cannot continue
324 // execution and must throw a fatal error.
325 if (sizeEvicted == 0) {
326 return DAWN_OUT_OF_MEMORY_ERROR(
327 "MakeResident has failed due to excessive video memory usage.");
328 }
329
330 hr =
331 mDevice->GetD3D12Device()->MakeResident(numberOfObjectsToMakeResident, allocations);
Brandon Jones7982cc02020-03-31 15:31:56 +0000332 }
333
334 return {};
335 }
336
Brandon Jones635239f2020-04-23 21:50:32 +0000337 // Inserts a heap at the bottom of the LRU. The passed heap must be resident or scheduled to
Brandon Jones94350682020-06-12 08:17:41 +0000338 // become resident within the current serial. Failing to call this function when an allocation
339 // is implicitly made resident will cause the residency manager to view the allocation as
340 // non-resident and call MakeResident - which will make D3D12's internal residency refcount on
341 // the allocation out of sync with Dawn.
Brandon Jones9b544662020-05-19 10:15:12 +0000342 void ResidencyManager::TrackResidentAllocation(Pageable* pageable) {
Brandon Jones7982cc02020-03-31 15:31:56 +0000343 if (!mResidencyManagementEnabled) {
344 return;
345 }
346
Brandon Jones9b544662020-05-19 10:15:12 +0000347 ASSERT(pageable->IsInList() == false);
348 GetMemorySegmentInfo(pageable->GetMemorySegment())->lruCache.Append(pageable);
Brandon Jones7982cc02020-03-31 15:31:56 +0000349 }
Brandon Jonesab2c84f2020-04-09 23:31:02 +0000350
351 // Places an artifical cap on Dawn's budget so we can test in a predictable manner. If used,
352 // this function must be called before any resources have been created.
353 void ResidencyManager::RestrictBudgetForTesting(uint64_t artificialBudgetCap) {
Brandon Jones635239f2020-04-23 21:50:32 +0000354 ASSERT(mVideoMemoryInfo.nonLocal.lruCache.empty());
Brandon Jonesab2c84f2020-04-09 23:31:02 +0000355 ASSERT(!mRestrictBudgetForTesting);
356
357 mRestrictBudgetForTesting = true;
358 UpdateVideoMemoryInfo();
359
360 // Dawn has a non-zero memory usage even before any resources have been created, and this
361 // value can vary depending on the environment Dawn is running in. By adding this in
362 // addition to the artificial budget cap, we can create a predictable and reproducible
363 // budget for testing.
Brandon Jonesf56f1902020-04-22 21:57:00 +0000364 mVideoMemoryInfo.local.budget = mVideoMemoryInfo.local.usage + artificialBudgetCap;
365 if (!mDevice->GetDeviceInfo().isUMA) {
366 mVideoMemoryInfo.nonLocal.budget =
367 mVideoMemoryInfo.nonLocal.usage + artificialBudgetCap;
368 }
Brandon Jonesab2c84f2020-04-09 23:31:02 +0000369 }
370
Kai Ninomiya63283562020-07-10 18:19:38 +0000371}} // namespace dawn_native::d3d12