wgpu_core/device/life.rs
1use alloc::{sync::Arc, vec::Vec};
2
3use smallvec::SmallVec;
4use thiserror::Error;
5
6use crate::{
7 device::{
8 queue::{EncoderInFlight, SubmittedWorkDoneClosure, TempResource},
9 DeviceError,
10 },
11 ray_tracing::BlasCompactReadyPendingClosure,
12 resource::{Blas, Buffer, Texture, Trackable},
13 snatch::SnatchGuard,
14 SubmissionIndex,
15};
16
17/// A command submitted to the GPU for execution.
18///
19/// ## Keeping resources alive while the GPU is using them
20///
21/// [`wgpu_hal`] requires that, when a command is submitted to a queue, all the
22/// resources it uses must remain alive until it has finished executing.
23///
24/// [`wgpu_hal`]: hal
25/// [`ResourceInfo::submission_index`]: crate::resource::ResourceInfo
26struct ActiveSubmission {
27 /// The index of the submission we track.
28 ///
29 /// When `Device::fence`'s value is greater than or equal to this, our queue
30 /// submission has completed.
31 index: SubmissionIndex,
32
33 /// Buffers to be mapped once this submission has completed.
34 mapped: Vec<Arc<Buffer>>,
35
36 /// BLASes to have their compacted size read back once this submission has completed.
37 compact_read_back: Vec<Arc<Blas>>,
38
39 /// Command buffers used by this submission, and the encoder that owns them.
40 ///
41 /// [`wgpu_hal::Queue::submit`] requires the submitted command buffers to
42 /// remain alive until the submission has completed execution. Command
43 /// encoders double as allocation pools for command buffers, so holding them
44 /// here and cleaning them up in [`LifetimeTracker::triage_submissions`]
45 /// satisfies that requirement.
46 ///
47 /// Once this submission has completed, the command buffers are reset and
48 /// the command encoder is recycled.
49 ///
50 /// [`wgpu_hal::Queue::submit`]: hal::Queue::submit
51 encoders: Vec<EncoderInFlight>,
52
53 /// List of queue "on_submitted_work_done" closures to be called once this
54 /// submission has completed.
55 work_done_closures: SmallVec<[SubmittedWorkDoneClosure; 1]>,
56}
57
58impl ActiveSubmission {
59 /// Returns true if this submission contains the given buffer.
60 ///
61 /// This only uses constant-time operations.
62 pub fn contains_buffer(&self, buffer: &Buffer) -> bool {
63 for encoder in &self.encoders {
64 // The ownership location of buffers depends on where the command encoder
65 // came from. If it is the staging command encoder on the queue, it is
66 // in the pending buffer list. If it came from a user command encoder,
67 // it is in the tracker.
68
69 if encoder.trackers.buffers.contains(buffer) {
70 return true;
71 }
72
73 if encoder
74 .pending_buffers
75 .contains_key(&buffer.tracker_index())
76 {
77 return true;
78 }
79 }
80
81 false
82 }
83
84 /// Returns true if this submission contains the given texture.
85 ///
86 /// This only uses constant-time operations.
87 pub fn contains_texture(&self, texture: &Texture) -> bool {
88 for encoder in &self.encoders {
89 // The ownership location of textures depends on where the command encoder
90 // came from. If it is the staging command encoder on the queue, it is
91 // in the pending buffer list. If it came from a user command encoder,
92 // it is in the tracker.
93
94 if encoder.trackers.textures.contains(texture) {
95 return true;
96 }
97
98 if encoder
99 .pending_textures
100 .contains_key(&texture.tracker_index())
101 {
102 return true;
103 }
104 }
105
106 false
107 }
108
109 /// Returns true if this submission contains the given blas.
110 ///
111 /// This only uses constant-time operations.
112 pub fn contains_blas(&self, blas: &Blas) -> bool {
113 for encoder in &self.encoders {
114 if encoder.trackers.blas_s.contains(blas) {
115 return true;
116 }
117
118 if encoder.pending_blas_s.contains_key(&blas.tracker_index()) {
119 return true;
120 }
121 }
122
123 false
124 }
125}
126
127#[derive(Clone, Debug, Error)]
128#[non_exhaustive]
129pub enum WaitIdleError {
130 #[error(transparent)]
131 Device(#[from] DeviceError),
132 #[error("Tried to wait using a submission index ({0}) that has not been returned by a successful submission (last successful submission: {1})")]
133 WrongSubmissionIndex(SubmissionIndex, SubmissionIndex),
134 #[error("Timed out trying to wait for the given submission index.")]
135 Timeout,
136}
137
138impl WaitIdleError {
139 pub fn to_poll_error(&self) -> Option<wgt::PollError> {
140 match self {
141 WaitIdleError::Timeout => Some(wgt::PollError::Timeout),
142 _ => None,
143 }
144 }
145}
146
147/// Resource tracking for a device.
148///
149/// ## Host mapping buffers
150///
151/// A buffer cannot be mapped until all active queue submissions that use it
152/// have completed. To that end:
153///
154/// - Each buffer's `ResourceInfo::submission_index` records the index of the
155/// most recent queue submission that uses that buffer.
156///
157/// - When the device is polled, the following `LifetimeTracker` methods decide
158/// what should happen next:
159///
160/// 1) `triage_submissions` moves entries in `self.active[i]` for completed
161/// submissions to `self.ready_to_map`. At this point, both
162/// `self.active` and `self.ready_to_map` are up to date with the given
163/// submission index.
164///
165/// 2) `handle_mapping` drains `self.ready_to_map` and actually maps the
166/// buffers, collecting a list of notification closures to call.
167///
168/// Only calling `Global::buffer_map_async` clones a new `Arc` for the
169/// buffer. This new `Arc` is only dropped by `handle_mapping`.
170pub(crate) struct LifetimeTracker {
171 /// Resources used by queue submissions still in flight. One entry per
172 /// submission, with older submissions appearing before younger.
173 ///
174 /// Entries are added by `track_submission` and drained by
175 /// `LifetimeTracker::triage_submissions`. Lots of methods contribute data
176 /// to particular entries.
177 active: Vec<ActiveSubmission>,
178
179 /// Buffers the user has asked us to map, and which are not used by any
180 /// queue submission still in flight.
181 ready_to_map: Vec<Arc<Buffer>>,
182
183 /// BLASes the user has asked us to prepare to compact, and which are not used by any
184 /// queue submission still in flight.
185 ready_to_compact: Vec<Arc<Blas>>,
186
187 /// Queue "on_submitted_work_done" closures that were initiated for while there is no
188 /// currently pending submissions. These cannot be immediately invoked as they
189 /// must happen _after_ all mapped buffer callbacks are mapped, so we defer them
190 /// here until the next time the device is maintained.
191 work_done_closures: SmallVec<[SubmittedWorkDoneClosure; 1]>,
192}
193
194impl LifetimeTracker {
195 pub fn new() -> Self {
196 Self {
197 active: Vec::new(),
198 ready_to_map: Vec::new(),
199 ready_to_compact: Vec::new(),
200 work_done_closures: SmallVec::new(),
201 }
202 }
203
204 /// Return true if there are no queue submissions still in flight.
205 pub fn queue_empty(&self) -> bool {
206 self.active.is_empty()
207 }
208
209 /// Start tracking resources associated with a new queue submission.
210 pub fn track_submission(&mut self, index: SubmissionIndex, encoders: Vec<EncoderInFlight>) {
211 self.active.push(ActiveSubmission {
212 index,
213 mapped: Vec::new(),
214 compact_read_back: Vec::new(),
215 encoders,
216 work_done_closures: SmallVec::new(),
217 });
218 }
219
220 pub(crate) fn map(&mut self, buffer: &Arc<Buffer>) -> Option<SubmissionIndex> {
221 // Determine which buffers are ready to map, and which must wait for the GPU.
222 let submission = self
223 .active
224 .iter_mut()
225 .rev()
226 .find(|a| a.contains_buffer(buffer));
227
228 let maybe_submission_index = submission.as_ref().map(|s| s.index);
229
230 submission
231 .map_or(&mut self.ready_to_map, |a| &mut a.mapped)
232 .push(buffer.clone());
233
234 maybe_submission_index
235 }
236
237 pub(crate) fn prepare_compact(&mut self, blas: &Arc<Blas>) -> Option<SubmissionIndex> {
238 // Determine which BLASes are ready to map, and which must wait for the GPU.
239 let submission = self.active.iter_mut().rev().find(|a| a.contains_blas(blas));
240
241 let maybe_submission_index = submission.as_ref().map(|s| s.index);
242
243 submission
244 .map_or(&mut self.ready_to_compact, |a| &mut a.compact_read_back)
245 .push(blas.clone());
246
247 maybe_submission_index
248 }
249
250 /// Returns the submission index of the most recent submission that uses the
251 /// given buffer.
252 pub fn get_buffer_latest_submission_index(&self, buffer: &Buffer) -> Option<SubmissionIndex> {
253 // We iterate in reverse order, so that we can bail out early as soon
254 // as we find a hit.
255 self.active.iter().rev().find_map(|submission| {
256 if submission.contains_buffer(buffer) {
257 Some(submission.index)
258 } else {
259 None
260 }
261 })
262 }
263
264 /// Returns the submission index of the most recent submission that uses the
265 /// given texture.
266 pub fn get_texture_latest_submission_index(
267 &self,
268 texture: &Texture,
269 ) -> Option<SubmissionIndex> {
270 // We iterate in reverse order, so that we can bail out early as soon
271 // as we find a hit.
272 self.active.iter().rev().find_map(|submission| {
273 if submission.contains_texture(texture) {
274 Some(submission.index)
275 } else {
276 None
277 }
278 })
279 }
280
281 /// Sort out the consequences of completed submissions.
282 ///
283 /// Assume that all submissions up through `last_done` have completed.
284 ///
285 /// - Buffers used by those submissions are now ready to map, if requested.
286 /// Add any buffers in the submission's [`mapped`] list to
287 /// [`self.ready_to_map`], where [`LifetimeTracker::handle_mapping`]
288 /// will find them.
289 ///
290 /// Return a list of [`SubmittedWorkDoneClosure`]s to run.
291 ///
292 /// [`mapped`]: ActiveSubmission::mapped
293 /// [`self.ready_to_map`]: LifetimeTracker::ready_to_map
294 /// [`SubmittedWorkDoneClosure`]: crate::device::queue::SubmittedWorkDoneClosure
295 #[must_use]
296 pub fn triage_submissions(
297 &mut self,
298 last_done: SubmissionIndex,
299 ) -> SmallVec<[SubmittedWorkDoneClosure; 1]> {
300 profiling::scope!("triage_submissions");
301
302 //TODO: enable when `is_sorted_by_key` is stable
303 //debug_assert!(self.active.is_sorted_by_key(|a| a.index));
304 let done_count = self
305 .active
306 .iter()
307 .position(|a| a.index > last_done)
308 .unwrap_or(self.active.len());
309
310 let mut work_done_closures: SmallVec<_> = self.work_done_closures.drain(..).collect();
311 for a in self.active.drain(..done_count) {
312 self.ready_to_map.extend(a.mapped);
313 self.ready_to_compact.extend(a.compact_read_back);
314 for encoder in a.encoders {
315 // This involves actually decrementing the ref count of all command buffer
316 // resources, so can be _very_ expensive.
317 profiling::scope!("drop command buffer trackers");
318 drop(encoder);
319 }
320 work_done_closures.extend(a.work_done_closures);
321 }
322 work_done_closures
323 }
324
325 pub fn schedule_resource_destruction(
326 &mut self,
327 temp_resource: TempResource,
328 last_submit_index: SubmissionIndex,
329 ) {
330 let resources = self
331 .active
332 .iter_mut()
333 .find(|a| a.index == last_submit_index)
334 .map(|a| {
335 // Because this resource's `last_submit_index` matches `a.index`,
336 // we know that we must have done something with the resource,
337 // so `a.encoders` should not be empty.
338 &mut a.encoders.last_mut().unwrap().temp_resources
339 });
340 if let Some(resources) = resources {
341 resources.push(temp_resource);
342 }
343 }
344
345 pub fn add_work_done_closure(
346 &mut self,
347 closure: SubmittedWorkDoneClosure,
348 ) -> Option<SubmissionIndex> {
349 match self.active.last_mut() {
350 Some(active) => {
351 active.work_done_closures.push(closure);
352 Some(active.index)
353 }
354 // We must defer the closure until all previously occurring map_async closures
355 // have fired. This is required by the spec.
356 None => {
357 self.work_done_closures.push(closure);
358 None
359 }
360 }
361 }
362
363 /// Map the buffers in `self.ready_to_map`.
364 ///
365 /// Return a list of mapping notifications to send.
366 ///
367 /// See the documentation for [`LifetimeTracker`] for details.
368 #[must_use]
369 pub(crate) fn handle_mapping(
370 &mut self,
371 snatch_guard: &SnatchGuard,
372 ) -> Vec<super::BufferMapPendingClosure> {
373 if self.ready_to_map.is_empty() {
374 return Vec::new();
375 }
376 let mut pending_callbacks: Vec<super::BufferMapPendingClosure> =
377 Vec::with_capacity(self.ready_to_map.len());
378
379 for buffer in self.ready_to_map.drain(..) {
380 match buffer.map(snatch_guard) {
381 Some(cb) => pending_callbacks.push(cb),
382 None => continue,
383 }
384 }
385 pending_callbacks
386 }
387 /// Read back compact sizes from the BLASes in `self.ready_to_compact`.
388 ///
389 /// Return a list of mapping notifications to send.
390 ///
391 /// See the documentation for [`LifetimeTracker`] for details.
392 #[must_use]
393 pub(crate) fn handle_compact_read_back(&mut self) -> Vec<BlasCompactReadyPendingClosure> {
394 if self.ready_to_compact.is_empty() {
395 return Vec::new();
396 }
397 let mut pending_callbacks: Vec<BlasCompactReadyPendingClosure> =
398 Vec::with_capacity(self.ready_to_compact.len());
399
400 for blas in self.ready_to_compact.drain(..) {
401 match blas.read_back_compact_size() {
402 Some(cb) => pending_callbacks.push(cb),
403 None => continue,
404 }
405 }
406 pending_callbacks
407 }
408}