wgpu_core/device/
life.rs

1use alloc::{sync::Arc, vec::Vec};
2
3use smallvec::SmallVec;
4use thiserror::Error;
5
6use crate::{
7    device::{
8        queue::{EncoderInFlight, SubmittedWorkDoneClosure, TempResource},
9        DeviceError,
10    },
11    ray_tracing::BlasCompactReadyPendingClosure,
12    resource::{Blas, Buffer, Texture, Trackable},
13    snatch::SnatchGuard,
14    SubmissionIndex,
15};
16
17/// A command submitted to the GPU for execution.
18///
19/// ## Keeping resources alive while the GPU is using them
20///
21/// [`wgpu_hal`] requires that, when a command is submitted to a queue, all the
22/// resources it uses must remain alive until it has finished executing.
23///
24/// [`wgpu_hal`]: hal
25/// [`ResourceInfo::submission_index`]: crate::resource::ResourceInfo
26struct ActiveSubmission {
27    /// The index of the submission we track.
28    ///
29    /// When `Device::fence`'s value is greater than or equal to this, our queue
30    /// submission has completed.
31    index: SubmissionIndex,
32
33    /// Buffers to be mapped once this submission has completed.
34    mapped: Vec<Arc<Buffer>>,
35
36    /// BLASes to have their compacted size read back once this submission has completed.
37    compact_read_back: Vec<Arc<Blas>>,
38
39    /// Command buffers used by this submission, and the encoder that owns them.
40    ///
41    /// [`wgpu_hal::Queue::submit`] requires the submitted command buffers to
42    /// remain alive until the submission has completed execution. Command
43    /// encoders double as allocation pools for command buffers, so holding them
44    /// here and cleaning them up in [`LifetimeTracker::triage_submissions`]
45    /// satisfies that requirement.
46    ///
47    /// Once this submission has completed, the command buffers are reset and
48    /// the command encoder is recycled.
49    ///
50    /// [`wgpu_hal::Queue::submit`]: hal::Queue::submit
51    encoders: Vec<EncoderInFlight>,
52
53    /// List of queue "on_submitted_work_done" closures to be called once this
54    /// submission has completed.
55    work_done_closures: SmallVec<[SubmittedWorkDoneClosure; 1]>,
56}
57
58impl ActiveSubmission {
59    /// Returns true if this submission contains the given buffer.
60    ///
61    /// This only uses constant-time operations.
62    pub fn contains_buffer(&self, buffer: &Buffer) -> bool {
63        for encoder in &self.encoders {
64            // The ownership location of buffers depends on where the command encoder
65            // came from. If it is the staging command encoder on the queue, it is
66            // in the pending buffer list. If it came from a user command encoder,
67            // it is in the tracker.
68
69            if encoder.trackers.buffers.contains(buffer) {
70                return true;
71            }
72
73            if encoder
74                .pending_buffers
75                .contains_key(&buffer.tracker_index())
76            {
77                return true;
78            }
79        }
80
81        false
82    }
83
84    /// Returns true if this submission contains the given texture.
85    ///
86    /// This only uses constant-time operations.
87    pub fn contains_texture(&self, texture: &Texture) -> bool {
88        for encoder in &self.encoders {
89            // The ownership location of textures depends on where the command encoder
90            // came from. If it is the staging command encoder on the queue, it is
91            // in the pending buffer list. If it came from a user command encoder,
92            // it is in the tracker.
93
94            if encoder.trackers.textures.contains(texture) {
95                return true;
96            }
97
98            if encoder
99                .pending_textures
100                .contains_key(&texture.tracker_index())
101            {
102                return true;
103            }
104        }
105
106        false
107    }
108
109    /// Returns true if this submission contains the given blas.
110    ///
111    /// This only uses constant-time operations.
112    pub fn contains_blas(&self, blas: &Blas) -> bool {
113        for encoder in &self.encoders {
114            if encoder.trackers.blas_s.contains(blas) {
115                return true;
116            }
117
118            if encoder.pending_blas_s.contains_key(&blas.tracker_index()) {
119                return true;
120            }
121        }
122
123        false
124    }
125}
126
127#[derive(Clone, Debug, Error)]
128#[non_exhaustive]
129pub enum WaitIdleError {
130    #[error(transparent)]
131    Device(#[from] DeviceError),
132    #[error("Tried to wait using a submission index ({0}) that has not been returned by a successful submission (last successful submission: {1})")]
133    WrongSubmissionIndex(SubmissionIndex, SubmissionIndex),
134    #[error("Timed out trying to wait for the given submission index.")]
135    Timeout,
136}
137
138impl WaitIdleError {
139    pub fn to_poll_error(&self) -> Option<wgt::PollError> {
140        match self {
141            WaitIdleError::Timeout => Some(wgt::PollError::Timeout),
142            _ => None,
143        }
144    }
145}
146
147/// Resource tracking for a device.
148///
149/// ## Host mapping buffers
150///
151/// A buffer cannot be mapped until all active queue submissions that use it
152/// have completed. To that end:
153///
154/// -   Each buffer's `ResourceInfo::submission_index` records the index of the
155///     most recent queue submission that uses that buffer.
156///
157/// -   When the device is polled, the following `LifetimeTracker` methods decide
158///     what should happen next:
159///
160///     1)  `triage_submissions` moves entries in `self.active[i]` for completed
161///         submissions to `self.ready_to_map`.  At this point, both
162///         `self.active` and `self.ready_to_map` are up to date with the given
163///         submission index.
164///
165///     2)  `handle_mapping` drains `self.ready_to_map` and actually maps the
166///         buffers, collecting a list of notification closures to call.
167///
168/// Only calling `Global::buffer_map_async` clones a new `Arc` for the
169/// buffer. This new `Arc` is only dropped by `handle_mapping`.
170pub(crate) struct LifetimeTracker {
171    /// Resources used by queue submissions still in flight. One entry per
172    /// submission, with older submissions appearing before younger.
173    ///
174    /// Entries are added by `track_submission` and drained by
175    /// `LifetimeTracker::triage_submissions`. Lots of methods contribute data
176    /// to particular entries.
177    active: Vec<ActiveSubmission>,
178
179    /// Buffers the user has asked us to map, and which are not used by any
180    /// queue submission still in flight.
181    ready_to_map: Vec<Arc<Buffer>>,
182
183    /// BLASes the user has asked us to prepare to compact, and which are not used by any
184    /// queue submission still in flight.
185    ready_to_compact: Vec<Arc<Blas>>,
186
187    /// Queue "on_submitted_work_done" closures that were initiated for while there is no
188    /// currently pending submissions. These cannot be immediately invoked as they
189    /// must happen _after_ all mapped buffer callbacks are mapped, so we defer them
190    /// here until the next time the device is maintained.
191    work_done_closures: SmallVec<[SubmittedWorkDoneClosure; 1]>,
192}
193
194impl LifetimeTracker {
195    pub fn new() -> Self {
196        Self {
197            active: Vec::new(),
198            ready_to_map: Vec::new(),
199            ready_to_compact: Vec::new(),
200            work_done_closures: SmallVec::new(),
201        }
202    }
203
204    /// Return true if there are no queue submissions still in flight.
205    pub fn queue_empty(&self) -> bool {
206        self.active.is_empty()
207    }
208
209    /// Start tracking resources associated with a new queue submission.
210    pub fn track_submission(&mut self, index: SubmissionIndex, encoders: Vec<EncoderInFlight>) {
211        self.active.push(ActiveSubmission {
212            index,
213            mapped: Vec::new(),
214            compact_read_back: Vec::new(),
215            encoders,
216            work_done_closures: SmallVec::new(),
217        });
218    }
219
220    pub(crate) fn map(&mut self, buffer: &Arc<Buffer>) -> Option<SubmissionIndex> {
221        // Determine which buffers are ready to map, and which must wait for the GPU.
222        let submission = self
223            .active
224            .iter_mut()
225            .rev()
226            .find(|a| a.contains_buffer(buffer));
227
228        let maybe_submission_index = submission.as_ref().map(|s| s.index);
229
230        submission
231            .map_or(&mut self.ready_to_map, |a| &mut a.mapped)
232            .push(buffer.clone());
233
234        maybe_submission_index
235    }
236
237    pub(crate) fn prepare_compact(&mut self, blas: &Arc<Blas>) -> Option<SubmissionIndex> {
238        // Determine which BLASes are ready to map, and which must wait for the GPU.
239        let submission = self.active.iter_mut().rev().find(|a| a.contains_blas(blas));
240
241        let maybe_submission_index = submission.as_ref().map(|s| s.index);
242
243        submission
244            .map_or(&mut self.ready_to_compact, |a| &mut a.compact_read_back)
245            .push(blas.clone());
246
247        maybe_submission_index
248    }
249
250    /// Returns the submission index of the most recent submission that uses the
251    /// given buffer.
252    pub fn get_buffer_latest_submission_index(&self, buffer: &Buffer) -> Option<SubmissionIndex> {
253        // We iterate in reverse order, so that we can bail out early as soon
254        // as we find a hit.
255        self.active.iter().rev().find_map(|submission| {
256            if submission.contains_buffer(buffer) {
257                Some(submission.index)
258            } else {
259                None
260            }
261        })
262    }
263
264    /// Returns the submission index of the most recent submission that uses the
265    /// given texture.
266    pub fn get_texture_latest_submission_index(
267        &self,
268        texture: &Texture,
269    ) -> Option<SubmissionIndex> {
270        // We iterate in reverse order, so that we can bail out early as soon
271        // as we find a hit.
272        self.active.iter().rev().find_map(|submission| {
273            if submission.contains_texture(texture) {
274                Some(submission.index)
275            } else {
276                None
277            }
278        })
279    }
280
281    /// Sort out the consequences of completed submissions.
282    ///
283    /// Assume that all submissions up through `last_done` have completed.
284    ///
285    /// -   Buffers used by those submissions are now ready to map, if requested.
286    ///     Add any buffers in the submission's [`mapped`] list to
287    ///     [`self.ready_to_map`], where [`LifetimeTracker::handle_mapping`]
288    ///     will find them.
289    ///
290    /// Return a list of [`SubmittedWorkDoneClosure`]s to run.
291    ///
292    /// [`mapped`]: ActiveSubmission::mapped
293    /// [`self.ready_to_map`]: LifetimeTracker::ready_to_map
294    /// [`SubmittedWorkDoneClosure`]: crate::device::queue::SubmittedWorkDoneClosure
295    #[must_use]
296    pub fn triage_submissions(
297        &mut self,
298        last_done: SubmissionIndex,
299    ) -> SmallVec<[SubmittedWorkDoneClosure; 1]> {
300        profiling::scope!("triage_submissions");
301
302        //TODO: enable when `is_sorted_by_key` is stable
303        //debug_assert!(self.active.is_sorted_by_key(|a| a.index));
304        let done_count = self
305            .active
306            .iter()
307            .position(|a| a.index > last_done)
308            .unwrap_or(self.active.len());
309
310        let mut work_done_closures: SmallVec<_> = self.work_done_closures.drain(..).collect();
311        for a in self.active.drain(..done_count) {
312            self.ready_to_map.extend(a.mapped);
313            self.ready_to_compact.extend(a.compact_read_back);
314            for encoder in a.encoders {
315                // This involves actually decrementing the ref count of all command buffer
316                // resources, so can be _very_ expensive.
317                profiling::scope!("drop command buffer trackers");
318                drop(encoder);
319            }
320            work_done_closures.extend(a.work_done_closures);
321        }
322        work_done_closures
323    }
324
325    pub fn schedule_resource_destruction(
326        &mut self,
327        temp_resource: TempResource,
328        last_submit_index: SubmissionIndex,
329    ) {
330        let resources = self
331            .active
332            .iter_mut()
333            .find(|a| a.index == last_submit_index)
334            .map(|a| {
335                // Because this resource's `last_submit_index` matches `a.index`,
336                // we know that we must have done something with the resource,
337                // so `a.encoders` should not be empty.
338                &mut a.encoders.last_mut().unwrap().temp_resources
339            });
340        if let Some(resources) = resources {
341            resources.push(temp_resource);
342        }
343    }
344
345    pub fn add_work_done_closure(
346        &mut self,
347        closure: SubmittedWorkDoneClosure,
348    ) -> Option<SubmissionIndex> {
349        match self.active.last_mut() {
350            Some(active) => {
351                active.work_done_closures.push(closure);
352                Some(active.index)
353            }
354            // We must defer the closure until all previously occurring map_async closures
355            // have fired. This is required by the spec.
356            None => {
357                self.work_done_closures.push(closure);
358                None
359            }
360        }
361    }
362
363    /// Map the buffers in `self.ready_to_map`.
364    ///
365    /// Return a list of mapping notifications to send.
366    ///
367    /// See the documentation for [`LifetimeTracker`] for details.
368    #[must_use]
369    pub(crate) fn handle_mapping(
370        &mut self,
371        snatch_guard: &SnatchGuard,
372    ) -> Vec<super::BufferMapPendingClosure> {
373        if self.ready_to_map.is_empty() {
374            return Vec::new();
375        }
376        let mut pending_callbacks: Vec<super::BufferMapPendingClosure> =
377            Vec::with_capacity(self.ready_to_map.len());
378
379        for buffer in self.ready_to_map.drain(..) {
380            match buffer.map(snatch_guard) {
381                Some(cb) => pending_callbacks.push(cb),
382                None => continue,
383            }
384        }
385        pending_callbacks
386    }
387    /// Read back compact sizes from the BLASes in `self.ready_to_compact`.
388    ///
389    /// Return a list of mapping notifications to send.
390    ///
391    /// See the documentation for [`LifetimeTracker`] for details.
392    #[must_use]
393    pub(crate) fn handle_compact_read_back(&mut self) -> Vec<BlasCompactReadyPendingClosure> {
394        if self.ready_to_compact.is_empty() {
395            return Vec::new();
396        }
397        let mut pending_callbacks: Vec<BlasCompactReadyPendingClosure> =
398            Vec::with_capacity(self.ready_to_compact.len());
399
400        for blas in self.ready_to_compact.drain(..) {
401            match blas.read_back_compact_size() {
402                Some(cb) => pending_callbacks.push(cb),
403                None => continue,
404            }
405        }
406        pending_callbacks
407    }
408}