bevy_picking/backend.rs
1//! This module provides a simple interface for implementing a picking backend.
2//!
3//! Don't be dissuaded by terminology like "backend"; the idea is dead simple. `bevy_picking`
4//! will tell you where pointers are, all you have to do is send an event if the pointers are
5//! hitting something. That's it. The rest of this documentation explains the requirements in more
6//! detail.
7//!
8//! Because `bevy_picking` is very loosely coupled with its backends, you can mix and match as
9//! many backends as you want. For example, you could use the `rapier` backend to raycast against
10//! physics objects, a picking shader backend to pick non-physics meshes, and the `bevy_ui` backend
11//! for your UI. The [`PointerHits`] instances produced by these various backends will be combined,
12//! sorted, and used as a homogeneous input for the picking systems that consume these events.
13//!
14//! ## Implementation
15//!
16//! - A picking backend only has one job: read [`PointerLocation`](crate::pointer::PointerLocation)
17//! components and produce [`PointerHits`] events. In plain English, a backend is provided the
18//! location of pointers, and is asked to provide a list of entities under those pointers.
19//!
20//! - The [`PointerHits`] events produced by a backend do **not** need to be sorted or filtered, all
21//! that is needed is an unordered list of entities and their [`HitData`].
22//!
23//! - Backends do not need to consider the [`Pickable`](crate::Pickable) component, though they may
24//! use it for optimization purposes. For example, a backend that traverses a spatial hierarchy
25//! may want to exit early if it intersects an entity that blocks lower entities from being
26//! picked.
27//!
28//! ### Raycasting Backends
29//!
30//! Backends that require a ray to cast into the scene should use [`ray::RayMap`]. This
31//! automatically constructs rays in world space for all cameras and pointers, handling details like
32//! viewports and DPI for you.
33
34use alloc::sync::Arc;
35use core::{any::Any, fmt};
36
37use bevy_ecs::prelude::*;
38use bevy_math::Vec3;
39use bevy_reflect::Reflect;
40
41/// The picking backend prelude.
42///
43/// This includes the most common types in this module, re-exported for your convenience.
44pub mod prelude {
45 pub use super::{ray::RayMap, HitData, HitDataExtra, PointerHits};
46 pub use crate::{
47 pointer::{PointerId, PointerLocation},
48 Pickable, PickingSystems,
49 };
50}
51
52/// Extra data attached to a [`HitData`] by a picking backend.
53///
54/// Use this for backend-specific data like triangle indices, UVs, or material
55/// information.
56/// Any `Send + Sync + fmt::Debug + 'static` type implements this trait
57/// automatically. `Clone` is not required: extra data is stored in an [`Arc`],
58/// so [`HitData`] can still implement [`Clone`]. `Clone` requires knowing the
59/// size of the type, which is not possible with dynamically dispatched types,
60/// so it cannot be used for `dyn HitDataExtra`.
61///
62/// ```rust
63/// #[derive(Debug)]
64/// struct MyHitInfo { triangle_index: u32 }
65/// ```
66///
67/// Read it back with [`HitData::extra_as`]:
68///
69/// ```rust
70/// # use bevy_picking::backend::HitData;
71/// # #[derive(Debug)] struct MyHitInfo { triangle_index: u32 }
72/// fn read_extra(hit: &HitData) {
73/// if let Some(info) = hit.extra_as::<MyHitInfo>() {
74/// println!("Hit triangle {}", info.triangle_index);
75/// }
76/// }
77/// ```
78pub trait HitDataExtra: Any + Send + Sync + fmt::Debug {}
79
80impl<T: Send + Sync + fmt::Debug + Any + 'static> HitDataExtra for T {}
81
82/// A message produced by a picking backend after it has run its hit tests, describing the entities
83/// under a pointer.
84///
85/// Some backends may only support providing the topmost entity; this is a valid limitation. For
86/// example, a picking shader might only have data on the topmost rendered output from its buffer.
87///
88/// Note that systems reading these messages in [`PreUpdate`](bevy_app::PreUpdate) will not report ordering
89/// ambiguities with picking backends. Take care to ensure such systems are explicitly ordered
90/// against [`PickingSystems::Backend`](crate::PickingSystems::Backend), or better, avoid reading `PointerHits` in `PreUpdate`.
91#[derive(Message, Debug, Clone, Reflect)]
92#[reflect(Debug, Clone)]
93pub struct PointerHits {
94 /// The pointer associated with this hit test.
95 pub pointer: prelude::PointerId,
96 /// An unordered collection of entities and their distance (depth) from the cursor.
97 pub picks: Vec<(Entity, HitData)>,
98 /// Set the order of this group of picks. Normally, this is the
99 /// [`bevy_camera::Camera::order`].
100 ///
101 /// Used to allow multiple `PointerHits` submitted for the same pointer to be ordered.
102 /// `PointerHits` with a higher `order` will be checked before those with a lower `order`,
103 /// regardless of the depth of each entity pick.
104 ///
105 /// In other words, when pick data is coalesced across all backends, the data is grouped by
106 /// pointer, then sorted by order, and checked sequentially, sorting each `PointerHits` by
107 /// entity depth. Events with a higher `order` are effectively on top of events with a lower
108 /// order.
109 ///
110 /// ### Why is this an `f32`???
111 ///
112 /// Bevy UI is special in that it can share a camera with other things being rendered. in order
113 /// to properly sort them, we need a way to make `bevy_ui`'s order a tiny bit higher, like adding
114 /// 0.5 to the order. We can't use integers, and we want users to be using camera.order by
115 /// default, so this is the best solution at the moment.
116 pub order: f32,
117}
118
119impl PointerHits {
120 /// Construct [`PointerHits`].
121 pub fn new(pointer: prelude::PointerId, picks: Vec<(Entity, HitData)>, order: f32) -> Self {
122 Self {
123 pointer,
124 picks,
125 order,
126 }
127 }
128}
129
130/// Holds data from a successful pointer hit test. See [`HitData::depth`] for important details.
131///
132/// Backends can attach arbitrary typed data via [`HitData::extra`]. See [`HitDataExtra`].
133#[derive(Debug, Reflect)]
134#[reflect(Debug)]
135pub struct HitData {
136 /// The camera entity used to detect this hit. Useful when you need to find the ray that was
137 /// cast for this hit when using a raycasting backend.
138 pub camera: Entity,
139 /// `depth` only needs to be self-consistent with other [`PointerHits`]s using the same
140 /// [`RenderTarget`](bevy_camera::RenderTarget). However, it is recommended to use the
141 /// distance from the pointer to the hit, measured from the near plane of the camera, to the
142 /// point, in world space.
143 pub depth: f32,
144 /// The position reported by the backend, if the data is available. Position data may be in any
145 /// space (e.g. World space, Screen space, Local space), specified by the backend providing it.
146 pub position: Option<Vec3>,
147 /// The normal vector of the hit test, if the data is available from the backend.
148 pub normal: Option<Vec3>,
149 /// Optional backend-specific extra data attached to this hit. Read it with [`HitData::extra_as`].
150 ///
151 /// This is stored in an [`Arc`] so cloning [`HitData`] stays cheap. This field is excluded
152 /// from [`PartialEq`] because value equality for trait objects would require extra dynamic
153 /// downcasting that the picking pipeline does not need.
154 #[reflect(ignore)]
155 pub extra: Option<Arc<dyn HitDataExtra>>,
156}
157
158impl Clone for HitData {
159 fn clone(&self) -> Self {
160 Self {
161 camera: self.camera,
162 depth: self.depth,
163 position: self.position,
164 normal: self.normal,
165 extra: self.extra.as_ref().map(Arc::clone),
166 }
167 }
168}
169
170impl PartialEq for HitData {
171 fn eq(&self, other: &Self) -> bool {
172 self.camera == other.camera
173 && self.depth == other.depth
174 && self.position == other.position
175 && self.normal == other.normal
176 }
177}
178
179impl HitData {
180 /// Construct a [`HitData`].
181 pub fn new(camera: Entity, depth: f32, position: Option<Vec3>, normal: Option<Vec3>) -> Self {
182 Self {
183 camera,
184 depth,
185 position,
186 normal,
187 extra: None,
188 }
189 }
190
191 /// Returns any attached extra data as `T` if available.
192 ///
193 /// This returns `None` if no extra data was attached, or if the hit stores a
194 /// different concrete extra data type.
195 pub fn extra_as<T: Any>(&self) -> Option<&T> {
196 let extra: &dyn Any = self.extra.as_deref()?;
197 extra.downcast_ref::<T>()
198 }
199
200 /// Creates a [`HitData`] with backend-specific extra data. `extra` can be
201 /// any [`HitDataExtra`].
202 ///
203 /// # Example
204 ///
205 /// ```rust
206 /// # use bevy_ecs::prelude::*;
207 /// # use bevy_picking::backend::HitData;
208 /// #[derive(Debug)]
209 /// struct MyHitInfo { triangle_index: u32 }
210 ///
211 /// # let camera = Entity::PLACEHOLDER;
212 /// let hit = HitData::new_with_extra(camera, 1.0, None, None, MyHitInfo { triangle_index: 7 });
213 /// ```
214 pub fn new_with_extra(
215 camera: Entity,
216 depth: f32,
217 position: Option<Vec3>,
218 normal: Option<Vec3>,
219 extra: impl HitDataExtra,
220 ) -> Self {
221 Self {
222 camera,
223 depth,
224 position,
225 normal,
226 extra: Some(Arc::new(extra)),
227 }
228 }
229}
230
231pub mod ray {
232 //! Types and systems for constructing rays from cameras and pointers.
233
234 use crate::backend::prelude::{PointerId, PointerLocation};
235 use bevy_camera::{Camera, RenderTarget};
236 use bevy_ecs::prelude::*;
237 use bevy_math::Ray3d;
238 use bevy_platform::collections::{hash_map::Iter, HashMap};
239 use bevy_reflect::Reflect;
240 use bevy_transform::prelude::GlobalTransform;
241 use bevy_window::PrimaryWindow;
242
243 /// Identifies a ray constructed from some (pointer, camera) combination. A pointer can be over
244 /// multiple cameras, which is why a single pointer may have multiple rays.
245 #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Reflect)]
246 #[reflect(Clone, PartialEq, Hash)]
247 pub struct RayId {
248 /// The camera whose projection was used to calculate the ray.
249 pub camera: Entity,
250 /// The pointer whose pixel coordinates were used to calculate the ray.
251 pub pointer: PointerId,
252 }
253
254 impl RayId {
255 /// Construct a [`RayId`].
256 pub fn new(camera: Entity, pointer: PointerId) -> Self {
257 Self { camera, pointer }
258 }
259 }
260
261 /// A map from [`RayId`] to [`Ray3d`].
262 ///
263 /// This map is cleared and re-populated every frame before any backends run. Ray-based picking
264 /// backends should use this when possible, as it automatically handles viewports, DPI, and
265 /// other details of building rays from pointer locations.
266 ///
267 /// ## Usage
268 ///
269 /// Iterate over each [`Ray3d`] and its [`RayId`] with [`RayMap::iter`].
270 ///
271 /// ```
272 /// # use bevy_ecs::prelude::*;
273 /// # use bevy_picking::backend::ray::RayMap;
274 /// # use bevy_picking::backend::PointerHits;
275 /// // My raycasting backend
276 /// pub fn update_hits(ray_map: Res<RayMap>, mut output_messages: MessageWriter<PointerHits>,) {
277 /// for (&ray_id, &ray) in ray_map.iter() {
278 /// // Run a raycast with each ray, returning any `PointerHits` found.
279 /// }
280 /// }
281 /// ```
282 #[derive(Clone, Debug, Default, Resource)]
283 pub struct RayMap {
284 /// Cartesian product of all pointers and all cameras
285 /// Add your rays here to support picking through indirections,
286 /// e.g. rendered-to-texture cameras
287 pub map: HashMap<RayId, Ray3d>,
288 }
289
290 impl RayMap {
291 /// Iterates over all world space rays for every picking pointer.
292 pub fn iter(&self) -> Iter<'_, RayId, Ray3d> {
293 self.map.iter()
294 }
295
296 /// Clears the [`RayMap`] and re-populates it with one ray for each
297 /// combination of pointer entity and camera entity where the pointer
298 /// intersects the camera's viewport.
299 pub fn repopulate(
300 mut ray_map: ResMut<Self>,
301 primary_window_entity: Query<Entity, With<PrimaryWindow>>,
302 cameras: Query<(Entity, &Camera, &RenderTarget, &GlobalTransform)>,
303 pointers: Query<(&PointerId, &PointerLocation)>,
304 ) {
305 ray_map.map.clear();
306
307 for (camera_entity, camera, render_target, camera_tfm) in &cameras {
308 if !camera.is_active {
309 continue;
310 }
311
312 for (&pointer_id, pointer_loc) in &pointers {
313 if let Some(ray) = make_ray(
314 &primary_window_entity,
315 camera,
316 render_target,
317 camera_tfm,
318 pointer_loc,
319 ) {
320 ray_map
321 .map
322 .insert(RayId::new(camera_entity, pointer_id), ray);
323 }
324 }
325 }
326 }
327 }
328
329 fn make_ray(
330 primary_window_entity: &Query<Entity, With<PrimaryWindow>>,
331 camera: &Camera,
332 render_target: &RenderTarget,
333 camera_tfm: &GlobalTransform,
334 pointer_loc: &PointerLocation,
335 ) -> Option<Ray3d> {
336 let pointer_loc = pointer_loc.location()?;
337 if !pointer_loc.is_in_viewport(camera, render_target, primary_window_entity) {
338 return None;
339 }
340 camera
341 .viewport_to_world(camera_tfm, pointer_loc.position)
342 .ok()
343 }
344}
345
346#[cfg(test)]
347mod tests {
348 use super::*;
349
350 #[derive(Debug, PartialEq)]
351 struct TriangleHitInfo {
352 triangle_index: u32,
353 }
354
355 #[derive(Debug, PartialEq)]
356 struct OtherHitInfo {
357 triangle_index: u32,
358 }
359
360 #[test]
361 fn hit_data_extra() {
362 let camera = Entity::PLACEHOLDER;
363
364 let hit = HitData::new_with_extra(
365 camera,
366 1.0,
367 Some(Vec3::new(1.0, 2.0, 3.0)),
368 Some(Vec3::Y),
369 TriangleHitInfo { triangle_index: 7 },
370 );
371
372 assert_eq!(
373 hit.extra_as::<TriangleHitInfo>(),
374 Some(&TriangleHitInfo { triangle_index: 7 })
375 );
376 assert_eq!(hit.extra_as::<OtherHitInfo>(), None);
377
378 let cloned = hit.clone();
379 assert_eq!(
380 cloned.extra_as::<TriangleHitInfo>(),
381 Some(&TriangleHitInfo { triangle_index: 7 })
382 );
383
384 let other_extra = HitData::new_with_extra(
385 camera,
386 1.0,
387 Some(Vec3::new(1.0, 2.0, 3.0)),
388 Some(Vec3::Y),
389 TriangleHitInfo { triangle_index: 99 },
390 );
391
392 assert_eq!(hit, other_extra);
393 }
394}