From c3c118c20c984db1cbba5acf1e6042bca5f7bba8 Mon Sep 17 00:00:00 2001 From: Martin Edlund Date: Thu, 23 Apr 2026 18:18:18 +0200 Subject: [PATCH] Add support for custom picking data (#23245) # Objective - Adds the ability for picking backends to add custom hit data - Fixes #16186 ## Solution - Adds an `extra` field to the HitData struct which can take any Data that implements HitDataExtra. This is stored in an Arc which can be downcast with a helper function in the system that consumes the hit. - In the original ticket I suggested using a generic, however this caused extensive code changes and complications down the line with the HoverMap and OverMap so I decided against it. ## Testing - I added an example custom_hit_data to test the new feature - I tested all other picking examples to ensure they still work as expected - I tested the examples on Ubuntu - I additionally tested the custom_hit_data example in wasm --- ## Showcase
Click to view showcase ### Creating custom hits: ```rust let picks: Vec<(Entity, HitData)> = ray_cast .cast_ray(ray, &settings) .iter() .map(|(entity, hit)| { let extra = TriangleHitInfo { triangle_vertices: hit.triangle, }; let hit_data = HitData::new_with_extra( ray_id.camera, hit.distance, Some(hit.point), Some(hit.normal), extra, ); (*entity, hit_data) }) .collect(); ``` ### Reading custom hits: ```rust for hits in pointer_hits.read() { for (_, hit) in &hits.picks { let Some(info) = hit.extra_as::() else { continue; }; let Some(vertices) = info.triangle_vertices else { continue; }; // do something cool with your custom hit data } } ``` ### An example of what you can do with this image
--- Cargo.toml | 12 + crates/bevy_picking/src/backend.rs | 159 +++++++++++++- crates/bevy_picking/src/events.rs | 1 + crates/bevy_picking/src/hover.rs | 3 + examples/README.md | 1 + examples/picking/custom_hit_data.rs | 205 ++++++++++++++++++ .../ui/navigation/directional_navigation.rs | 1 + .../directional_navigation_overrides.rs | 1 + 8 files changed, 380 insertions(+), 3 deletions(-) create mode 100644 examples/picking/custom_hit_data.rs diff --git a/Cargo.toml b/Cargo.toml index b257dd99a1..a7d1c05c11 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4983,6 +4983,18 @@ description = "Demonstrates picking meshes" category = "Picking" wasm = true +[[example]] +name = "custom_hit_data" +path = "examples/picking/custom_hit_data.rs" +doc-scrape-examples = true +required-features = ["mesh_picking"] + +[package.metadata.example.custom_hit_data] +name = "Custom Hit Data" +description = "Demonstrates a custom picking backend with custom hit data." +category = "Picking" +wasm = true + [[example]] name = "simple_picking" path = "examples/picking/simple_picking.rs" diff --git a/crates/bevy_picking/src/backend.rs b/crates/bevy_picking/src/backend.rs index 83ba7fc010..9e3f91b09a 100644 --- a/crates/bevy_picking/src/backend.rs +++ b/crates/bevy_picking/src/backend.rs @@ -31,6 +31,9 @@ //! automatically constructs rays in world space for all cameras and pointers, handling details like //! viewports and DPI for you. +use alloc::sync::Arc; +use core::{any::Any, fmt}; + use bevy_ecs::prelude::*; use bevy_math::Vec3; use bevy_reflect::Reflect; @@ -39,13 +42,43 @@ use bevy_reflect::Reflect; /// /// This includes the most common types in this module, re-exported for your convenience. pub mod prelude { - pub use super::{ray::RayMap, HitData, PointerHits}; + pub use super::{ray::RayMap, HitData, HitDataExtra, PointerHits}; pub use crate::{ pointer::{PointerId, PointerLocation}, Pickable, PickingSystems, }; } +/// Extra data attached to a [`HitData`] by a picking backend. +/// +/// Use this for backend-specific data like triangle indices, UVs, or material +/// information. +/// Any `Send + Sync + fmt::Debug + 'static` type implements this trait +/// automatically. `Clone` is not required: extra data is stored in an [`Arc`], +/// so [`HitData`] can still implement [`Clone`]. `Clone` requires knowing the +/// size of the type, which is not possible with dynamically dispatched types, +/// so it cannot be used for `dyn HitDataExtra`. +/// +/// ```rust +/// #[derive(Debug)] +/// struct MyHitInfo { triangle_index: u32 } +/// ``` +/// +/// Read it back with [`HitData::extra_as`]: +/// +/// ```rust +/// # use bevy_picking::backend::HitData; +/// # #[derive(Debug)] struct MyHitInfo { triangle_index: u32 } +/// fn read_extra(hit: &HitData) { +/// if let Some(info) = hit.extra_as::() { +/// println!("Hit triangle {}", info.triangle_index); +/// } +/// } +/// ``` +pub trait HitDataExtra: Any + Send + Sync + fmt::Debug {} + +impl HitDataExtra for T {} + /// A message produced by a picking backend after it has run its hit tests, describing the entities /// under a pointer. /// @@ -95,8 +128,10 @@ impl PointerHits { } /// Holds data from a successful pointer hit test. See [`HitData::depth`] for important details. -#[derive(Clone, Debug, PartialEq, Reflect)] -#[reflect(Clone, PartialEq)] +/// +/// Backends can attach arbitrary typed data via [`HitData::extra`]. See [`HitDataExtra`]. +#[derive(Debug, Reflect)] +#[reflect(Debug)] pub struct HitData { /// The camera entity used to detect this hit. Useful when you need to find the ray that was /// cast for this hit when using a raycasting backend. @@ -111,6 +146,34 @@ pub struct HitData { pub position: Option, /// The normal vector of the hit test, if the data is available from the backend. pub normal: Option, + /// Optional backend-specific extra data attached to this hit. Read it with [`HitData::extra_as`]. + /// + /// This is stored in an [`Arc`] so cloning [`HitData`] stays cheap. This field is excluded + /// from [`PartialEq`] because value equality for trait objects would require extra dynamic + /// downcasting that the picking pipeline does not need. + #[reflect(ignore)] + pub extra: Option>, +} + +impl Clone for HitData { + fn clone(&self) -> Self { + Self { + camera: self.camera, + depth: self.depth, + position: self.position, + normal: self.normal, + extra: self.extra.as_ref().map(Arc::clone), + } + } +} + +impl PartialEq for HitData { + fn eq(&self, other: &Self) -> bool { + self.camera == other.camera + && self.depth == other.depth + && self.position == other.position + && self.normal == other.normal + } } impl HitData { @@ -121,6 +184,46 @@ impl HitData { depth, position, normal, + extra: None, + } + } + + /// Returns any attached extra data as `T` if available. + /// + /// This returns `None` if no extra data was attached, or if the hit stores a + /// different concrete extra data type. + pub fn extra_as(&self) -> Option<&T> { + let extra: &dyn Any = self.extra.as_deref()?; + extra.downcast_ref::() + } + + /// Creates a [`HitData`] with backend-specific extra data. `extra` can be + /// any [`HitDataExtra`]. + /// + /// # Example + /// + /// ```rust + /// # use bevy_ecs::prelude::*; + /// # use bevy_picking::backend::HitData; + /// #[derive(Debug)] + /// struct MyHitInfo { triangle_index: u32 } + /// + /// # let camera = Entity::PLACEHOLDER; + /// let hit = HitData::new_with_extra(camera, 1.0, None, None, MyHitInfo { triangle_index: 7 }); + /// ``` + pub fn new_with_extra( + camera: Entity, + depth: f32, + position: Option, + normal: Option, + extra: impl HitDataExtra, + ) -> Self { + Self { + camera, + depth, + position, + normal, + extra: Some(Arc::new(extra)), } } } @@ -239,3 +342,53 @@ pub mod ray { .ok() } } + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Debug, PartialEq)] + struct TriangleHitInfo { + triangle_index: u32, + } + + #[derive(Debug, PartialEq)] + struct OtherHitInfo { + triangle_index: u32, + } + + #[test] + fn hit_data_extra() { + let camera = Entity::PLACEHOLDER; + + let hit = HitData::new_with_extra( + camera, + 1.0, + Some(Vec3::new(1.0, 2.0, 3.0)), + Some(Vec3::Y), + TriangleHitInfo { triangle_index: 7 }, + ); + + assert_eq!( + hit.extra_as::(), + Some(&TriangleHitInfo { triangle_index: 7 }) + ); + assert_eq!(hit.extra_as::(), None); + + let cloned = hit.clone(); + assert_eq!( + cloned.extra_as::(), + Some(&TriangleHitInfo { triangle_index: 7 }) + ); + + let other_extra = HitData::new_with_extra( + camera, + 1.0, + Some(Vec3::new(1.0, 2.0, 3.0)), + Some(Vec3::Y), + TriangleHitInfo { triangle_index: 99 }, + ); + + assert_eq!(hit, other_extra); + } +} diff --git a/crates/bevy_picking/src/events.rs b/crates/bevy_picking/src/events.rs index ee2cf13372..25efafcfe2 100644 --- a/crates/bevy_picking/src/events.rs +++ b/crates/bevy_picking/src/events.rs @@ -1248,6 +1248,7 @@ mod tests { camera, position: None, normal: None, + extra: None, }, ); } diff --git a/crates/bevy_picking/src/hover.rs b/crates/bevy_picking/src/hover.rs index 7a7723885e..aeb7365e35 100644 --- a/crates/bevy_picking/src/hover.rs +++ b/crates/bevy_picking/src/hover.rs @@ -462,6 +462,7 @@ mod tests { camera, position: None, normal: None, + extra: None, }, ); hover_map.insert(PointerId::Mouse, entity_map); @@ -514,6 +515,7 @@ mod tests { camera, position: None, normal: None, + extra: None, }, ); hover_map.insert(PointerId::Mouse, entity_map); @@ -579,6 +581,7 @@ mod tests { camera, position: None, normal: None, + extra: None, }, ); hover_map.insert(PointerId::Mouse, entity_map); diff --git a/examples/README.md b/examples/README.md index da73df5181..0db6f5acf2 100644 --- a/examples/README.md +++ b/examples/README.md @@ -438,6 +438,7 @@ Example | Description Example | Description --- | --- +[Custom Hit Data](../examples/picking/custom_hit_data.rs) | Demonstrates a custom picking backend with custom hit data. [Drag and Drop](../examples/picking/dragdrop_picking.rs) | Demonstrates drag and drop using picking events [Mesh Picking](../examples/picking/mesh_picking.rs) | Demonstrates picking meshes [Picking Debug Tools](../examples/picking/debug_picking.rs) | Demonstrates picking debug overlay diff --git a/examples/picking/custom_hit_data.rs b/examples/picking/custom_hit_data.rs new file mode 100644 index 0000000000..dabbfa159e --- /dev/null +++ b/examples/picking/custom_hit_data.rs @@ -0,0 +1,205 @@ +//! Demonstrates a custom picking backend with custom hit data. +//! +//! The example contains pickable 3D meshes. When a mesh is hovered, a custom +//! picking backend performs a ray cast against the mesh and retrieves the +//! triangle that was hit. The triangle vertices are stored in a custom struct +//! (`TriangleHitInfo`) that implements `HitDataExtra`, and saved into `HitData` +//! structs. This information is not available by default in `HitData` and thus +//! requires its `extra` field. A follow-up system reads the hit data and draws +//! an outline around the hovered triangle using gizmos. + +use bevy::{ + color::palettes::css::*, + picking::{ + backend::{ray::RayMap, HitData, PointerHits}, + mesh_picking::{ + ray_cast::{MeshRayCast, MeshRayCastSettings, RayCastVisibility}, + MeshPickingSettings, + }, + prelude::*, + PickingSettings, PickingSystems, + }, + prelude::*, +}; + +fn main() { + App::new() + .add_plugins((DefaultPlugins, MeshPickingPlugin)) + .insert_resource(MeshPickingSettings { + require_markers: true, + ..default() + }) + .insert_resource(PickingSettings { + is_window_picking_enabled: false, + ..default() + }) + .init_resource::() + .add_systems(Startup, (setup_gizmos, setup_scene)) + .add_systems( + PreUpdate, + ( + custom_backend_system.in_set(PickingSystems::Backend), + cache_hovered_triangles.after(PickingSystems::Backend), + ), + ) + .add_systems(Update, draw_hit_gizmos) + .run(); +} + +/// The custom hit data used by our picking backend. All structs that implement +/// `Send + Sync + fmt::Debug + 'static` automatically implement `HitDataExtra` +/// and can be used as extra data in `HitData`. +#[derive(Debug)] +struct TriangleHitInfo { + triangle_vertices: Option<[Vec3; 3]>, +} + +#[derive(Resource, Default)] +struct HoveredTriangles(Vec); + +struct TriangleOverlay { + position: Vec3, + normal: Vec3, + vertices: [Vec3; 3], +} + +fn setup_scene( + mut commands: Commands, + mut meshes: ResMut>, + mut materials: ResMut>, +) { + let shapes: [(Mesh, Color); 3] = [ + (Cuboid::default().into(), RED.into()), + (Sphere::default().mesh().ico(2).unwrap(), GREEN.into()), + (Cylinder::default().into(), BLUE.into()), + ]; + + for (i, (mesh, color)) in shapes.iter().enumerate() { + let x = i as f32 * 1.5 - 1.5; + let material = materials.add(StandardMaterial::from_color(*color)); + + commands.spawn(( + Mesh3d(meshes.add(mesh.clone())), + MeshMaterial3d(material), + Transform::from_xyz(x, 0.5, 0.0), + Pickable::default(), + )); + } + + commands.spawn(( + Mesh3d(meshes.add(Plane3d::default().mesh().size(30.0, 30.0))), + MeshMaterial3d(materials.add(Color::from(DARK_GRAY))), + Pickable::IGNORE, + )); + + commands.spawn((PointLight::default(), Transform::from_xyz(0.0, 8.0, 4.0))); + + commands.spawn(( + Camera3d::default(), + Transform::from_xyz(0.0, 2.5, 6.0).looking_at(Vec3::new(0.0, 0.3, 0.0), Vec3::Y), + )); +} + +fn setup_gizmos(mut config_store: ResMut) { + let (config, _) = config_store.config_mut::(); + config.depth_bias = -1.0; + config.line.width = 3.0; +} + +fn custom_backend_system( + ray_map: Res, + cameras: Query<&Camera>, + pickables: Query<&Pickable>, + mut ray_cast: MeshRayCast, + mut pointer_hits: MessageWriter, +) { + for (&ray_id, &ray) in ray_map.iter() { + let Ok(camera) = cameras.get(ray_id.camera) else { + continue; + }; + + let settings = MeshRayCastSettings { + visibility: RayCastVisibility::VisibleInView, + filter: &|e| pickables.get(e).is_ok_and(|p| p.is_hoverable), + early_exit_test: &|entity_hit| { + pickables + .get(entity_hit) + .is_ok_and(|p| p.should_block_lower) + }, + }; + + let picks: Vec<(Entity, HitData)> = ray_cast + .cast_ray(ray, &settings) + .iter() + .map(|(entity, hit)| { + let extra = TriangleHitInfo { + triangle_vertices: hit.triangle, + }; + + let hit_data = HitData::new_with_extra( + ray_id.camera, + hit.distance, + Some(hit.point), + Some(hit.normal), + extra, + ); + + (*entity, hit_data) + }) + .collect(); + + if !picks.is_empty() { + pointer_hits.write(PointerHits::new(ray_id.pointer, picks, camera.order as f32)); + } + } +} + +fn cache_hovered_triangles( + mut pointer_hits: MessageReader, + mut hovered_triangles: ResMut, +) { + hovered_triangles.0.clear(); + + for hits in pointer_hits.read() { + for (_, hit) in &hits.picks { + let (Some(position), Some(normal)) = (hit.position, hit.normal) else { + continue; + }; + + let Some(info) = hit.extra_as::() else { + continue; + }; + let Some(vertices) = info.triangle_vertices else { + continue; + }; + + hovered_triangles.0.push(TriangleOverlay { + position, + normal, + vertices, + }); + } + } +} + +fn draw_hit_gizmos(hovered_triangles: Res, mut gizmos: Gizmos) { + for triangle in &hovered_triangles.0 { + gizmos.arrow( + triangle.position, + triangle.position + triangle.normal.normalize() * 0.5, + WHITE, + ); + + let vertices = triangle.vertices; + let center = (vertices[0] + vertices[1] + vertices[2]) / 3.0; + let offset = triangle.normal.normalize_or_zero() * 0.025; + + // The outline is made bigger and offset a bit to prevent being covered + // by the mesh + let outline = vertices.map(|vertex| center + (vertex - center) * 1.05 + offset); + + gizmos.line(outline[0], outline[1], WHITE); + gizmos.line(outline[1], outline[2], WHITE); + gizmos.line(outline[2], outline[0], WHITE); + } +} diff --git a/examples/ui/navigation/directional_navigation.rs b/examples/ui/navigation/directional_navigation.rs index 78c125dc5a..ce67448627 100644 --- a/examples/ui/navigation/directional_navigation.rs +++ b/examples/ui/navigation/directional_navigation.rs @@ -462,6 +462,7 @@ fn interact_with_focused_button( depth: 0.0, position: None, normal: None, + extra: None, }, duration: Duration::from_secs_f32(0.1), }, diff --git a/examples/ui/navigation/directional_navigation_overrides.rs b/examples/ui/navigation/directional_navigation_overrides.rs index f0f39e86a0..351d964233 100644 --- a/examples/ui/navigation/directional_navigation_overrides.rs +++ b/examples/ui/navigation/directional_navigation_overrides.rs @@ -855,6 +855,7 @@ fn interact_with_focused_button( depth: 0.0, position: None, normal: None, + extra: None, }, duration: Duration::from_secs_f32(0.1), },