Add support for custom picking data (#23245)

# Objective

- Adds the ability for picking backends to add custom hit data
- Fixes #16186 

## Solution

- Adds an `extra` field to the HitData struct which can take any Data
that implements HitDataExtra. This is stored in an Arc which can be
downcast with a helper function in the system that consumes the hit.
- In the original ticket I suggested using a generic, however this
caused extensive code changes and complications down the line with the
HoverMap and OverMap so I decided against it.

## Testing

- I added an example custom_hit_data to test the new feature
- I tested all other picking examples to ensure they still work as
expected
- I tested the examples on Ubuntu
- I additionally tested the custom_hit_data example in wasm 

---

## Showcase

<details>
  <summary>Click to view showcase</summary>

### Creating custom hits:
```rust
let picks: Vec<(Entity, HitData)> = ray_cast
            .cast_ray(ray, &settings)
            .iter()
            .map(|(entity, hit)| {
                let extra = TriangleHitInfo {
                    triangle_vertices: hit.triangle,
                };

                let hit_data = HitData::new_with_extra(
                    ray_id.camera,
                    hit.distance,
                    Some(hit.point),
                    Some(hit.normal),
                    extra,
                );

                (*entity, hit_data)
            })
            .collect();
```

### Reading custom hits:
```rust
   for hits in pointer_hits.read() {
        for (_, hit) in &hits.picks {
            let Some(info) = hit.extra_as::<TriangleHitInfo>() else {
                continue;
            };
            let Some(vertices) = info.triangle_vertices else {
                continue;
            };

            // do something cool with your custom hit data
        }
    }
```

### An example of what you can do with this
<img width="1291" height="730" alt="image"
src="https://github.com/user-attachments/assets/2d0e8aed-7059-470a-a0f6-1453356cfe6a"
/>


</details>

Cargo.toml

@@ -4983,6 +4983,18 @@ description = "Demonstrates picking meshes"
 category = "Picking"
 wasm = true
 
+[[example]]
+name = "custom_hit_data"
+path = "examples/picking/custom_hit_data.rs"
+doc-scrape-examples = true
+required-features = ["mesh_picking"]
+
+[package.metadata.example.custom_hit_data]
+name = "Custom Hit Data"
+description = "Demonstrates a custom picking backend with custom hit data."
+category = "Picking"
+wasm = true
+
 [[example]]
 name = "simple_picking"
 path = "examples/picking/simple_picking.rs"

crates/bevy_picking/src/backend.rs

@@ -31,6 +31,9 @@
 //! automatically constructs rays in world space for all cameras and pointers, handling details like
 //! viewports and DPI for you.
 
+use alloc::sync::Arc;
+use core::{any::Any, fmt};
+
 use bevy_ecs::prelude::*;
 use bevy_math::Vec3;
 use bevy_reflect::Reflect;
@@ -39,13 +42,43 @@ use bevy_reflect::Reflect;
 ///
 /// This includes the most common types in this module, re-exported for your convenience.
 pub mod prelude {
-    pub use super::{ray::RayMap, HitData, PointerHits};
+    pub use super::{ray::RayMap, HitData, HitDataExtra, PointerHits};
     pub use crate::{
         pointer::{PointerId, PointerLocation},
         Pickable, PickingSystems,
     };
 }
 
+/// Extra data attached to a [`HitData`] by a picking backend.
+///
+/// Use this for backend-specific data like triangle indices, UVs, or material
+/// information.
+/// Any `Send + Sync + fmt::Debug + 'static` type implements this trait
+/// automatically. `Clone` is not required: extra data is stored in an [`Arc`],
+/// so [`HitData`] can still implement [`Clone`]. `Clone` requires knowing the
+/// size of the type, which is not possible with dynamically dispatched types,
+/// so it cannot be used for `dyn HitDataExtra`.
+///
+/// ```rust
+/// #[derive(Debug)]
+/// struct MyHitInfo { triangle_index: u32 }
+/// ```
+///
+/// Read it back with [`HitData::extra_as`]:
+///
+/// ```rust
+/// # use bevy_picking::backend::HitData;
+/// # #[derive(Debug)] struct MyHitInfo { triangle_index: u32 }
+/// fn read_extra(hit: &HitData) {
+///     if let Some(info) = hit.extra_as::<MyHitInfo>() {
+///         println!("Hit triangle {}", info.triangle_index);
+///     }
+/// }
+/// ```
+pub trait HitDataExtra: Any + Send + Sync + fmt::Debug {}
+
+impl<T: Send + Sync + fmt::Debug + Any + 'static> HitDataExtra for T {}
+
 /// A message produced by a picking backend after it has run its hit tests, describing the entities
 /// under a pointer.
 ///
@@ -95,8 +128,10 @@ impl PointerHits {
 }
 
 /// Holds data from a successful pointer hit test. See [`HitData::depth`] for important details.
-#[derive(Clone, Debug, PartialEq, Reflect)]
-#[reflect(Clone, PartialEq)]
+///
+/// Backends can attach arbitrary typed data via [`HitData::extra`]. See [`HitDataExtra`].
+#[derive(Debug, Reflect)]
+#[reflect(Debug)]
 pub struct HitData {
     /// The camera entity used to detect this hit. Useful when you need to find the ray that was
     /// cast for this hit when using a raycasting backend.
@@ -111,6 +146,34 @@ pub struct HitData {
     pub position: Option<Vec3>,
     /// The normal vector of the hit test, if the data is available from the backend.
     pub normal: Option<Vec3>,
+    /// Optional backend-specific extra data attached to this hit. Read it with [`HitData::extra_as`].
+    ///
+    /// This is stored in an [`Arc`] so cloning [`HitData`] stays cheap. This field is excluded
+    /// from [`PartialEq`] because value equality for trait objects would require extra dynamic
+    /// downcasting that the picking pipeline does not need.
+    #[reflect(ignore)]
+    pub extra: Option<Arc<dyn HitDataExtra>>,
+}
+
+impl Clone for HitData {
+    fn clone(&self) -> Self {
+        Self {
+            camera: self.camera,
+            depth: self.depth,
+            position: self.position,
+            normal: self.normal,
+            extra: self.extra.as_ref().map(Arc::clone),
+        }
+    }
+}
+
+impl PartialEq for HitData {
+    fn eq(&self, other: &Self) -> bool {
+        self.camera == other.camera
+            && self.depth == other.depth
+            && self.position == other.position
+            && self.normal == other.normal
+    }
 }
 
 impl HitData {
@@ -121,6 +184,46 @@ impl HitData {
             depth,
             position,
             normal,
+            extra: None,
+        }
+    }
+
+    /// Returns any attached extra data as `T` if available.
+    ///
+    /// This returns `None` if no extra data was attached, or if the hit stores a
+    /// different concrete extra data type.
+    pub fn extra_as<T: Any>(&self) -> Option<&T> {
+        let extra: &dyn Any = self.extra.as_deref()?;
+        extra.downcast_ref::<T>()
+    }
+
+    /// Creates a [`HitData`] with backend-specific extra data. `extra` can be
+    /// any [`HitDataExtra`].
+    ///
+    /// # Example
+    ///
+    /// ```rust
+    /// # use bevy_ecs::prelude::*;
+    /// # use bevy_picking::backend::HitData;
+    /// #[derive(Debug)]
+    /// struct MyHitInfo { triangle_index: u32 }
+    ///
+    /// # let camera = Entity::PLACEHOLDER;
+    /// let hit = HitData::new_with_extra(camera, 1.0, None, None, MyHitInfo { triangle_index: 7 });
+    /// ```
+    pub fn new_with_extra(
+        camera: Entity,
+        depth: f32,
+        position: Option<Vec3>,
+        normal: Option<Vec3>,
+        extra: impl HitDataExtra,
+    ) -> Self {
+        Self {
+            camera,
+            depth,
+            position,
+            normal,
+            extra: Some(Arc::new(extra)),
         }
     }
 }
@@ -239,3 +342,53 @@ pub mod ray {
             .ok()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[derive(Debug, PartialEq)]
+    struct TriangleHitInfo {
+        triangle_index: u32,
+    }
+
+    #[derive(Debug, PartialEq)]
+    struct OtherHitInfo {
+        triangle_index: u32,
+    }
+
+    #[test]
+    fn hit_data_extra() {
+        let camera = Entity::PLACEHOLDER;
+
+        let hit = HitData::new_with_extra(
+            camera,
+            1.0,
+            Some(Vec3::new(1.0, 2.0, 3.0)),
+            Some(Vec3::Y),
+            TriangleHitInfo { triangle_index: 7 },
+        );
+
+        assert_eq!(
+            hit.extra_as::<TriangleHitInfo>(),
+            Some(&TriangleHitInfo { triangle_index: 7 })
+        );
+        assert_eq!(hit.extra_as::<OtherHitInfo>(), None);
+
+        let cloned = hit.clone();
+        assert_eq!(
+            cloned.extra_as::<TriangleHitInfo>(),
+            Some(&TriangleHitInfo { triangle_index: 7 })
+        );
+
+        let other_extra = HitData::new_with_extra(
+            camera,
+            1.0,
+            Some(Vec3::new(1.0, 2.0, 3.0)),
+            Some(Vec3::Y),
+            TriangleHitInfo { triangle_index: 99 },
+        );
+
+        assert_eq!(hit, other_extra);
+    }
+}

crates/bevy_picking/src/events.rs

@@ -1248,6 +1248,7 @@ mod tests {
                     camera,
                     position: None,
                     normal: None,
+                    extra: None,
                 },
             );
         }

crates/bevy_picking/src/hover.rs

@@ -462,6 +462,7 @@ mod tests {
                 camera,
                 position: None,
                 normal: None,
+                extra: None,
             },
         );
         hover_map.insert(PointerId::Mouse, entity_map);
@@ -514,6 +515,7 @@ mod tests {
                 camera,
                 position: None,
                 normal: None,
+                extra: None,
             },
         );
         hover_map.insert(PointerId::Mouse, entity_map);
@@ -579,6 +581,7 @@ mod tests {
                 camera,
                 position: None,
                 normal: None,
+                extra: None,
             },
         );
         hover_map.insert(PointerId::Mouse, entity_map);

examples/README.md

@@ -438,6 +438,7 @@ Example | Description
 
 Example | Description
 --- | ---
+[Custom Hit Data](../examples/picking/custom_hit_data.rs) | Demonstrates a custom picking backend with custom hit data.
 [Drag and Drop](../examples/picking/dragdrop_picking.rs) | Demonstrates drag and drop using picking events
 [Mesh Picking](../examples/picking/mesh_picking.rs) | Demonstrates picking meshes
 [Picking Debug Tools](../examples/picking/debug_picking.rs) | Demonstrates picking debug overlay

examples/picking/custom_hit_data.rs

@@ -0,0 +1,205 @@
+//! Demonstrates a custom picking backend with custom hit data.
+//!
+//! The example contains pickable 3D meshes. When a mesh is hovered, a custom
+//! picking backend performs a ray cast against the mesh and retrieves the
+//! triangle that was hit. The triangle vertices are stored in a custom struct
+//! (`TriangleHitInfo`) that implements `HitDataExtra`, and saved into `HitData`
+//! structs. This information is not available by default in `HitData` and thus
+//! requires its `extra` field. A follow-up system reads the hit data and draws
+//! an outline around the hovered triangle using gizmos.
+
+use bevy::{
+    color::palettes::css::*,
+    picking::{
+        backend::{ray::RayMap, HitData, PointerHits},
+        mesh_picking::{
+            ray_cast::{MeshRayCast, MeshRayCastSettings, RayCastVisibility},
+            MeshPickingSettings,
+        },
+        prelude::*,
+        PickingSettings, PickingSystems,
+    },
+    prelude::*,
+};
+
+fn main() {
+    App::new()
+        .add_plugins((DefaultPlugins, MeshPickingPlugin))
+        .insert_resource(MeshPickingSettings {
+            require_markers: true,
+            ..default()
+        })
+        .insert_resource(PickingSettings {
+            is_window_picking_enabled: false,
+            ..default()
+        })
+        .init_resource::<HoveredTriangles>()
+        .add_systems(Startup, (setup_gizmos, setup_scene))
+        .add_systems(
+            PreUpdate,
+            (
+                custom_backend_system.in_set(PickingSystems::Backend),
+                cache_hovered_triangles.after(PickingSystems::Backend),
+            ),
+        )
+        .add_systems(Update, draw_hit_gizmos)
+        .run();
+}
+
+/// The custom hit data used by our picking backend. All structs that implement
+/// `Send + Sync + fmt::Debug + 'static` automatically implement `HitDataExtra`
+/// and can be used as extra data in `HitData`.
+#[derive(Debug)]
+struct TriangleHitInfo {
+    triangle_vertices: Option<[Vec3; 3]>,
+}
+
+#[derive(Resource, Default)]
+struct HoveredTriangles(Vec<TriangleOverlay>);
+
+struct TriangleOverlay {
+    position: Vec3,
+    normal: Vec3,
+    vertices: [Vec3; 3],
+}
+
+fn setup_scene(
+    mut commands: Commands,
+    mut meshes: ResMut<Assets<Mesh>>,
+    mut materials: ResMut<Assets<StandardMaterial>>,
+) {
+    let shapes: [(Mesh, Color); 3] = [
+        (Cuboid::default().into(), RED.into()),
+        (Sphere::default().mesh().ico(2).unwrap(), GREEN.into()),
+        (Cylinder::default().into(), BLUE.into()),
+    ];
+
+    for (i, (mesh, color)) in shapes.iter().enumerate() {
+        let x = i as f32 * 1.5 - 1.5;
+        let material = materials.add(StandardMaterial::from_color(*color));
+
+        commands.spawn((
+            Mesh3d(meshes.add(mesh.clone())),
+            MeshMaterial3d(material),
+            Transform::from_xyz(x, 0.5, 0.0),
+            Pickable::default(),
+        ));
+    }
+
+    commands.spawn((
+        Mesh3d(meshes.add(Plane3d::default().mesh().size(30.0, 30.0))),
+        MeshMaterial3d(materials.add(Color::from(DARK_GRAY))),
+        Pickable::IGNORE,
+    ));
+
+    commands.spawn((PointLight::default(), Transform::from_xyz(0.0, 8.0, 4.0)));
+
+    commands.spawn((
+        Camera3d::default(),
+        Transform::from_xyz(0.0, 2.5, 6.0).looking_at(Vec3::new(0.0, 0.3, 0.0), Vec3::Y),
+    ));
+}
+
+fn setup_gizmos(mut config_store: ResMut<GizmoConfigStore>) {
+    let (config, _) = config_store.config_mut::<DefaultGizmoConfigGroup>();
+    config.depth_bias = -1.0;
+    config.line.width = 3.0;
+}
+
+fn custom_backend_system(
+    ray_map: Res<RayMap>,
+    cameras: Query<&Camera>,
+    pickables: Query<&Pickable>,
+    mut ray_cast: MeshRayCast,
+    mut pointer_hits: MessageWriter<PointerHits>,
+) {
+    for (&ray_id, &ray) in ray_map.iter() {
+        let Ok(camera) = cameras.get(ray_id.camera) else {
+            continue;
+        };
+
+        let settings = MeshRayCastSettings {
+            visibility: RayCastVisibility::VisibleInView,
+            filter: &|e| pickables.get(e).is_ok_and(|p| p.is_hoverable),
+            early_exit_test: &|entity_hit| {
+                pickables
+                    .get(entity_hit)
+                    .is_ok_and(|p| p.should_block_lower)
+            },
+        };
+
+        let picks: Vec<(Entity, HitData)> = ray_cast
+            .cast_ray(ray, &settings)
+            .iter()
+            .map(|(entity, hit)| {
+                let extra = TriangleHitInfo {
+                    triangle_vertices: hit.triangle,
+                };
+
+                let hit_data = HitData::new_with_extra(
+                    ray_id.camera,
+                    hit.distance,
+                    Some(hit.point),
+                    Some(hit.normal),
+                    extra,
+                );
+
+                (*entity, hit_data)
+            })
+            .collect();
+
+        if !picks.is_empty() {
+            pointer_hits.write(PointerHits::new(ray_id.pointer, picks, camera.order as f32));
+        }
+    }
+}
+
+fn cache_hovered_triangles(
+    mut pointer_hits: MessageReader<PointerHits>,
+    mut hovered_triangles: ResMut<HoveredTriangles>,
+) {
+    hovered_triangles.0.clear();
+
+    for hits in pointer_hits.read() {
+        for (_, hit) in &hits.picks {
+            let (Some(position), Some(normal)) = (hit.position, hit.normal) else {
+                continue;
+            };
+
+            let Some(info) = hit.extra_as::<TriangleHitInfo>() else {
+                continue;
+            };
+            let Some(vertices) = info.triangle_vertices else {
+                continue;
+            };
+
+            hovered_triangles.0.push(TriangleOverlay {
+                position,
+                normal,
+                vertices,
+            });
+        }
+    }
+}
+
+fn draw_hit_gizmos(hovered_triangles: Res<HoveredTriangles>, mut gizmos: Gizmos) {
+    for triangle in &hovered_triangles.0 {
+        gizmos.arrow(
+            triangle.position,
+            triangle.position + triangle.normal.normalize() * 0.5,
+            WHITE,
+        );
+
+        let vertices = triangle.vertices;
+        let center = (vertices[0] + vertices[1] + vertices[2]) / 3.0;
+        let offset = triangle.normal.normalize_or_zero() * 0.025;
+
+        // The outline is made bigger and offset a bit to prevent being covered
+        // by the mesh
+        let outline = vertices.map(|vertex| center + (vertex - center) * 1.05 + offset);
+
+        gizmos.line(outline[0], outline[1], WHITE);
+        gizmos.line(outline[1], outline[2], WHITE);
+        gizmos.line(outline[2], outline[0], WHITE);
+    }
+}

examples/ui/navigation/directional_navigation.rs

@@ -462,6 +462,7 @@ fn interact_with_focused_button(
                     depth: 0.0,
                     position: None,
                     normal: None,
+                    extra: None,
                 },
                 duration: Duration::from_secs_f32(0.1),
             },

examples/ui/navigation/directional_navigation_overrides.rs

@@ -855,6 +855,7 @@ fn interact_with_focused_button(
                     depth: 0.0,
                     position: None,
                     normal: None,
+                    extra: None,
                 },
                 duration: Duration::from_secs_f32(0.1),
             },