fix(client_openxr): 🐛 Try fixing crash on Quest 2 related to multimod…

…al (#2654) * fix(client_openxr): 🐛 Try fixing crash on Quest 2 related to multimodal * Address comments
alvr-org · Jan 19, 2025 · 1b3d1c7 · 1b3d1c7
1 parent 2db4d38
commit 1b3d1c7
Show file tree

Hide file tree

Showing 5 changed files with 157 additions and 96 deletions.
diff --git a/alvr/client_openxr/src/extra_extensions/eye_gaze_interaction.rs b/alvr/client_openxr/src/extra_extensions/eye_gaze_interaction.rs
@@ -1,27 +1,12 @@
 use openxr::{self as xr, sys};
 use std::ptr;
 
-fn get_props<G, T>(session: &xr::Session<G>, system: xr::SystemId, default_struct: T) -> Option<T> {
-    let instance = session.instance();
-
-    let mut props = default_struct;
-    let mut system_properties = sys::SystemProperties::out((&mut props as *mut T).cast());
-    let result = unsafe {
-        (instance.fp().get_system_properties)(
-            instance.as_raw(),
-            system,
-            system_properties.as_mut_ptr(),
-        )
-    };
-    (result.into_raw() >= 0).then_some(props)
-}
-
 pub fn supports_eye_gaze_interaction<G>(session: &xr::Session<G>, system: xr::SystemId) -> bool {
     if session.instance().exts().ext_eye_gaze_interaction.is_none() {
         return false;
     }
 
-    get_props(
+    super::get_props(
         session,
         system,
         sys::SystemEyeGazeInteractionPropertiesEXT {

diff --git a/alvr/client_openxr/src/extra_extensions/mod.rs b/alvr/client_openxr/src/extra_extensions/mod.rs
@@ -25,3 +25,23 @@ fn xr_res(result: sys::Result) -> xr::Result<()> {
         Err(result)
     }
 }
+
+fn get_props<G, T>(
+    session: &xr::Session<G>,
+    system: xr::SystemId,
+    default_struct: T,
+) -> xr::Result<T> {
+    let instance = session.instance();
+
+    let mut props = default_struct;
+    let mut system_properties = sys::SystemProperties::out((&mut props as *mut T).cast());
+    let result = unsafe {
+        (instance.fp().get_system_properties)(
+            instance.as_raw(),
+            system,
+            system_properties.as_mut_ptr(),
+        )
+    };
+
+    xr_res(result).map(|_| props)
+}
diff --git a/alvr/client_openxr/src/extra_extensions/multimodal_input.rs b/alvr/client_openxr/src/extra_extensions/multimodal_input.rs
@@ -12,87 +12,139 @@ pub const META_SIMULTANEOUS_HANDS_AND_CONTROLLERS_EXTENSION_NAME: &str =
     "XR_META_simultaneous_hands_and_controllers";
 pub const META_DETACHED_CONTROLLERS_EXTENSION_NAME: &str = "XR_META_detached_controllers";
 
+static TYPE_SYSTEM_SIMULTANEOUS_HANDS_AND_CONTROLLERS_PROPERTIES_META: Lazy<xr::StructureType> =
+    Lazy::new(|| xr::StructureType::from_raw(1000532001));
 static TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_RESUME_INFO_META: Lazy<xr::StructureType> =
     Lazy::new(|| xr::StructureType::from_raw(1000532002));
 static TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_PAUSE_INFO_META: Lazy<xr::StructureType> =
     Lazy::new(|| xr::StructureType::from_raw(1000532003));
 
 #[repr(C)]
-pub struct SimultaneousHandsAndControllersTrackingResumeInfoMETA {
+struct SystemSymultaneousHandsAndControllersPropertiesMETA {
+    ty: xr::StructureType,
+    next: *const c_void,
+    supports_simultaneous_hands_and_controllers: sys::Bool32,
+}
+
+#[repr(C)]
+struct SimultaneousHandsAndControllersTrackingResumeInfoMETA {
     ty: xr::StructureType,
     next: *const c_void,
 }
 #[repr(C)]
-pub struct SimultaneousHandsAndControllersTrackingPauseInfoMETA {
+struct SimultaneousHandsAndControllersTrackingPauseInfoMETA {
     ty: xr::StructureType,
     next: *const c_void,
 }
 
-pub type ResumeSimultaneousHandsAndControllersTrackingMETA =
-    unsafe extern "system" fn(
-        sys::Session,
-        *const SimultaneousHandsAndControllersTrackingResumeInfoMETA,
-    ) -> sys::Result;
-pub type PauseSimultaneousHandsAndControllersTrackingMETA =
-    unsafe extern "system" fn(
-        sys::Session,
-        *const SimultaneousHandsAndControllersTrackingPauseInfoMETA,
-    ) -> sys::Result;
-
-pub fn resume_simultaneous_hands_and_controllers_tracking_meta<G>(
-    session: &xr::Session<G>,
-) -> xr::Result<()> {
-    let resume_simultaneous_hands_and_controllers_tracking_meta = unsafe {
-        let mut resume_simultaneous_hands_and_controllers_tracking_meta = None;
-        let _ = (session.instance().fp().get_instance_proc_addr)(
-            session.instance().as_raw(),
-            c"xrResumeSimultaneousHandsAndControllersTrackingMETA".as_ptr(),
-            &mut resume_simultaneous_hands_and_controllers_tracking_meta,
-        );
-
-        resume_simultaneous_hands_and_controllers_tracking_meta.map(|pfn| {
-            mem::transmute::<VoidFunction, ResumeSimultaneousHandsAndControllersTrackingMETA>(pfn)
-        })
-    }
-    .ok_or(sys::Result::ERROR_EXTENSION_NOT_PRESENT)?;
-
-    let resume_info = SimultaneousHandsAndControllersTrackingResumeInfoMETA {
-        ty: *TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_RESUME_INFO_META,
-        next: ptr::null(),
-    };
-    unsafe {
-        super::xr_res(resume_simultaneous_hands_and_controllers_tracking_meta(
-            session.as_raw(),
-            &resume_info,
-        ))
-    }
+type ResumeSimultaneousHandsAndControllersTrackingMETA = unsafe extern "system" fn(
+    sys::Session,
+    *const SimultaneousHandsAndControllersTrackingResumeInfoMETA,
+) -> sys::Result;
+type PauseSimultaneousHandsAndControllersTrackingMETA = unsafe extern "system" fn(
+    sys::Session,
+    *const SimultaneousHandsAndControllersTrackingPauseInfoMETA,
+) -> sys::Result;
+
+pub struct MultimodalMeta {
+    session: xr::Session<xr::AnyGraphics>,
+    resume_simultaneous_hands_and_controllers_tracking_meta:
+        ResumeSimultaneousHandsAndControllersTrackingMETA,
+    pause_simultaneous_hands_and_controllers_tracking_meta:
+        PauseSimultaneousHandsAndControllersTrackingMETA,
 }
 
-pub fn pause_simultaneous_hands_and_controllers_tracking_meta<G>(
-    session: &xr::Session<G>,
-) -> xr::Result<()> {
-    let pause_simultaneous_hands_and_controllers_tracking_meta = unsafe {
-        let mut pause_simultaneous_hands_and_controllers_tracking_meta = None;
-        let _ = (session.instance().fp().get_instance_proc_addr)(
-            session.instance().as_raw(),
-            c"xrPauseSimultaneousHandsAndControllersTrackingMETA".as_ptr(),
-            &mut pause_simultaneous_hands_and_controllers_tracking_meta,
-        );
-
-        pause_simultaneous_hands_and_controllers_tracking_meta.map(|pfn| {
-            mem::transmute::<VoidFunction, PauseSimultaneousHandsAndControllersTrackingMETA>(pfn)
-        })
+impl MultimodalMeta {
+    pub fn new<G>(
+        session: xr::Session<G>,
+        extra_extensions: &[String],
+        system: xr::SystemId,
+    ) -> xr::Result<Self> {
+        if !extra_extensions
+            .contains(&META_SIMULTANEOUS_HANDS_AND_CONTROLLERS_EXTENSION_NAME.to_owned())
+            || !extra_extensions.contains(&META_DETACHED_CONTROLLERS_EXTENSION_NAME.to_owned())
+        {
+            return Err(sys::Result::ERROR_EXTENSION_NOT_PRESENT);
+        }
+
+        let resume_simultaneous_hands_and_controllers_tracking_meta = unsafe {
+            let mut resume_simultaneous_hands_and_controllers_tracking_meta = None;
+            let _ = (session.instance().fp().get_instance_proc_addr)(
+                session.instance().as_raw(),
+                c"xrResumeSimultaneousHandsAndControllersTrackingMETA".as_ptr(),
+                &mut resume_simultaneous_hands_and_controllers_tracking_meta,
+            );
+
+            resume_simultaneous_hands_and_controllers_tracking_meta.map(|pfn| {
+                mem::transmute::<VoidFunction, ResumeSimultaneousHandsAndControllersTrackingMETA>(
+                    pfn,
+                )
+            })
+        }
+        .ok_or(sys::Result::ERROR_EXTENSION_NOT_PRESENT)?;
+
+        let pause_simultaneous_hands_and_controllers_tracking_meta = unsafe {
+            let mut pause_simultaneous_hands_and_controllers_tracking_meta = None;
+            let _ = (session.instance().fp().get_instance_proc_addr)(
+                session.instance().as_raw(),
+                c"xrPauseSimultaneousHandsAndControllersTrackingMETA".as_ptr(),
+                &mut pause_simultaneous_hands_and_controllers_tracking_meta,
+            );
+
+            pause_simultaneous_hands_and_controllers_tracking_meta.map(|pfn| {
+                mem::transmute::<VoidFunction, PauseSimultaneousHandsAndControllersTrackingMETA>(
+                    pfn,
+                )
+            })
+        }
+        .ok_or(sys::Result::ERROR_EXTENSION_NOT_PRESENT)?;
+
+        let props = super::get_props(
+            &session,
+            system,
+            SystemSymultaneousHandsAndControllersPropertiesMETA {
+                ty: *TYPE_SYSTEM_SIMULTANEOUS_HANDS_AND_CONTROLLERS_PROPERTIES_META,
+                next: std::ptr::null(),
+                supports_simultaneous_hands_and_controllers: xr::sys::FALSE,
+            },
+        )?;
+
+        if props.supports_simultaneous_hands_and_controllers.into() {
+            Ok(Self {
+                session: session.into_any_graphics(),
+                resume_simultaneous_hands_and_controllers_tracking_meta,
+                pause_simultaneous_hands_and_controllers_tracking_meta,
+            })
+        } else {
+            Err(sys::Result::ERROR_FEATURE_UNSUPPORTED)
+        }
+    }
+
+    pub fn resume(&self) -> xr::Result<()> {
+        let resume_info = SimultaneousHandsAndControllersTrackingResumeInfoMETA {
+            ty: *TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_RESUME_INFO_META,
+            next: ptr::null(),
+        };
+        unsafe {
+            super::xr_res((self
+                .resume_simultaneous_hands_and_controllers_tracking_meta)(
+                self.session.as_raw(),
+                &resume_info,
+            ))
+        }
     }
-    .ok_or(sys::Result::ERROR_EXTENSION_NOT_PRESENT)?;
-
-    let pause_info = SimultaneousHandsAndControllersTrackingPauseInfoMETA {
-        ty: *TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_PAUSE_INFO_META,
-        next: ptr::null(),
-    };
-    unsafe {
-        super::xr_res(pause_simultaneous_hands_and_controllers_tracking_meta(
-            session.as_raw(),
-            &pause_info,
-        ))
+
+    pub fn pause(&self) -> xr::Result<()> {
+        let pause_info = SimultaneousHandsAndControllersTrackingPauseInfoMETA {
+            ty: *TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_PAUSE_INFO_META,
+            next: ptr::null(),
+        };
+        unsafe {
+            super::xr_res((self
+                .pause_simultaneous_hands_and_controllers_tracking_meta)(
+                self.session.as_raw(),
+                &pause_info,
+            ))
+        }
     }
 }
diff --git a/alvr/client_openxr/src/interaction.rs b/alvr/client_openxr/src/interaction.rs
@@ -1,6 +1,6 @@
 use crate::{
     extra_extensions::{
-        self, BodyTrackerFB, EyeTrackerSocial, FaceTracker2FB, FacialTrackerHTC,
+        self, BodyTrackerFB, EyeTrackerSocial, FaceTracker2FB, FacialTrackerHTC, MultimodalMeta,
         BODY_JOINT_SET_FULL_BODY_META, FULL_BODY_JOINT_COUNT_META,
         FULL_BODY_JOINT_LEFT_FOOT_BALL_META, FULL_BODY_JOINT_LEFT_LOWER_LEG_META,
         FULL_BODY_JOINT_RIGHT_FOOT_BALL_META, FULL_BODY_JOINT_RIGHT_LOWER_LEG_META,
@@ -108,6 +108,7 @@ pub struct InteractionContext {
     pub action_set: xr::ActionSet,
     pub button_actions: HashMap<u64, ButtonAction>,
     pub hands_interaction: [HandInteraction; 2],
+    multimodal_handle: Option<MultimodalMeta>,
     pub multimodal_hands_enabled: bool,
     pub face_sources: FaceSources,
     pub body_sources: BodySources,
@@ -116,9 +117,9 @@ pub struct InteractionContext {
 impl InteractionContext {
     pub fn new(
         xr_session: xr::Session<xr::OpenGlEs>,
+        extra_extensions: &[String],
         xr_system: xr::SystemId,
         platform: Platform,
-        supports_multimodal: bool,
     ) -> Self {
         let xr_instance = xr_session.instance();
 
@@ -229,9 +230,13 @@ impl InteractionContext {
             "/user/hand/right/output/haptic",
         ));
 
+        let multimodal_handle = create_ext_object("MultimodalMeta", Some(true), || {
+            MultimodalMeta::new(xr_session.clone(), extra_extensions, xr_system)
+        });
+
         let left_detached_controller_pose_action;
         let right_detached_controller_pose_action;
-        if supports_multimodal {
+        if multimodal_handle.is_some() {
             // Note: when multimodal input is enabled, both controllers and hands will always be active.
             // To be able to detect when controllers are actually held, we have to register detached
             // controllers pose; the controller pose will be diverted to the detached controllers when
@@ -355,6 +360,7 @@ impl InteractionContext {
                     skeleton_tracker: right_hand_tracker,
                 },
             ],
+            multimodal_handle,
             multimodal_hands_enabled: false,
             face_sources: FaceSources {
                 combined_eyes_source,
@@ -371,8 +377,9 @@ impl InteractionContext {
 
     pub fn select_sources(&mut self, config: &InteractionSourcesConfig) {
         // First of all, disable/delete all sources. This ensures there are no conflicts
-        extra_extensions::pause_simultaneous_hands_and_controllers_tracking_meta(&self.xr_session)
-            .ok();
+        if let Some(handle) = &mut self.multimodal_handle {
+            handle.pause().ok();
+        }
         self.multimodal_hands_enabled = false;
         self.face_sources.eye_tracker_fb = None;
         self.face_sources.face_tracker_fb = None;
@@ -407,14 +414,12 @@ impl InteractionContext {
 
         // Note: We cannot enable multimodal if fb body tracking is active. It would result in a
         // ERROR_RUNTIME_FAILURE crash.
-        if config.body_tracking.is_none()
-            && config.prefers_multimodal_input
-            && extra_extensions::resume_simultaneous_hands_and_controllers_tracking_meta(
-                &self.xr_session,
-            )
-            .is_ok()
-        {
-            self.multimodal_hands_enabled = true;
+        if config.body_tracking.is_none() && config.prefers_multimodal_input {
+            if let Some(handle) = &mut self.multimodal_handle {
+                if handle.resume().is_ok() {
+                    self.multimodal_hands_enabled = true;
+                }
+            }
         }
 
         self.face_sources.eye_tracker_fb = create_ext_object(

diff --git a/alvr/client_openxr/src/lib.rs b/alvr/client_openxr/src/lib.rs
@@ -267,10 +267,9 @@ pub fn entry_point() {
 
         let interaction_context = Arc::new(RwLock::new(InteractionContext::new(
             xr_session.clone(),
+            &exts.other,
             xr_system,
             platform,
-            exts.other
-                .contains(&META_SIMULTANEOUS_HANDS_AND_CONTROLLERS_EXTENSION_NAME.to_owned()),
         )));
 
         let mut lobby = Lobby::new(