Skip to content

Commit

Permalink
fix(client_openxr): 🐛 Try fixing crash on Quest 2 related to multimodal
Browse files Browse the repository at this point in the history
  • Loading branch information
@zmerp
zmerp committed Jan 19, 2025
1 parent a785d72 commit 1a1f022
Show file tree
Hide file tree
Showing 6 changed files with 157 additions and 88 deletions.
17 changes: 1 addition & 16 deletions alvr/client_openxr/src/extra_extensions/eye_gaze_interaction.rs
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,12 @@
use openxr::{self as xr, sys};
use std::ptr;

fn get_props<G, T>(session: &xr::Session<G>, system: xr::SystemId, default_struct: T) -> Option<T> {
let instance = session.instance();

let mut props = default_struct;
let mut system_properties = sys::SystemProperties::out((&mut props as *mut T).cast());
let result = unsafe {
(instance.fp().get_system_properties)(
instance.as_raw(),
system,
system_properties.as_mut_ptr(),
)
};
(result.into_raw() >= 0).then_some(props)
}

pub fn supports_eye_gaze_interaction<G>(session: &xr::Session<G>, system: xr::SystemId) -> bool {
if session.instance().exts().ext_eye_gaze_interaction.is_none() {
return false;
}

get_props(
super::get_props(
session,
system,
sys::SystemEyeGazeInteractionPropertiesEXT {
Expand Down
20 changes: 20 additions & 0 deletions alvr/client_openxr/src/extra_extensions/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,23 @@ fn xr_res(result: sys::Result) -> xr::Result<()> {
Err(result)
}
}

fn get_props<G, T>(
session: &xr::Session<G>,
system: xr::SystemId,
default_struct: T,
) -> xr::Result<T> {
let instance = session.instance();

let mut props = default_struct;
let mut system_properties = sys::SystemProperties::out((&mut props as *mut T).cast());
let result = unsafe {
(instance.fp().get_system_properties)(
instance.as_raw(),
system,
system_properties.as_mut_ptr(),
)
};

xr_res(result).map(|_| props)
}
158 changes: 106 additions & 52 deletions alvr/client_openxr/src/extra_extensions/multimodal_input.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,20 @@ pub const META_SIMULTANEOUS_HANDS_AND_CONTROLLERS_EXTENSION_NAME: &str =
"XR_META_simultaneous_hands_and_controllers";
pub const META_DETACHED_CONTROLLERS_EXTENSION_NAME: &str = "XR_META_detached_controllers";

static TYPE_SYSTEM_SIMULTANEOUS_HANDS_AND_CONTROLLERS_PROPERTIES_META: Lazy<xr::StructureType> =
Lazy::new(|| xr::StructureType::from_raw(1000532001));
static TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_RESUME_INFO_META: Lazy<xr::StructureType> =
Lazy::new(|| xr::StructureType::from_raw(1000532002));
static TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_PAUSE_INFO_META: Lazy<xr::StructureType> =
Lazy::new(|| xr::StructureType::from_raw(1000532003));

#[repr(C)]
pub struct SystemSymultaneousHandsAndControllersPropertiesMETA {
ty: xr::StructureType,
next: *const c_void,
supports_simultaneous_hands_and_controllers: sys::Bool32,
}

#[repr(C)]
pub struct SimultaneousHandsAndControllersTrackingResumeInfoMETA {
ty: xr::StructureType,
Expand All @@ -39,60 +48,105 @@ pub type PauseSimultaneousHandsAndControllersTrackingMETA =
*const SimultaneousHandsAndControllersTrackingPauseInfoMETA,
) -> sys::Result;

pub fn resume_simultaneous_hands_and_controllers_tracking_meta<G>(
session: &xr::Session<G>,
) -> xr::Result<()> {
let resume_simultaneous_hands_and_controllers_tracking_meta = unsafe {
let mut resume_simultaneous_hands_and_controllers_tracking_meta = None;
let _ = (session.instance().fp().get_instance_proc_addr)(
session.instance().as_raw(),
c"xrResumeSimultaneousHandsAndControllersTrackingMETA".as_ptr(),
&mut resume_simultaneous_hands_and_controllers_tracking_meta,
);

resume_simultaneous_hands_and_controllers_tracking_meta.map(|pfn| {
mem::transmute::<VoidFunction, ResumeSimultaneousHandsAndControllersTrackingMETA>(pfn)
})
}
.ok_or(sys::Result::ERROR_EXTENSION_NOT_PRESENT)?;

let resume_info = SimultaneousHandsAndControllersTrackingResumeInfoMETA {
ty: *TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_RESUME_INFO_META,
next: ptr::null(),
};
unsafe {
super::xr_res(resume_simultaneous_hands_and_controllers_tracking_meta(
session.as_raw(),
&resume_info,
))
}
pub struct MultimodalMeta {
pub session: xr::Session<xr::AnyGraphics>,
pub resume_simultaneous_hands_and_controllers_tracking_meta:
ResumeSimultaneousHandsAndControllersTrackingMETA,
pub pause_simultaneous_hands_and_controllers_tracking_meta:
PauseSimultaneousHandsAndControllersTrackingMETA,
}

pub fn pause_simultaneous_hands_and_controllers_tracking_meta<G>(
session: &xr::Session<G>,
) -> xr::Result<()> {
let pause_simultaneous_hands_and_controllers_tracking_meta = unsafe {
let mut pause_simultaneous_hands_and_controllers_tracking_meta = None;
let _ = (session.instance().fp().get_instance_proc_addr)(
session.instance().as_raw(),
c"xrPauseSimultaneousHandsAndControllersTrackingMETA".as_ptr(),
&mut pause_simultaneous_hands_and_controllers_tracking_meta,
);

pause_simultaneous_hands_and_controllers_tracking_meta.map(|pfn| {
mem::transmute::<VoidFunction, PauseSimultaneousHandsAndControllersTrackingMETA>(pfn)
})
impl MultimodalMeta {
pub fn new<G>(
session: xr::Session<G>,
extra_extensions: &[String],
system: xr::SystemId,
) -> xr::Result<Self> {
if !extra_extensions
.contains(&META_SIMULTANEOUS_HANDS_AND_CONTROLLERS_EXTENSION_NAME.to_owned())
|| !extra_extensions.contains(&META_DETACHED_CONTROLLERS_EXTENSION_NAME.to_owned())
{
return Err(sys::Result::ERROR_EXTENSION_NOT_PRESENT);
}

let resume_simultaneous_hands_and_controllers_tracking_meta = unsafe {
let mut resume_simultaneous_hands_and_controllers_tracking_meta = None;
let _ = (session.instance().fp().get_instance_proc_addr)(
session.instance().as_raw(),
c"xrResumeSimultaneousHandsAndControllersTrackingMETA".as_ptr(),
&mut resume_simultaneous_hands_and_controllers_tracking_meta,
);

resume_simultaneous_hands_and_controllers_tracking_meta.map(|pfn| {
mem::transmute::<VoidFunction, ResumeSimultaneousHandsAndControllersTrackingMETA>(
pfn,
)
})
}
.ok_or(sys::Result::ERROR_EXTENSION_NOT_PRESENT)?;

let pause_simultaneous_hands_and_controllers_tracking_meta = unsafe {
let mut pause_simultaneous_hands_and_controllers_tracking_meta = None;
let _ = (session.instance().fp().get_instance_proc_addr)(
session.instance().as_raw(),
c"xrPauseSimultaneousHandsAndControllersTrackingMETA".as_ptr(),
&mut pause_simultaneous_hands_and_controllers_tracking_meta,
);

pause_simultaneous_hands_and_controllers_tracking_meta.map(|pfn| {
mem::transmute::<VoidFunction, PauseSimultaneousHandsAndControllersTrackingMETA>(
pfn,
)
})
}
.ok_or(sys::Result::ERROR_EXTENSION_NOT_PRESENT)?;

let props = super::get_props(
&session,
system,
SystemSymultaneousHandsAndControllersPropertiesMETA {
ty: *TYPE_SYSTEM_SIMULTANEOUS_HANDS_AND_CONTROLLERS_PROPERTIES_META,
next: std::ptr::null(),
supports_simultaneous_hands_and_controllers: xr::sys::FALSE,
},
)?;

if props.supports_simultaneous_hands_and_controllers.into() {
Ok(Self {
session: session.into_any_graphics(),
resume_simultaneous_hands_and_controllers_tracking_meta,
pause_simultaneous_hands_and_controllers_tracking_meta,
})
} else {
Err(sys::Result::ERROR_FEATURE_UNSUPPORTED)
}
}

pub fn resume(&self) -> xr::Result<()> {
let resume_info = SimultaneousHandsAndControllersTrackingResumeInfoMETA {
ty: *TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_RESUME_INFO_META,
next: ptr::null(),
};
unsafe {
super::xr_res((self
.resume_simultaneous_hands_and_controllers_tracking_meta)(
self.session.as_raw(),
&resume_info,
))
}
}
.ok_or(sys::Result::ERROR_EXTENSION_NOT_PRESENT)?;

let pause_info = SimultaneousHandsAndControllersTrackingPauseInfoMETA {
ty: *TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_PAUSE_INFO_META,
next: ptr::null(),
};
unsafe {
super::xr_res(pause_simultaneous_hands_and_controllers_tracking_meta(
session.as_raw(),
&pause_info,
))

pub fn pause(&self) -> xr::Result<()> {
let pause_info = SimultaneousHandsAndControllersTrackingPauseInfoMETA {
ty: *TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_PAUSE_INFO_META,
next: ptr::null(),
};
unsafe {
super::xr_res((self
.pause_simultaneous_hands_and_controllers_tracking_meta)(
self.session.as_raw(),
&pause_info,
))
}
}
}
38 changes: 22 additions & 16 deletions alvr/client_openxr/src/interaction.rs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use crate::{
extra_extensions::{
self, BodyTrackerFB, EyeTrackerSocial, FaceTracker2FB, FacialTrackerHTC,
self, BodyTrackerFB, EyeTrackerSocial, FaceTracker2FB, FacialTrackerHTC, MultimodalMeta,
BODY_JOINT_SET_FULL_BODY_META, FULL_BODY_JOINT_COUNT_META,
FULL_BODY_JOINT_LEFT_FOOT_BALL_META, FULL_BODY_JOINT_LEFT_LOWER_LEG_META,
FULL_BODY_JOINT_RIGHT_FOOT_BALL_META, FULL_BODY_JOINT_RIGHT_LOWER_LEG_META,
Expand Down Expand Up @@ -108,17 +108,17 @@ pub struct InteractionContext {
pub action_set: xr::ActionSet,
pub button_actions: HashMap<u64, ButtonAction>,
pub hands_interaction: [HandInteraction; 2],
pub multimodal_hands_enabled: bool,
pub multimodal_hands: Option<(MultimodalMeta, bool)>,
pub face_sources: FaceSources,
pub body_sources: BodySources,
}

impl InteractionContext {
pub fn new(
xr_session: xr::Session<xr::OpenGlEs>,
extra_extensions: &[String],
xr_system: xr::SystemId,
platform: Platform,
supports_multimodal: bool,
) -> Self {
let xr_instance = xr_session.instance();

Expand Down Expand Up @@ -229,9 +229,13 @@ impl InteractionContext {
"/user/hand/right/output/haptic",
));

let multimodal = create_ext_object("MultimodalMeta", Some(true), || {
MultimodalMeta::new(xr_session.clone(), &extra_extensions, xr_system)

Check warning on line 233 in alvr/client_openxr/src/interaction.rs

View workflow job for this annotation

GitHub Actions / check-linux 

warning: this expression creates a reference which is immediately dereferenced by the compiler
   --> alvr/client_openxr/src/interaction.rs:233:53
    |
233 |             MultimodalMeta::new(xr_session.clone(), &extra_extensions, xr_system)
    |                                                     ^^^^^^^^^^^^^^^^^ help: change this to: `extra_extensions`
    |
    = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#needless_borrow
    = note: `#[warn(clippy::needless_borrow)]` on by default

Check warning on line 233 in alvr/client_openxr/src/interaction.rs

View workflow job for this annotation

GitHub Actions / check-windows 

warning: this expression creates a reference which is immediately dereferenced by the compiler
   --> alvr\client_openxr\src\interaction.rs:233:53
    |
233 |             MultimodalMeta::new(xr_session.clone(), &extra_extensions, xr_system)
    |                                                     ^^^^^^^^^^^^^^^^^ help: change this to: `extra_extensions`
    |
    = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#needless_borrow
    = note: `#[warn(clippy::needless_borrow)]` on by default
});

let left_detached_controller_pose_action;
let right_detached_controller_pose_action;
if supports_multimodal {
if multimodal.is_some() {
// Note: when multimodal input is enabled, both controllers and hands will always be active.
// To be able to detect when controllers are actually held, we have to register detached
// controllers pose; the controller pose will be diverted to the detached controllers when
Expand Down Expand Up @@ -355,7 +359,7 @@ impl InteractionContext {
skeleton_tracker: right_hand_tracker,
},
],
multimodal_hands_enabled: false,
multimodal_hands: multimodal.map(|m| (m, false)),
face_sources: FaceSources {
combined_eyes_source,
eye_tracker_fb: None,
Expand All @@ -371,9 +375,12 @@ impl InteractionContext {

pub fn select_sources(&mut self, config: &InteractionSourcesConfig) {
// First of all, disable/delete all sources. This ensures there are no conflicts
extra_extensions::pause_simultaneous_hands_and_controllers_tracking_meta(&self.xr_session)
.ok();
self.multimodal_hands_enabled = false;
if let Some((handle, enabled)) = &mut self.multimodal_hands {
if *enabled {
handle.pause().ok();
*enabled = false;
}
}
self.face_sources.eye_tracker_fb = None;
self.face_sources.face_tracker_fb = None;
self.face_sources.eye_tracker_htc = None;
Expand Down Expand Up @@ -407,14 +414,13 @@ impl InteractionContext {

// Note: We cannot enable multimodal if fb body tracking is active. It would result in a
// ERROR_RUNTIME_FAILURE crash.
if config.body_tracking.is_none()
&& config.prefers_multimodal_input
&& extra_extensions::resume_simultaneous_hands_and_controllers_tracking_meta(
&self.xr_session,
)
.is_ok()
{
self.multimodal_hands_enabled = true;
if config.body_tracking.is_none() && config.prefers_multimodal_input {
if let Some((handle, enabled)) = &mut self.multimodal_hands {
if !*enabled {
handle.resume().ok();
*enabled = true;
}
}
}

self.face_sources.eye_tracker_fb = create_ext_object(
Expand Down
3 changes: 1 addition & 2 deletions alvr/client_openxr/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,10 +267,9 @@ pub fn entry_point() {

let interaction_context = Arc::new(RwLock::new(InteractionContext::new(
xr_session.clone(),
&exts.other,
xr_system,
platform,
exts.other
.contains(&META_SIMULTANEOUS_HANDS_AND_CONTROLLERS_EXTENSION_NAME.to_owned()),
)));

let mut lobby = Lobby::new(
Expand Down
9 changes: 7 additions & 2 deletions alvr/client_openxr/src/stream.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -510,12 +510,17 @@ fn stream_input_loop(

// Note: When multimodal input is enabled, we are sure that when free hands are used
// (not holding controllers) the controller data is None.
if int_ctx.multimodal_hands_enabled || left_hand_skeleton.is_none() {
let multimodal_enabled = int_ctx
.multimodal_hands
.as_ref()
.map(|(_, enabled)| *enabled)
.unwrap_or(false);
if multimodal_enabled || left_hand_skeleton.is_none() {
if let Some(motion) = left_hand_motion {
device_motions.push((*HAND_LEFT_ID, motion));
}
}
if int_ctx.multimodal_hands_enabled || right_hand_skeleton.is_none() {
if multimodal_enabled || right_hand_skeleton.is_none() {
if let Some(motion) = right_hand_motion {
device_motions.push((*HAND_RIGHT_ID, motion));
}
Expand Down

0 comments on commit 1a1f022

Please sign in to comment.