Support for Simultaneous RIR Rendering for Multiple Sound Sources

[Redrain54]

Habitat-Sim version

vx.x.x

Habitat is under active development, and we advise users to restrict themselves to stable releases. Are you using the latest release version of Habitat-Sim? Your question may already be addressed in the latest version. We may also not be able to help with problems in earlier versions because they sometimes lack the more verbose logging needed for debugging.

Main branch contains 'bleeding edge' code and should be used at your own risk.

Docs and Tutorials

Did you read the docs? https://aihabitat.org/docs/habitat-sim/

Did you check out the tutorials? https://aihabitat.org/tutorial/2020/

Perhaps your question is answered there. If not, carry on!

❓ Questions and Help

class Scene:
"""
Soundspaces scene including room, receiver, and source list
"""

def __init__(self,
             room: str,
             source_name_list: T.List[str],
             receiver: Receiver = None,
             source_list: T.List[Source] = None,
             include_visual_sensor: bool = True,
             device: torch.device = torch.device('cpu'),
             image_size: T.Tuple[int, int] = (512, 256),
             hfov: float = 90.0,
             use_default_material: bool = False,
             channel_type: str = 'Ambisonics',
             channel_order: int = 1
             ):

    # Set scene
    self.room = room
    self.n_sources = len(source_name_list)
    assert self.n_sources > 0
    self.receiver = receiver
    self.source_list = source_list
    self.source_current = None
    self.include_visual_sensor = include_visual_sensor
    self.device = device  # where to store IR

    # Set channel config for soundspaces
    self.channel = {}
    self.channel['type'] = channel_type
    self.channel['order'] = channel_order
    if channel_type == 'Ambisonics':
        self.channel_count = (self.channel['order'] + 1)**2
    elif channel_type == 'Binaural':
        self.channel_count = 2
    elif channel_type == 'Mono':
        self.channel_count = 1

    # Set aihabitat config for soundspaces
    self.aihabitat = {}
    self.aihabitat['default_agent'] = 0
    self.aihabitat['sensor_height'] = 1.5
    self.aihabitat['height'] = image_size[0]
    self.aihabitat['width'] = image_size[1]
    self.aihabitat['hfov'] = hfov

    # Set acoustics config for soundspaces
    self.acoustic_config = {}
    self.acoustic_config['sampleRate'] = 16000
    self.acoustic_config['direct'] = True
    self.acoustic_config['indirect'] = True
    self.acoustic_config['diffraction'] = True
    self.acoustic_config['transmission'] = True
    self.acoustic_config['directSHOrder'] = 5
    self.acoustic_config['indirectSHOrder'] = 3
    self.acoustic_config['unitScale'] = 1
    self.acoustic_config['frequencyBands'] = 32
    self.acoustic_config['indirectRayCount'] = 50000

    # Set audio material
    if use_default_material:
        self.audio_material = 'SonicSet/material/mp3d_material_config_default.json'
    else:
        self.audio_material = 'SonicSet/material/mp3d_material_config.json'

    # Create simulation
    self.create_scene()

    # Randomly set source and receiver position
    source_position, source_rotation = None, None
    receiver_position, receiver_rotation = None, None

    # Create receiver (inside the room)
    if self.receiver is None:
        # random receiver
        self.create_receiver(receiver_position, receiver_rotation)
    else:
        # input receiver
        self.update_receiver(self.receiver)

def create_scene(self):
    """
    Given the configuration, create a scene for soundspaces
    """

    # Set backend configuration
    backend_cfg = habitat_sim.SimulatorConfiguration()
    backend_cfg.scene_id = f'mp3d/{self.room}/{self.room}.glb'
    backend_cfg.scene_dataset_config_file = 'SonicSet/material/mp3d.scene_dataset_config.json'
    backend_cfg.load_semantic_mesh = True  
    backend_cfg.enable_physics = False     

    agent_config = habitat_sim.AgentConfiguration()

    if self.include_visual_sensor:
        # Set color sensor
        rgb_sensor_spec = habitat_sim.CameraSensorSpec()
        rgb_sensor_spec.uuid = "color_sensor"
        rgb_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR
        rgb_sensor_spec.resolution = [self.aihabitat['height'], self.aihabitat['width']]
        rgb_sensor_spec.position = [0.0, self.aihabitat["sensor_height"], 0.0]
        rgb_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE # “针孔”（Pinhole）相机模型
        rgb_sensor_spec.hfov = self.aihabitat["hfov"]
        agent_config.sensor_specifications = [rgb_sensor_spec]

        # Set depth sensor
        depth_sensor_spec = habitat_sim.CameraSensorSpec()
        depth_sensor_spec.uuid = "depth_sensor"
        depth_sensor_spec.sensor_type = habitat_sim.SensorType.DEPTH
        depth_sensor_spec.resolution = [self.aihabitat["height"], self.aihabitat["width"]]
        depth_sensor_spec.position = [0.0, self.aihabitat["sensor_height"], 0.0]
        depth_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
        depth_sensor_spec.hfov = self.aihabitat["hfov"]
        agent_config.sensor_specifications.append(depth_sensor_spec)

    # Set simulator configuration
    cfg = habitat_sim.Configuration(backend_cfg, [agent_config])

    # Set simulator
    sim = habitat_sim.Simulator(cfg)

    # set navmesh path for searching for navigatable points
    navmesh = f'mp3d/{self.room}/{self.room}.navmesh'
    sim.pathfinder.load_nav_mesh(navmesh)

    # seed for navmesh
    sim.seed(random.randint(0, 1024))

    # Set simulation
    self.sim = sim
    print('Scene created!')

    return self

def add_audio_sensor(self):
    """
    Add audio sensor to the scene
    """

    # set audio sensor
    audio_sensor_spec = habitat_sim.AudioSensorSpec()
    audio_sensor_spec.uuid = "audio_sensor"
    audio_sensor_spec.enableMaterials = True  # make sure _semantic.ply file is in the scene folder
    audio_sensor_spec.channelLayout.channelType = getattr(habitat_sim.sensor.RLRAudioPropagationChannelLayoutType, self.channel['type'])
    audio_sensor_spec.channelLayout.channelCount = self.channel_count  # ambisonics

    # Set acoustic configuration
    audio_sensor_spec.acousticsConfig.sampleRate = self.acoustic_config['sampleRate']
    audio_sensor_spec.acousticsConfig.direct = self.acoustic_config['direct']
    audio_sensor_spec.acousticsConfig.indirect = self.acoustic_config['indirect']
    audio_sensor_spec.acousticsConfig.diffraction = self.acoustic_config['diffraction']
    audio_sensor_spec.acousticsConfig.transmission = self.acoustic_config['transmission']
    audio_sensor_spec.acousticsConfig.directSHOrder = self.acoustic_config['directSHOrder']
    audio_sensor_spec.acousticsConfig.indirectSHOrder = self.acoustic_config['indirectSHOrder']
    audio_sensor_spec.acousticsConfig.unitScale = self.acoustic_config['unitScale']
    audio_sensor_spec.acousticsConfig.frequencyBands = self.acoustic_config['frequencyBands']
    audio_sensor_spec.acousticsConfig.indirectRayCount = self.acoustic_config['indirectRayCount']

    # Initialize receiver
    audio_sensor_spec.position = [0.0, self.aihabitat['sensor_height'], 0.0]  # audio sensor has a height of 1.5m
    self.sim.add_sensor(audio_sensor_spec)

    audio_sensor = self.sim.get_agent(self.aihabitat['default_agent'])._sensors['audio_sensor']
    audio_sensor.setAudioMaterialsJSON(self.audio_material)

    return self

def render_ir(self,
              source_id: int
              ) -> torch.Tensor:
    """
    Render IR given the source ID
    """

    source = self.source_list[source_id]
    self.update_source(source)
    ir = torch.tensor(self.sim.get_sensor_observations()['audio_sensor'], device=self.device)

    return ir

Above is my code about to simulate an audio with rir in the scenes. If I want to get simultaneous rir rendering for multiple sound sources, is there an API to ues? I'm sorry that I can't find this current in your tutorial. I'm appreciate it if you can answer me.