diff --git a/src/esp/bindings/SceneBindings.cpp b/src/esp/bindings/SceneBindings.cpp index d2ed3b7467..6b0db7bb53 100644 --- a/src/esp/bindings/SceneBindings.cpp +++ b/src/esp/bindings/SceneBindings.cpp @@ -228,24 +228,38 @@ void initSceneBindings(py::module& m) { "file"_a, "scene"_a, "rotation"_a) .def_property_readonly("aabb", &SemanticScene::aabb) .def_property_readonly("categories", &SemanticScene::categories, - "All semantic categories in the house") + "All semantic categories in the scene") .def_property_readonly("levels", &SemanticScene::levels, - "All levels in the house") + "All levels in the scene") .def_property_readonly("regions", &SemanticScene::regions, - "All regions in the house") + "All regions in the scene") .def_property_readonly("objects", &SemanticScene::objects, - "All object in the house") + "All object in the scene") .def_property_readonly("semantic_index_map", &SemanticScene::getSemanticIndexMap) .def("semantic_index_to_object_index", &SemanticScene::semanticIndexToObjectIndex) .def("get_regions_for_point", &SemanticScene::getRegionsForPoint, - "Compute all SemanticRegions which contain the point and return a " - "list of indices for the regions in this SemanticScene.") + R"(Compute all SemanticRegions which contain the point and return a + list of indices for the regions in this SemanticScene.)", + "point"_a) + .def("get_weighted_regions_for_point", + &SemanticScene::getWeightedRegionsForPoint, + R"("Find all SemanticRegions which contain the point and return a + sorted list of tuple pairs of the region index and a score of that + region, derived as + 1 - (region_area/ttl_region_area) + where ttl_region_area is the area of all the regions containing + the point, so that smaller regions are weighted higher. If only + one region contains the passed point, its weight will be 1.)", + "point"_a) .def("get_regions_for_points", &SemanticScene::getRegionsForPoints, - "Compute SemanticRegion containment for a set of points. Return a " - "sorted list of tuple pairs with each containing region index and " - "the percentage of points contained by that region."); + R"("Compute SemanticRegion containment for a set of points. Return a + sorted list of tuple pairs with each containing region index and + the percentage of points contained by that region. In the case of nested + regions, points are considered belonging to every region the point is + found in.)", + "points"_a); // ==== ObjectControls ==== py::class_(m, "ObjectControls") diff --git a/src/esp/scene/SemanticScene.cpp b/src/esp/scene/SemanticScene.cpp index 4563de6fdb..e8fd81511f 100644 --- a/src/esp/scene/SemanticScene.cpp +++ b/src/esp/scene/SemanticScene.cpp @@ -179,6 +179,8 @@ bool SemanticScene:: int eIdx = 0; float yExtrusion = static_cast(regionPtr->extrusionHeight_ + regionPtr->floorHeight_); + + float polyArea = 0.0f; for (std::size_t i = 0; i < numPts; ++i) { Mn::Vector3 currPoint = loopPoints[i]; regionPtr->polyLoopPoints_[i] = {currPoint.x(), currPoint.z()}; @@ -188,6 +190,10 @@ bool SemanticScene:: std::size_t nextIdx = ((i + 1) % numPts); Mn::Vector3 nextPoint = loopPoints[nextIdx]; Mn::Vector3 nextExtPt = {nextPoint.x(), yExtrusion, nextPoint.z()}; + // Find region projection area based on Green's Theorem (.5 * abs(sum + // (xprod of sequential edges))) + polyArea += + currPoint.x() * nextPoint.z() - nextPoint.x() * currPoint.z(); // Horizontal edge regionPtr->visEdges_[eIdx++] = {currPoint, nextPoint}; // Vertical edge @@ -197,7 +203,8 @@ bool SemanticScene:: // Diagonal edge regionPtr->visEdges_[eIdx++] = {currPoint, nextExtPt}; } - + // Set the polyloop area + regionPtr->area_ = .5 * abs(polyArea); scene.regions_.emplace_back(std::move(regionPtr)); } } else { // if semantic attributes specifes region annotations @@ -600,29 +607,85 @@ std::vector SemanticScene::getRegionsForPoint( } } return containingRegions; -} +} // SemanticScene::getRegionsForPoint + +std::vector> SemanticScene::getWeightedRegionsForPoint( + const Mn::Vector3& point) const { + std::vector containingRegions = getRegionsForPoint(point); + if (containingRegions.size() == 0) { + return {}; + } + + std::vector> containingRegionWeights; + containingRegionWeights.reserve(containingRegions.size()); + // Only 1 containing region, so return region idx and weight of 1 + if (containingRegions.size() == 1) { + containingRegionWeights.emplace_back( + std::pair(containingRegions[0], 1.0f)); + return containingRegionWeights; + } + + // Sum up all areas containing point + double ttlArea = 0.0f; + for (int rix : containingRegions) { + ttlArea += regions_[rix]->getArea(); + } + + for (int rix : containingRegions) { + containingRegionWeights.emplace_back(std::pair( + rix, 1.0f - (regions_[rix]->getArea() / ttlArea))); + } -std::vector> SemanticScene::getRegionsForPoints( + std::sort(containingRegionWeights.begin(), containingRegionWeights.end(), + [](const std::pair& a, std::pair& b) { + return a.second > b.second; + }); + return containingRegionWeights; + +} // SemanticScene::getWeightedRegionsForPoint + +std::vector> SemanticScene::getRegionsForPoints( const std::vector& points) const { - std::vector> containingRegionWeights; + // Weights for every point for every region + std::vector> regAreaWeightsForPoints; + regAreaWeightsForPoints.reserve(points.size()); + for (int i = 0; i < points.size(); ++i) { + // Get this point's weighted regions + auto regWeightsForPoint = getWeightedRegionsForPoint(points[i]); + // Initialize all region weights to be 0 + std::vector allRegionWeights(regions_.size(), 0); + // Set the weight for the containing region with the smallest area (if + // nested regions) for this particular point. + // Set the vote for each region to be equal. + for (const std::pair& regionWeight : regWeightsForPoint) { + allRegionWeights[regionWeight.first] = 1.0; + } + // Save this points region weight vector + regAreaWeightsForPoints.emplace_back(allRegionWeights); + } + + std::vector> containingRegionWeights; + // Will only have at max the number of regions in the scene + containingRegionWeights.reserve(regions_.size()); for (int rix = 0; rix < regions_.size(); ++rix) { - float containmentCount = 0; - for (const auto& point : points) { - if (regions_[rix]->contains(point)) { - containmentCount += 1; - } + double containmentWeight = 0; + for (int i = 0; i < points.size(); ++i) { + std::vector regWtsForPoint = regAreaWeightsForPoints[i]; + containmentWeight += regWtsForPoint[rix]; } - if (containmentCount > 0) { + if (containmentWeight > 0) { containingRegionWeights.emplace_back( - std::pair(rix, containmentCount / points.size())); + std::pair(rix, containmentWeight / points.size())); } } + // Free up unused capacity - every region probably does not contain a tested + // point + containingRegionWeights.shrink_to_fit(); std::sort(containingRegionWeights.begin(), containingRegionWeights.end(), - [](const std::pair& a, std::pair& b) { + [](const std::pair& a, std::pair& b) { return a.second > b.second; }); return containingRegionWeights; -} - +} // SemanticScene::getRegionsForPoints } // namespace scene } // namespace esp diff --git a/src/esp/scene/SemanticScene.h b/src/esp/scene/SemanticScene.h index d02a5efeee..d978cdc594 100644 --- a/src/esp/scene/SemanticScene.h +++ b/src/esp/scene/SemanticScene.h @@ -293,13 +293,26 @@ class SemanticScene { std::vector getRegionsForPoint(const Mn::Vector3& point) const; /** - * @brief Compute SemanticRegion containment for a set of points. + * @brief Compute all the SemanticRegions that contain the passed point, and + * return a vector of indices and weights for each region, where the weights + * are inverted area of the region (smaller regions weighted higher) + * @param point The query point. + * @return std::vector> A sorted list of tuples + * containing region index and inverse area of that region + */ + std::vector> getWeightedRegionsForPoint( + const Mn::Vector3& point) const; + + /** + * @brief Compute SemanticRegion containment for a set of points. It is + * assumed the set of points belong to the same construct (i.e. points from an + * individual object's mesh) * @param points A set of points to test for semantic containment. * @return std::vector> A sorted list of tuples * containing region index and percentage of input points contained in that * region. */ - std::vector> getRegionsForPoints( + std::vector> getRegionsForPoints( const std::vector& points) const; protected: @@ -521,6 +534,17 @@ class SemanticRegion { SemanticCategory::ptr category() const { return category_; } + /** + * @brief Returns the area of the polyloop forming the base of the region + * extrusion + */ + double getArea() const { return area_; } + /** + * @brief Returns the volume of the polyloop-based extrusion defining the + * bounds of this region. + */ + double getVolume() const { return area_ * extrusionHeight_; } + protected: int index_{}; int parentIndex_{}; @@ -530,6 +554,9 @@ class SemanticRegion { std::string name_; + // The area of the surface enclosed by the region + double area_{}; + // Height of extrusion for Extruded poly-loop-based volumes double extrusionHeight_{}; // Floor height @@ -545,7 +572,7 @@ class SemanticRegion { std::shared_ptr level_; friend SemanticScene; ESP_SMART_POINTERS(SemanticRegion) -}; +}; // class SemanticRegion //! Represents a distinct semantically annotated object class SemanticObject { diff --git a/src/esp/sim/Simulator.h b/src/esp/sim/Simulator.h index 14aca8fb7f..0a0ef65dbe 100644 --- a/src/esp/sim/Simulator.h +++ b/src/esp/sim/Simulator.h @@ -73,16 +73,6 @@ class Simulator { void seed(uint32_t newSeed); std::shared_ptr getRenderer() { return renderer_; } - std::shared_ptr getSemanticScene() { - return resourceManager_->getSemanticScene(); - } - - /** - * @brief Return a view of the currently set Semantic scene colormap. - */ - const std::vector& getSemanticSceneColormap() const { - return resourceManager_->getSemanticSceneColormap(); - } inline void getRenderGLContext() { // acquire GL context from background thread, if background rendering @@ -92,29 +82,6 @@ class Simulator { } } - /** @brief check if the semantic scene exists.*/ - bool semanticSceneExists() const { - return resourceManager_->semanticSceneExists(); - } - - /** - * @brief get the current active scene graph - */ - scene::SceneGraph& getActiveSceneGraph() { - CORRADE_INTERNAL_ASSERT(std::size_t(activeSceneID_) < sceneID_.size()); - return sceneManager_->getSceneGraph(activeSceneID_); - } - - /** @brief Check to see if there is a SemanticSceneGraph for rendering */ - bool semanticSceneGraphExists() const { - return std::size_t(activeSemanticSceneID_) < sceneID_.size(); - } - - /** @brief get the semantic scene's SceneGraph for rendering */ - scene::SceneGraph& getActiveSemanticSceneGraph() { - CORRADE_INTERNAL_ASSERT(semanticSceneGraphExists()); - return sceneManager_->getSceneGraph(activeSemanticSceneID_); - } std::shared_ptr getGfxReplayManager() { return gfxReplayMgr_; } @@ -218,7 +185,44 @@ class Simulator { } /** - * @brief Build a map keyed by semantic color/id referencing a vector + * @brief get the current active scene graph + */ + scene::SceneGraph& getActiveSceneGraph() { + CORRADE_INTERNAL_ASSERT(std::size_t(activeSceneID_) < sceneID_.size()); + return sceneManager_->getSceneGraph(activeSceneID_); + } + + /////////////////////////// + // Semantic Scene and Data + std::shared_ptr getSemanticScene() { + return resourceManager_->getSemanticScene(); + } + + /** + * @brief Return a view of the currently set Semantic scene colormap. + */ + const std::vector& getSemanticSceneColormap() const { + return resourceManager_->getSemanticSceneColormap(); + } + + /** @brief check if the semantic scene exists.*/ + bool semanticSceneExists() const { + return resourceManager_->semanticSceneExists(); + } + + /** @brief Check to see if there is a SemanticSceneGraph for rendering */ + bool semanticSceneGraphExists() const { + return std::size_t(activeSemanticSceneID_) < sceneID_.size(); + } + + /** @brief get the semantic scene's SceneGraph for rendering */ + scene::SceneGraph& getActiveSemanticSceneGraph() { + CORRADE_INTERNAL_ASSERT(semanticSceneGraphExists()); + return sceneManager_->getSceneGraph(activeSemanticSceneID_); + } + + /** + * @brief Build a map keyed by semantic color/id referencing a list of * connected component-based Semantic objects. */ std::unordered_map> @@ -248,6 +252,9 @@ class Simulator { return {}; } + /////////////////////////// + // End Semantic Scene and Data + /** * @brief Builds a @ref esp::metadata::attributes::SceneInstanceAttributes describing the * current scene configuration, and saves it to a JSON file, using @p diff --git a/tests/test_semantic_scene.py b/tests/test_semantic_scene.py index db150968bf..726d6d9fbb 100644 --- a/tests/test_semantic_scene.py +++ b/tests/test_semantic_scene.py @@ -135,7 +135,6 @@ def test_semantic_regions(): (-1 * p) for p in hit_test_points ] # add one less to create imbalance regions_weights = semantic_scene.get_regions_for_points(mixed_points) - print(f"regions_weights = {regions_weights}") assert regions_weights[0][0] == 1 # bathroom with more points comes first assert ( regions_weights[0][1] >= 0.51