diff --git a/com.unity.perception/CHANGELOG.md b/com.unity.perception/CHANGELOG.md index 944e1b559..12aec4910 100644 --- a/com.unity.perception/CHANGELOG.md +++ b/com.unity.perception/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Upgrade Notes ### Added +Added support for hierarchical label aggregation in instance segmentation labeler ### Changed diff --git a/com.unity.perception/Editor/GroundTruth/LabelConfigEditor.cs b/com.unity.perception/Editor/GroundTruth/LabelConfigEditor.cs index e9326b975..f994e19ef 100644 --- a/com.unity.perception/Editor/GroundTruth/LabelConfigEditor.cs +++ b/com.unity.perception/Editor/GroundTruth/LabelConfigEditor.cs @@ -135,7 +135,7 @@ void InitUi() m_AddNewLabelButton.clicked += () => { AddNewLabel(m_AddedLabels); }; #if UNITY_2020_1_OR_NEWER - m_LabelListView.onSelectionChange += UpdateMoveButtonState; + m_LabelListView.selectionChanged += UpdateMoveButtonState; #else m_LabelListView.onSelectionChanged += UpdateMoveButtonState; #endif diff --git a/com.unity.perception/Editor/Unity.Perception.Editor.api b/com.unity.perception/Editor/Unity.Perception.Editor.api index 0b3cce45c..740362a37 100644 --- a/com.unity.perception/Editor/Unity.Perception.Editor.api +++ b/com.unity.perception/Editor/Unity.Perception.Editor.api @@ -80,7 +80,7 @@ namespace UnityEngine.Perception.Settings namespace UnityEngine.Perception.UIElements { - public class UIntField : UnityEditor.UIElements.TextValueField + public class UIntField : UnityEngine.UIElements.TextValueField { public static readonly string inputUssClassName; public static readonly string labelUssClassName; @@ -88,7 +88,7 @@ namespace UnityEngine.Perception.UIElements public UIntField() {} public UIntField(int maxLength) {} public UIntField(string label, int maxLength = -1) {} - public override void ApplyInputDeviceDelta(Vector3 delta, UnityEditor.UIElements.DeltaSpeed speed, System.UInt32 startValue); + public override void ApplyInputDeviceDelta(Vector3 delta, UnityEngine.UIElements.DeltaSpeed speed, System.UInt32 startValue); public static System.UInt32 ClampInput(long input); protected override System.UInt32 StringToValue(string str); protected override string ValueToString(System.UInt32 v); @@ -96,7 +96,7 @@ namespace UnityEngine.Perception.UIElements { public UxmlFactory() {} } - public class UxmlTraits : UnityEditor.UIElements.TextValueFieldTraits + public class UxmlTraits : UnityEngine.UIElements.TextValueFieldTraits { public UxmlTraits() {} } diff --git a/com.unity.perception/Runtime/GroundTruth/LabelManagement/IdLabelConfig.cs b/com.unity.perception/Runtime/GroundTruth/LabelManagement/IdLabelConfig.cs index 2ed60cf07..620f1bfc0 100644 --- a/com.unity.perception/Runtime/GroundTruth/LabelManagement/IdLabelConfig.cs +++ b/com.unity.perception/Runtime/GroundTruth/LabelManagement/IdLabelConfig.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; using JetBrains.Annotations; diff --git a/com.unity.perception/Runtime/GroundTruth/Labelers/InstanceSegmentation/InstanceSegmentationLabeler.cs b/com.unity.perception/Runtime/GroundTruth/Labelers/InstanceSegmentation/InstanceSegmentationLabeler.cs index c3947e17d..d272a8638 100644 --- a/com.unity.perception/Runtime/GroundTruth/Labelers/InstanceSegmentation/InstanceSegmentationLabeler.cs +++ b/com.unity.perception/Runtime/GroundTruth/Labelers/InstanceSegmentation/InstanceSegmentationLabeler.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using Unity.Collections; using UnityEngine; using UnityEngine.Experimental.Rendering; @@ -49,6 +50,14 @@ public sealed class InstanceSegmentationLabeler : CameraLabeler, IOverlayPanelPr /// public IdLabelConfig idLabelConfig; + /// + /// Should child objects, defined by their label hierarchy be reported as an individual instance, or as + /// a part of their parent object. If this value is true, the children will be reported as a part of their + /// parent. + /// + [Tooltip("Should the instance segmentation capture the single instance of the parent gameobject, or the individual sub-components")] + public bool aggregateChildren = false; + /// public override string description => InstanceSegmentationDefinition.labelDescription; @@ -115,6 +124,44 @@ protected override void Cleanup() } } + NativeArray GetSegmentationColors(int frame) + { + var instanceIndices = LabelManager.singleton.instanceIds; + var max = uint.MinValue; + foreach (var i in instanceIndices) + { + if (i > max) max = i; + } + + var activeColors = new NativeArray((int)(max + 1), Allocator.Temp, NativeArrayOptions.ClearMemory); + + for (var i = 0; i < max + 1; i++) + { + activeColors[i] = InstanceIdToColorMapping.GetColorFromInstanceId((uint)i); + } + + if (!aggregateChildren) return activeColors; + + if (!PerceptionCamera.savedHierarchies.TryGetValue(frame, out var hierarchyInformation)) + { + Debug.LogError($"Could not get the scene hierarchy info for the current frame: {frame}"); + return activeColors; + } + + foreach (var i in instanceIndices) + { + if (!hierarchyInformation.hierarchy.TryGetValue(i, out var node)) + { + continue; + } + + var idx = (int)(node?.parentInstanceId ?? i); + activeColors[(int)i] = activeColors[idx]; + } + + return activeColors; + } + /// protected override void OnEndRendering(ScriptableRenderContext ctx) { @@ -126,9 +173,10 @@ protected override void OnEndRendering(ScriptableRenderContext ctx) // Create a compute buffer that maps instanceIndices to unique instance segmentation colors. var instanceIndices = LabelManager.singleton.instanceIds; - var instanceSegmentationColors = LabelManager.singleton.instanceSegmentationColors; + var colors = GetSegmentationColors(Time.frameCount); + var colorBuffer = new ComputeBuffer(instanceIndices.Length, sizeof(uint)); - cmd.SetBufferData(colorBuffer, instanceSegmentationColors.AsArray(), 0, 0, colorBuffer.count); + cmd.SetBufferData(colorBuffer, colors, 0, 0, colorBuffer.count); // Use a compute shader to map each pixel instance index to a unique color // to create the instance segmentation color texture. @@ -149,6 +197,7 @@ protected override void OnEndRendering(ScriptableRenderContext ctx) colorBuffer.Dispose(); }); + colors.Dispose(); ctx.ExecuteCommandBuffer(cmd); CommandBufferPool.Release(cmd); } @@ -159,23 +208,76 @@ void OnRenderedObjectInfosCalculated( SceneHierarchyInformation hierarchyInfo ) { - var instances = new List(); + // In order to support aggregate segmentation, we need to use the parent values, if requested, + // of the instances to report the instance type for color + + var instances = new Dictionary(); foreach (var objectInfo in renderedObjectInfos) { - if (!idLabelConfig.TryGetLabelEntryFromInstanceId(objectInfo.instanceId, out var labelEntry)) + if (!hierarchyInfo.hierarchy.TryGetValue(objectInfo.instanceId, out var node)) + { + Debug.LogError($"Could not find hierarchy info for instance id: {objectInfo.instanceId}"); continue; + } + + // If collecting aggregate info, use the parent id, else use the objectInfo.instanceId + var idx = aggregateChildren ? node?.parentInstanceId ?? objectInfo.instanceId : objectInfo.instanceId; + var intIdx = (int)idx; + + // Ok, this exists because we have no good way to look up label information at runtime, if a label + // is not associated with an object that is captured in objectrenderinfo pass. This is seen routinely + // in parent geometry using hierarchical labeling not having geometry of its own, but just aggregating + // labeled child objects. To support this we have to create a way to query labels from the id label config. + // This is not the most performant way to do this, things that we could do in the future to speed this up + // * rework the way we are calculating labeled data + // * cache this + // Also, this only supports a 1-1 mapping of int id to label and label to int id. Re-using labels will + // break this but right now is *probably* supported in perception. We need to revisit that and codify + // that label strings need to be unique. + var labelMap = new Dictionary(); + var labels = idLabelConfig.GetAnnotationSpecification(); + foreach (var l in labels) + { + labelMap[l.label_name] = l.label_id; + } - instances.Add(new InstanceSegmentationEntry + if (!instances.ContainsKey(intIdx)) { - instanceId = (int)objectInfo.instanceId, - labelId = labelEntry.id, - labelName = labelEntry.label, - color = objectInfo.instanceColor - }); + var registeredLabels = LabelManager.singleton.registeredLabels; + var targetNodes = registeredLabels.Where(x => x.instanceId == idx).ToList(); + + if (targetNodes.Count != 1) + { + Debug.LogWarning($"Something went wrong when trying to find the node for label {idx}, query came back with {targetNodes.Count} entries"); + continue; + } + + var targetNode = targetNodes.First(); + + if (!InstanceIdToColorMapping.TryGetColorFromInstanceId(idx, out var color)) + { + Debug.LogWarning($"Could not find the instance color for ID: {idx}"); + color = Color.black; + } + + var labelName = targetNode.labels.First(); + if (!labelMap.TryGetValue(labelName, out var labelId)) + { + Debug.LogWarning($"Could not find a labelId for the label: {labelName}"); + } + + instances[intIdx] = new InstanceSegmentationEntry + { + instanceId = intIdx, + labelId = labelId, + labelName = labelName, + color = color + }; + } } - m_PendingEntries[frame] = instances; + m_PendingEntries[frame] = instances.Values.ToList(); ReportFrameIfReady(frame); } diff --git a/com.unity.perception/Runtime/Unity.Perception.Runtime.api b/com.unity.perception/Runtime/Unity.Perception.Runtime.api index 5ed8aa0cc..e3eb6e050 100644 --- a/com.unity.perception/Runtime/Unity.Perception.Runtime.api +++ b/com.unity.perception/Runtime/Unity.Perception.Runtime.api @@ -867,6 +867,7 @@ namespace UnityEngine.Perception.GroundTruth.Labelers [UnityEngine.Scripting.APIUpdating.MovedFrom(@"UnityEngine.Perception.GroundTruth")] public sealed class InstanceSegmentationLabeler : CameraLabeler, IOverlayPanelProvider { + [Tooltip(@"Should the instance segmentation capture the single instance of the parent gameobject, or the individual sub-components")] public bool aggregateChildren = false; [Tooltip(@"The id to associate with instance segmentation annotations in the dataset.")] public string annotationId = @"instance segmentation"; public UnityEngine.Perception.GroundTruth.LabelManagement.IdLabelConfig idLabelConfig; public System.Action, RenderTexture> imageReadback; diff --git a/com.unity.perception/Tests/Runtime/GroundTruthTests/InstanceSegmentationHierarchyTests.cs b/com.unity.perception/Tests/Runtime/GroundTruthTests/InstanceSegmentationHierarchyTests.cs new file mode 100644 index 000000000..bcfbcb338 --- /dev/null +++ b/com.unity.perception/Tests/Runtime/GroundTruthTests/InstanceSegmentationHierarchyTests.cs @@ -0,0 +1,194 @@ +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using UnityEngine; +using UnityEngine.Perception.GroundTruth; +using UnityEngine.Perception.GroundTruth.Labelers; +using UnityEngine.Perception.GroundTruth.LabelManagement; +using UnityEngine.TestTools; + +namespace GroundTruthTests +{ + [TestFixture] + public class InstanceSegmentationHierarchyTests : GroundTruthTestBase + { + IEnumerator ExecuteTest(bool aggregate, List expected) + { + Screen.SetResolution(640, 480, false); + yield return null; + + var labelingConfiguration = CreateLabelingConfiguration(); + var labeler = new InstanceSegmentationLabeler(labelingConfiguration) + { + aggregateChildren = aggregate + }; + + var camera = SetupCamera(pc => + { + pc.AddLabeler(labeler); + }); + + var collector = new CollectEndpoint(); + DatasetCapture.OverrideEndpoint(collector); + DatasetCapture.ResetSimulation(); + + CreateSceneComponents(); + + yield return null; + DatasetCapture.ResetSimulation(); + + Assert.AreEqual(1, collector.currentRun.frames.Count); + var f = collector.currentRun.frames.First(); + Assert.NotNull(f); + + Assert.AreEqual(1, f.sensors.Count); + var s = f.sensors[0]; + Assert.NotNull(s); + + Assert.AreEqual(1, s.annotations.Count); + var annotation = s.annotations[0]; + Assert.NotNull(annotation); + + Assert.AreEqual("type.unity.com/unity.solo.InstanceSegmentationAnnotation", annotation.modelType); + var instanceAnnotation = (InstanceSegmentationAnnotation)annotation; + Assert.NotNull(instanceAnnotation); + + var instances = instanceAnnotation.instances.ToList(); + Assert.NotNull(instances); + Assert.AreEqual(expected.Count, instances.Count); + + foreach (var e in expected) + { + var tList = instances.Where(x => x.labelId == e.labelId); + Assert.NotNull(tList); + + Assert.AreEqual(1, tList.Count()); + var t = tList.First(); + + Assert.AreEqual(e.labelId, t.labelId); + Assert.AreEqual(e.labelName, t.labelName); + } + } + + [UnityTest] + public IEnumerator InstanceSegmentation_Hierarchy_AggregationOn() + { + var expected = new List + { + new() + { + labelId = 1, + labelName = "the_big_one" + } + }; + + return ExecuteTest(true, expected); + } + + [UnityTest] + public IEnumerator InstanceSegmentation_Hierarchy_AggregationOff() + { + var expected = new List + { + new() + { + labelId = 2, + labelName = "box1" + }, + new() + { + labelId = 3, + labelName = "box2" + }, + new() + { + labelId = 4, + labelName = "box3" + }, + new() + { + labelId = 5, + labelName = "box4" + } + }; + + return ExecuteTest(false, expected); + } + + GameObject CreateSceneComponents() + { + var go = new GameObject("Props"); + AddTestObjectForCleanup(go); + + var theBigOne = new GameObject("the_big_one"); + AddTestObjectForCleanup(theBigOne); + theBigOne.transform.position = new Vector3(0, 1.17f, 0); + theBigOne.transform.localScale = new Vector3(0.5f, 0.5f, 0.5f); + theBigOne.AddComponent().labels.Add("the_big_one"); + theBigOne.transform.parent = go.transform; + + var box1 = GameObject.CreatePrimitive(PrimitiveType.Cube); + AddTestObjectForCleanup(box1); + box1.transform.parent = theBigOne.transform; + box1.transform.localPosition = new Vector3(-0.5f, -1f, 0); + box1.AddComponent().labels.Add("box1"); + + var box2 = GameObject.CreatePrimitive(PrimitiveType.Cube); + AddTestObjectForCleanup(box2); + box2.transform.parent = theBigOne.transform; + box2.transform.localPosition = new Vector3(0.75f, -1f, 0); + box2.AddComponent().labels.Add("box2"); + + var box3 = GameObject.CreatePrimitive(PrimitiveType.Cube); + AddTestObjectForCleanup(box3); + box3.transform.parent = theBigOne.transform; + box3.transform.localPosition = new Vector3(0.75f, 0.5f, 0); + box3.AddComponent().labels.Add("box3"); + + var box4 = GameObject.CreatePrimitive(PrimitiveType.Cube); + AddTestObjectForCleanup(box4); + box4.transform.parent = theBigOne.transform; + box4.transform.localPosition = new Vector3(-0.5f, 0.5f, 0); + box4.AddComponent().labels.Add("box4"); + + return go; + } + + static IdLabelConfig CreateLabelingConfiguration() + { + var labelConfig = ScriptableObject.CreateInstance(); + + labelConfig.Init(new List + { + new() + { + id = 1, + label = "the_big_one" + }, + new() + { + id = 2, + label = "box1" + }, + new() + { + id = 3, + label = "box2" + }, + new() + { + id = 4, + label = "box3" + }, + new() + { + id = 5, + label = "box4" + } + }); + + return labelConfig; + } + } +} diff --git a/com.unity.perception/Tests/Runtime/GroundTruthTests/InstanceSegmentationHierarchyTests.cs.meta b/com.unity.perception/Tests/Runtime/GroundTruthTests/InstanceSegmentationHierarchyTests.cs.meta new file mode 100644 index 000000000..90bfc0e93 --- /dev/null +++ b/com.unity.perception/Tests/Runtime/GroundTruthTests/InstanceSegmentationHierarchyTests.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: e2e903be2e804ef693dfefd674945cf3 +timeCreated: 1673564817 \ No newline at end of file diff --git a/com.unity.perception/Tests/Runtime/GroundTruthTests/SoloFormatTests.cs b/com.unity.perception/Tests/Runtime/GroundTruthTests/SoloFormatTests.cs index 12e0596ca..798b0ddde 100644 --- a/com.unity.perception/Tests/Runtime/GroundTruthTests/SoloFormatTests.cs +++ b/com.unity.perception/Tests/Runtime/GroundTruthTests/SoloFormatTests.cs @@ -10,6 +10,7 @@ using UnityEngine.Perception.GroundTruth.DataModel; using UnityEngine.Perception.GroundTruth.Labelers; using UnityEngine.Perception.GroundTruth.LabelManagement; +using UnityEngine.Perception.GroundTruth.Utilities; using UnityEngine.TestTools; namespace GroundTruthTests @@ -748,6 +749,8 @@ public IEnumerator TestInstanceSegmentationSolo() var annDesc = "Produces an instance segmentation image for each frame. The image will render the pixels of each labeled object in a distinct color."; var instanceId = cube.GetComponent().instanceId; + var color = InstanceIdToColorMapping.GetColorFromInstanceId(instanceId); + var instances = new JArray { @@ -756,7 +759,7 @@ public IEnumerator TestInstanceSegmentationSolo() { "instanceId", instanceId }, { "labelId", 1 }, { "labelName", "test" }, - { "color", new JArray { 255, 0, 0, 255 } } + { "color", new JArray { color.r, color.g, color.b, color.a } } } };