[SYNCOPE-1830] Supporting membership attributes query with Neo4j

client/idrepo/console/src/main/java/org/apache/syncope/client/console/panels/AnyDirectoryPanel.java

@@ -188,13 +188,13 @@ protected List<IColumn<A, String>> getColumns() {
 
         PreferenceManager.getList(DisplayAttributesModalPanel.getPrefPlainAttributeView(type)).stream().
                 map(a -> plainSchemas.stream().filter(p -> p.getKey().equals(a)).findFirst()).
-                filter(Optional::isPresent).map(Optional::get).
+                flatMap(Optional::stream).
                 forEach(s -> prefcolumns.add(new AttrColumn<>(
                 s.getKey(), s.getLabel(SyncopeConsoleSession.get().getLocale()), SchemaType.PLAIN)));
 
         PreferenceManager.getList(DisplayAttributesModalPanel.getPrefDerivedAttributeView(type)).stream().
                 map(a -> derSchemas.stream().filter(p -> p.getKey().equals(a)).findFirst()).
-                filter(Optional::isPresent).map(Optional::get).
+                flatMap(Optional::stream).
                 forEach(s -> prefcolumns.add(new AttrColumn<>(
                 s.getKey(), s.getLabel(SyncopeConsoleSession.get().getLocale()), SchemaType.DERIVED)));
 

core/idrepo/logic/src/main/java/org/apache/syncope/core/logic/UserLogic.java

@@ -398,7 +398,7 @@ public void compliance(final ComplianceQuery query) {
                     orElseThrow(() -> new NotFoundException("Realm " + query.getRealm()));
         }
         Set<ExternalResource> resources = query.getResources().stream().
-                map(resourceDAO::findById).filter(Optional::isPresent).map(Optional::get).collect(Collectors.toSet());
+                map(resourceDAO::findById).flatMap(Optional::stream).collect(Collectors.toSet());
         if (realm == null && resources.isEmpty()) {
             sce.getElements().add("Nothing to check");
             throw sce;

core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/repo/AbstractAnyRepoExt.java

@@ -228,7 +228,7 @@ public List<String> findDynRealms(final String key) {
         List<Object> result = query.getResultList();
         return result.stream().
                 map(dynRealm -> dynRealmDAO.findById(dynRealm.toString())).
-                filter(Optional::isPresent).map(Optional::get).
+                flatMap(Optional::stream).
                 map(DynRealm::getKey).
                 distinct().
                 toList();

core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/repo/AnyObjectRepoExtImpl.java

@@ -232,7 +232,7 @@ public List<Group> findDynGroups(final String key) {
         List<Object> result = query.getResultList();
         return result.stream().
                 map(groupKey -> groupDAO.findById(groupKey.toString())).
-                filter(Optional::isPresent).map(Optional::get).
+                flatMap(Optional::stream).
                 distinct().
                 collect(Collectors.toList());
     }

core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/repo/UserRepoExtImpl.java

@@ -234,7 +234,7 @@ public List<Role> findDynRoles(final String key) {
         List<Object> result = query.getResultList();
         return result.stream().
                 map(roleKey -> roleDAO.findById(roleKey.toString())).
-                filter(Optional::isPresent).map(Optional::get).
+                flatMap(Optional::stream).
                 distinct().
                 collect(Collectors.toList());
     }
@@ -250,7 +250,7 @@ public List<Group> findDynGroups(final String key) {
         List<Object> result = query.getResultList();
         return result.stream().
                 map(groupKey -> groupDAO.findById(groupKey.toString())).
-                filter(Optional::isPresent).map(Optional::get).
+                flatMap(Optional::stream).
                 distinct().
                 collect(Collectors.toList());
     }

core/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/outer/AnySearchTest.java

@@ -32,6 +32,7 @@
 import org.apache.syncope.common.lib.types.ClientExceptionType;
 import org.apache.syncope.common.lib.types.IdRepoEntitlement;
 import org.apache.syncope.core.persistence.api.attrvalue.PlainAttrValidationManager;
+import org.apache.syncope.core.persistence.api.dao.AnyObjectDAO;
 import org.apache.syncope.core.persistence.api.dao.AnySearchDAO;
 import org.apache.syncope.core.persistence.api.dao.GroupDAO;
 import org.apache.syncope.core.persistence.api.dao.PlainSchemaDAO;
@@ -44,6 +45,9 @@
 import org.apache.syncope.core.persistence.api.dao.search.RoleCond;
 import org.apache.syncope.core.persistence.api.dao.search.SearchCond;
 import org.apache.syncope.core.persistence.api.entity.Role;
+import org.apache.syncope.core.persistence.api.entity.anyobject.AMembership;
+import org.apache.syncope.core.persistence.api.entity.anyobject.APlainAttr;
+import org.apache.syncope.core.persistence.api.entity.anyobject.AnyObject;
 import org.apache.syncope.core.persistence.api.entity.group.GPlainAttr;
 import org.apache.syncope.core.persistence.api.entity.group.Group;
 import org.apache.syncope.core.persistence.api.entity.user.UMembership;
@@ -66,6 +70,9 @@ public class AnySearchTest extends AbstractTest {
     @Autowired
     private GroupDAO groupDAO;
 
+    @Autowired
+    private AnyObjectDAO anyObjectDAO;
+
     @Autowired
     private AnySearchDAO searchDAO;
 
@@ -148,6 +155,38 @@ public void searchAsGroupOwner() {
         assertEquals(rossini.getKey(), users.get(0).getKey());
     }
 
+    @Test
+    public void searchByMembershipAttribute() {
+        AttrCond attrCond = new AttrCond(AttrCond.Type.EQ);
+        attrCond.setSchema("ctype");
+        attrCond.setExpression("otherchildctype");
+        SearchCond cond = SearchCond.getLeaf(attrCond);
+
+        List<AnyObject> results = searchDAO.search(cond, AnyTypeKind.ANY_OBJECT);
+        assertTrue(results.isEmpty());
+
+        // add any object membership and its plain attribute
+        AnyObject anyObject = anyObjectDAO.findById("8559d14d-58c2-46eb-a2d4-a7d35161e8f8").orElseThrow();
+        AMembership memb = entityFactory.newEntity(AMembership.class);
+        memb.setLeftEnd(anyObject);
+        memb.setRightEnd(groupDAO.findByName("otherchild").orElseThrow());
+        anyObject.add(memb);
+        anyObject = anyObjectDAO.save(anyObject);
+
+        APlainAttr attr = entityFactory.newEntity(APlainAttr.class);
+        attr.setSchema(plainSchemaDAO.findById("ctype").orElseThrow());
+        attr.add(validator, "otherchildctype", anyUtilsFactory.getInstance(AnyTypeKind.ANY_OBJECT));
+        attr.setOwner(anyObject);
+        attr.setMembership(anyObject.getMemberships().get(0));
+        anyObject.add(attr);
+        anyObjectDAO.save(anyObject);
+
+        results = searchDAO.search(cond, AnyTypeKind.ANY_OBJECT);
+        assertEquals(1, results.size());
+
+        assertTrue(results.stream().anyMatch(a -> "8559d14d-58c2-46eb-a2d4-a7d35161e8f8".equals(a.getKey())));
+    }
+
     @Test
     public void issueSYNCOPE95() {
         groupDAO.findAll().forEach(group -> groupDAO.deleteById(group.getKey()));

core/persistence-jpa/src/test/java/org/apache/syncope/core/persistence/jpa/outer/GroupTest.java

@@ -240,7 +240,7 @@ public List<Group> findDynGroups(final User user) {
         List<Object> result = query.getResultList();
         return result.stream().
                 map(groupKey -> groupDAO.findById(groupKey.toString())).
-                filter(Optional::isPresent).map(Optional::get).
+                flatMap(Optional::stream).
                 distinct().
                 collect(Collectors.toList());
     }
@@ -332,7 +332,7 @@ public List<Group> findDynGroups(final AnyObject anyObject) {
         List<Object> result = query.getResultList();
         return result.stream().
                 map(groupKey -> groupDAO.findById(groupKey.toString())).
-                filter(Optional::isPresent).map(Optional::get).
+                flatMap(Optional::stream).
                 distinct().
                 collect(Collectors.toList());
     }

core/persistence-neo4j/src/main/java/org/apache/syncope/core/persistence/neo4j/dao/AbstractDAO.java

@@ -112,7 +112,7 @@ protected <E extends Entity, N extends AbstractNode> List<E> toList(
 
         return result.stream().
                 map(found -> findById(found.get(property).toString(), domainType, cache)).
-                filter(Optional::isPresent).map(Optional::get).map(n -> (E) n).toList();
+                flatMap(Optional::stream).map(n -> (E) n).toList();
     }
 
     protected void cascadeDelete(

core/persistence-neo4j/src/main/java/org/apache/syncope/core/persistence/neo4j/dao/Neo4jAnySearchDAO.java

@@ -100,7 +100,11 @@ protected static record AdminRealmsFilter(String filter, Set<String> dynRealmKey
 
     }
 
-    protected static record QueryInfo(TextStringBuilder query, Set<String> fields, Set<PlainSchema> plainSchemas) {
+    protected static record QueryInfo(
+            TextStringBuilder query,
+            Set<String> fields,
+            Set<PlainSchema> plainSchemas,
+            List<Pair<String, PlainSchema>> membershipAttrConds) {
 
     }
 
@@ -754,6 +758,7 @@ protected QueryInfo getQuery(final AnyTypeKind kind, final SearchCond cond, fina
         TextStringBuilder query = new TextStringBuilder();
         Set<String> involvedFields = new HashSet<>();
         Set<PlainSchema> involvedPlainSchemas = new HashSet<>();
+        List<Pair<String, PlainSchema>> membershipAttrConds = new ArrayList<>();
 
         switch (cond.getType()) {
             case LEAF, NOT_LEAF -> {
@@ -804,6 +809,13 @@ protected QueryInfo getQuery(final AnyTypeKind kind, final SearchCond cond, fina
                             Pair<String, PlainSchema> attrCondResult = getQuery(kind, leaf, not, parameters);
                             query.append(attrCondResult.getLeft());
                             involvedPlainSchemas.add(attrCondResult.getRight());
+                            if (kind != AnyTypeKind.GROUP
+                                    && !not
+                                    && leaf.getType() != AttrCond.Type.ISNULL
+                                    && leaf.getType() != AttrCond.Type.ISNOTNULL) {
+
+                                membershipAttrConds.add(attrCondResult);
+                            }
                         }));
 
                 // allow for additional search conditions
@@ -813,10 +825,12 @@ protected QueryInfo getQuery(final AnyTypeKind kind, final SearchCond cond, fina
                 QueryInfo leftAndInfo = getQuery(kind, cond.getLeft(), parameters);
                 involvedFields.addAll(leftAndInfo.fields());
                 involvedPlainSchemas.addAll(leftAndInfo.plainSchemas());
+                membershipAttrConds.addAll(leftAndInfo.membershipAttrConds());
 
                 QueryInfo rigthAndInfo = getQuery(kind, cond.getRight(), parameters);
                 involvedFields.addAll(rigthAndInfo.fields());
                 involvedPlainSchemas.addAll(rigthAndInfo.plainSchemas());
+                membershipAttrConds.addAll(rigthAndInfo.membershipAttrConds());
 
                 queryOp(query, "AND", leftAndInfo, rigthAndInfo);
             }
@@ -825,10 +839,12 @@ protected QueryInfo getQuery(final AnyTypeKind kind, final SearchCond cond, fina
                 QueryInfo leftOrInfo = getQuery(kind, cond.getLeft(), parameters);
                 involvedFields.addAll(leftOrInfo.fields());
                 involvedPlainSchemas.addAll(leftOrInfo.plainSchemas());
+                membershipAttrConds.addAll(leftOrInfo.membershipAttrConds());
 
                 QueryInfo rigthOrInfo = getQuery(kind, cond.getRight(), parameters);
                 involvedFields.addAll(rigthOrInfo.fields());
                 involvedPlainSchemas.addAll(rigthOrInfo.plainSchemas());
+                membershipAttrConds.addAll(rigthOrInfo.membershipAttrConds());
 
                 queryOp(query, "OR", leftOrInfo, rigthOrInfo);
             }
@@ -837,7 +853,7 @@ protected QueryInfo getQuery(final AnyTypeKind kind, final SearchCond cond, fina
             }
         }
 
-        return new QueryInfo(query, involvedFields, involvedPlainSchemas);
+        return new QueryInfo(query, involvedFields, involvedPlainSchemas, membershipAttrConds);
     }
 
     protected void wrapQuery(
@@ -846,8 +862,6 @@ protected void wrapQuery(
             final AnyTypeKind kind,
             final String adminRealmsFilter) {
 
-        TextStringBuilder query = queryInfo.query();
-
         TextStringBuilder match = new TextStringBuilder("MATCH (n:").append(AnyRepoExt.node(kind)).append(") ").
                 append("WITH n.id AS id");
 
@@ -864,7 +878,7 @@ protected void wrapQuery(
         Stream.concat(
                 queryInfo.plainSchemas().stream(),
                 orderBy.stream().map(clause -> plainSchemaDAO.findById(clause.getProperty())).
-                        filter(Optional::isPresent).map(Optional::get)).distinct().forEach(schema -> {
+                        flatMap(Optional::stream)).distinct().forEach(schema -> {
 
             match.append(", apoc.convert.getJsonProperty(n, 'plainAttrs.").append(schema.getKey());
             if (schema.isUniqueConstraint()) {
@@ -875,6 +889,8 @@ protected void wrapQuery(
             match.append(" AS ").append(schema.getKey());
         });
 
+        TextStringBuilder query = queryInfo.query();
+
         // take realms into account
         if (query.startsWith("MATCH (n)")) {
             query.replaceFirst("MATCH (n)", match + " WHERE (EXISTS { MATCH (n)");
@@ -886,6 +902,75 @@ protected void wrapQuery(
         query.append(") AND EXISTS { ").append(adminRealmsFilter).append(" } ");
     }
 
+    protected void membershipAttrConds(
+            final TextStringBuilder query,
+            final QueryInfo queryInfo,
+            final List<String> orderBy,
+            final AnyTypeKind kind) {
+
+        if (kind == AnyTypeKind.GROUP) {
+            return;
+        }
+        if (queryInfo.membershipAttrConds().isEmpty()) {
+            return;
+        }
+
+        Set<String> orderByItems = orderBy.stream().
+                map(clause -> StringUtils.substringBefore(clause, " ")).
+                collect(Collectors.toSet());
+
+        AnyUtils anyUtils = anyUtilsFactory.getInstance(kind);
+        Set<String> fields = Stream.concat(
+                queryInfo.fields().stream().filter(f -> !"id".equals(f)),
+                orderByItems.stream().filter(item -> !"id".equals(item) && anyUtils.getField(item).isPresent())).
+                collect(Collectors.toSet());
+
+        Set<PlainSchema> plainSchemas = Stream.concat(
+                queryInfo.membershipAttrConds().stream().map(Pair::getRight),
+                orderByItems.stream().map(item -> plainSchemaDAO.findById(item)).flatMap(Optional::stream)).
+                collect(Collectors.toSet());
+
+        // call
+        query.insert(0, "CALL () { ");
+
+        // return
+        TextStringBuilder returnStmt = new TextStringBuilder("RETURN id");
+
+        fields.forEach(f -> returnStmt.append(", ").append(f));
+
+        plainSchemas.forEach(schema -> returnStmt.append(", ").append(schema.getKey()));
+
+        query.append(returnStmt);
+
+        // union
+        query.append(" UNION ").
+                append("MATCH (n:").append(AnyRepoExt.membNode(kind)).
+                append(")-[]-(m:").append(AnyRepoExt.node(kind) + ") ").
+                append("WITH m.id AS id ");
+
+        fields.forEach(f -> query.append(", m.").append(f).append(" AS ").append(f));
+
+        plainSchemas.forEach(schema -> {
+            query.append(", apoc.convert.getJsonProperty(n, 'plainAttrs.").append(schema.getKey());
+            if (schema.isUniqueConstraint()) {
+                query.append("', '$.uniqueValue')");
+            } else {
+                query.append("', '$.values')");
+            }
+            query.append(" AS ").append(schema.getKey());
+        });
+
+        query.append(" WHERE ");
+
+        query.append(queryInfo.membershipAttrConds().stream().
+                map(mac -> "(EXISTS { " + mac.getLeft() + "} )").
+                collect(Collectors.joining(" AND ")));
+
+        query.append(" AND EXISTS { (m)-[]-(r:Realm) WHERE r.id IN $param0 } ").
+                append(returnStmt).
+                append(" } ");
+    }
+
     @Override
     protected long doCount(
             final Realm base,
@@ -906,15 +991,18 @@ protected long doCount(
         wrapQuery(queryInfo, Streamable.empty(), kind, filter.filter());
         TextStringBuilder query = queryInfo.query();
 
-        // 3. prepare the count query
+        // 3. include membership plain attr queries
+        membershipAttrConds(query, queryInfo, List.of(), kind);
+
+        // 4. prepare the count query
         query.append("RETURN COUNT(id)");
 
         return neo4jTemplate.count(query.toString(), parameters);
     }
 
-    protected String parseOrderBy(
+    protected List<String> parseOrderBy(
             final AnyTypeKind kind,
-            final Stream<Sort.Order> orderBy) {
+            final Streamable<Sort.Order> orderBy) {
 
         AnyUtils anyUtils = anyUtilsFactory.getInstance(kind);
 
@@ -946,7 +1034,7 @@ protected String parseOrderBy(
             }
         });
 
-        return clauses.stream().collect(Collectors.joining(", "));
+        return clauses;
     }
 
     @Override
@@ -970,9 +1058,15 @@ protected <T extends Any<?>> List<T> doSearch(
             wrapQuery(queryInfo, pageable.getSort(), kind, filter.filter());
             TextStringBuilder query = queryInfo.query();
 
-            // 3. prepare the search query
+            List<String> orderBy = parseOrderBy(kind, pageable.getSort());
+            String orderByStmt = orderBy.stream().collect(Collectors.joining(", "));
+
+            // 3. include membership plain attr queries
+            membershipAttrConds(query, queryInfo, orderBy, kind);
+
+            // 4. prepare the search query
             query.append("RETURN id ").
-                    append("ORDER BY ").append(parseOrderBy(kind, pageable.getSort().get()));
+                    append("ORDER BY ").append(orderByStmt);
 
             if (pageable.isPaged()) {
                 query.append(" SKIP ").append(pageable.getPageSize() * pageable.getPageNumber()).
@@ -981,7 +1075,7 @@ protected <T extends Any<?>> List<T> doSearch(
 
             LOG.debug("Query with auth and order by statements: {}, parameters: {}", query, parameters);
 
-            // 4. Prepare the result (avoiding duplicates)
+            // 5. Prepare the result (avoiding duplicates)
             return buildResult(neo4jClient.query(query.toString()).bindAll(parameters).fetch().all().stream().
                     map(found -> found.get("id")).toList(), kind);
         } catch (SyncopeClientException e) {

core/persistence-neo4j/src/main/java/org/apache/syncope/core/persistence/neo4j/dao/Neo4jAuditEventDAO.java

@@ -183,6 +183,6 @@ public List<AuditEventTO> search(
         return neo4jClient.query(query.toString()).
                 bindAll(parameters).fetch().all().stream().
                 map(found -> neo4jTemplate.findById(found.get("n.id"), Neo4jAuditEvent.class)).
-                filter(Optional::isPresent).map(Optional::get).map(this::toAuditEventTO).toList();
+                flatMap(Optional::stream).map(this::toAuditEventTO).toList();
     }
 }