Merge branch 'feature/436475_new_search_functionality' into 'develop'

feature/436475_new_search_functionality See merge request upm-inesdata/inesdata-connector!28
oeg-upm · Jul 15, 2024 · 1c14d02 · 1c14d02
2 parents ee77925 + 6c574de
commit 1c14d02
Show file tree

Hide file tree

Showing 21 changed files with 453 additions and 45 deletions.
diff --git a/...-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java b/...-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java
@@ -10,9 +10,9 @@
 import org.eclipse.edc.spi.types.TypeManager;
 import org.eclipse.edc.transaction.spi.TransactionContext;
 import org.eclipse.edc.transform.spi.TypeTransformerRegistry;
+import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry;
 import org.eclipse.edc.web.spi.WebService;
 import org.eclipse.edc.web.spi.configuration.ApiContext;
-import org.eclipse.edc.web.spi.configuration.context.ManagementApiUrl;
 import org.upm.inesdata.countelements.controller.CountElementsApiController;
 import org.upm.inesdata.countelements.service.CountElementsServiceImpl;
 import org.upm.inesdata.countelements.transformer.JsonObjectFromCountElementTransformer;
@@ -49,6 +49,9 @@ public class CountElementsApiExtension implements ServiceExtension {
     @Inject
     private TransactionContext transactionContext;
 
+    @Inject
+    private JsonObjectValidatorRegistry validator;
+
     @Override
     public String name() {
         return NAME;
@@ -72,7 +75,7 @@ public void initialize(ServiceExtensionContext context) {
         var managementApiTransformerRegistry = transformerRegistry.forContext("management-api");
         managementApiTransformerRegistry.register(new JsonObjectFromCountElementTransformer(factory, jsonLdMapper));
 
-        var countElementsApiController = new CountElementsApiController(countElementsService(), managementApiTransformerRegistry);
+        var countElementsApiController = new CountElementsApiController(countElementsService(), managementApiTransformerRegistry, validator);
         webService.registerResource(ApiContext.MANAGEMENT, countElementsApiController);
     }
 }
diff --git a/...lements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApi.java b/...lements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApi.java
@@ -8,6 +8,7 @@
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.tags.Tag;
+import jakarta.json.JsonObject;
 import org.eclipse.edc.api.model.ApiCoreSchema;
 
 @OpenAPIDefinition(
@@ -32,5 +33,5 @@ public interface CountElementsApi {
                             content = @Content(array = @ArraySchema(schema = @Schema(implementation = ApiCoreSchema.ApiErrorDetailSchema.class))))
             }
     )
-    long countElements(String entityType);
+    long countElements(String entityType, JsonObject querySpecJson);
 }
diff --git a/...i/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApiController.java b/...i/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApiController.java
@@ -1,41 +1,62 @@
 package org.upm.inesdata.countelements.controller;
 
+import jakarta.json.JsonObject;
 import jakarta.ws.rs.BadRequestException;
-import jakarta.ws.rs.GET;
+import jakarta.ws.rs.Consumes;
+import jakarta.ws.rs.POST;
 import jakarta.ws.rs.Path;
 import jakarta.ws.rs.Produces;
 import jakarta.ws.rs.QueryParam;
 import jakarta.ws.rs.core.MediaType;
+import org.eclipse.edc.spi.query.QuerySpec;
 import org.eclipse.edc.transform.spi.TypeTransformerRegistry;
+import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry;
+import org.eclipse.edc.web.spi.exception.InvalidRequestException;
+import org.eclipse.edc.web.spi.exception.ValidationFailureException;
 import org.upm.inesdata.spi.countelements.service.CountElementsService;
 
 import java.util.Objects;
 
+import static org.eclipse.edc.spi.query.QuerySpec.EDC_QUERY_SPEC_TYPE;
+
 @Produces({MediaType.APPLICATION_JSON})
+@Consumes({ MediaType.APPLICATION_JSON })
 @Path("/pagination")
 public class CountElementsApiController implements CountElementsApi {
 
     private final CountElementsService service;
+    private final JsonObjectValidatorRegistry validator;
     private final TypeTransformerRegistry transformerRegistry;
 
-    public CountElementsApiController(CountElementsService service, TypeTransformerRegistry transformerRegistry) {
+    public CountElementsApiController(CountElementsService service, TypeTransformerRegistry transformerRegistry, JsonObjectValidatorRegistry validator) {
         this.service = service;
         this.transformerRegistry = transformerRegistry;
+        this.validator = validator;
     }
 
-    @GET
+    @POST
     @Path("/count")
     @Override
-    public long countElements(@QueryParam("type") String entityType) {
+    public long countElements(@QueryParam("type") String entityType, JsonObject querySpecJson) {
         if (!Objects.equals(entityType, "asset") && !Objects.equals(entityType, "policyDefinition")
                 && !Objects.equals(entityType, "contractDefinition")
                 && !Objects.equals(entityType, "contractAgreement")
                 && !Objects.equals(entityType, "transferProcess")
                 && !Objects.equals(entityType, "federatedCatalog")) {
-            throw new BadRequestException("Entity type provided is not valid");
+            throw new InvalidRequestException("Entity type provided is not valid: %s".formatted(entityType));
+        }
+
+        QuerySpec querySpec;
+        if (querySpecJson == null) {
+            querySpec = QuerySpec.Builder.newInstance().build();
+        } else {
+            validator.validate(EDC_QUERY_SPEC_TYPE, querySpecJson).orElseThrow(ValidationFailureException::new);
+
+            querySpec = transformerRegistry.transform(querySpecJson, QuerySpec.class)
+                    .orElseThrow(InvalidRequestException::new);
         }
 
-        var count = service.countElements(entityType);
+        var count = service.countElements(entityType, querySpec);
 
 //        JsonObject result =  transformerRegistry.transform(count, JsonObject.class)
 //                .orElseThrow(f -> new EdcException(f.getFailureDetail()));

diff --git a/...ts-api/src/main/java/org/upm/inesdata/countelements/service/CountElementsServiceImpl.java b/...ts-api/src/main/java/org/upm/inesdata/countelements/service/CountElementsServiceImpl.java
@@ -1,5 +1,6 @@
 package org.upm.inesdata.countelements.service;
 
+import org.eclipse.edc.spi.query.QuerySpec;
 import org.eclipse.edc.transaction.spi.TransactionContext;
 import org.upm.inesdata.spi.countelements.domain.CountElement;
 import org.upm.inesdata.spi.countelements.index.CountElementsIndex;
@@ -15,7 +16,7 @@ public CountElementsServiceImpl(CountElementsIndex countElementsIndex, Transacti
     }
 
     @Override
-    public CountElement countElements(String entityType) {
-        return transactionContext.execute(() -> countElementsIndex.countElements(entityType));
+    public CountElement countElements(String entityType, QuerySpec querySpec) {
+        return transactionContext.execute(() -> countElementsIndex.countElements(entityType, querySpec));
     }
 }
diff --git a/extensions/count-elements-sql/build.gradle.kts b/extensions/count-elements-sql/build.gradle.kts
@@ -6,6 +6,7 @@ plugins {
 dependencies {
     api(project(":spi:count-elements-spi"))
     implementation(project(":extensions:count-elements-api"))
+    api(project(":extensions:inesdata-search-extension"))
     api(libs.edc.spi.core)
     api(libs.edc.transaction.spi)
     implementation(libs.edc.transaction.spi)

diff --git a/...nts-sql/src/main/java/org/upm/inesdata/countelements/sql/index/SqlCountElementsIndex.java b/...nts-sql/src/main/java/org/upm/inesdata/countelements/sql/index/SqlCountElementsIndex.java
@@ -2,8 +2,10 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.eclipse.edc.spi.persistence.EdcPersistenceException;
+import org.eclipse.edc.spi.query.QuerySpec;
 import org.eclipse.edc.sql.QueryExecutor;
 import org.eclipse.edc.sql.store.AbstractSqlStore;
+import org.eclipse.edc.sql.translation.SqlQueryStatement;
 import org.eclipse.edc.transaction.datasource.spi.DataSourceRegistry;
 import org.eclipse.edc.transaction.spi.TransactionContext;
 import org.upm.inesdata.countelements.sql.index.schema.CountElementsStatements;
@@ -21,20 +23,28 @@ public class SqlCountElementsIndex extends AbstractSqlStore implements CountElem
     private final CountElementsStatements countElementsStatements;
 
     public SqlCountElementsIndex(DataSourceRegistry dataSourceRegistry,
-                              String dataSourceName, 
-                              TransactionContext transactionContext,
-                              ObjectMapper objectMapper, 
-                              CountElementsStatements countElementsStatements,
-                              QueryExecutor queryExecutor) {
+                                 String dataSourceName,
+                                 TransactionContext transactionContext,
+                                 ObjectMapper objectMapper,
+                                 CountElementsStatements countElementsStatements,
+                                 QueryExecutor queryExecutor) {
         super(dataSourceRegistry, dataSourceName, transactionContext, objectMapper, queryExecutor);
         this.countElementsStatements = Objects.requireNonNull(countElementsStatements);
     }
 
     @Override
-    public CountElement countElements(String entityType) {
+    public CountElement countElements(String entityType, QuerySpec querySpec) {
         try (var connection = getConnection()) {
-            var sql = countElementsStatements.getCount(entityType);
-            long count =  queryExecutor.single(connection, true, r -> r.getLong(1), sql);
+            long count;
+            if ("federatedCatalog".equals(entityType)) {
+                SqlQueryStatement dataSetQueryStatement = countElementsStatements.createCountDatasetQuery(entityType, querySpec);
+                count = queryExecutor.single(connection, true, r -> r.getLong(1),
+                        dataSetQueryStatement.getQueryAsString(), dataSetQueryStatement.getParameters());
+            } else {
+                var sql = countElementsStatements.getCount(entityType);
+                count = queryExecutor.single(connection, true, r -> r.getLong(1), sql);
+            }
+
             return CountElement.Builder.newInstance().count(count).build();
         } catch (SQLException e) {
             throw new EdcPersistenceException(e);

diff --git a/...c/main/java/org/upm/inesdata/countelements/sql/index/schema/BaseSqlDialectStatements.java b/...c/main/java/org/upm/inesdata/countelements/sql/index/schema/BaseSqlDialectStatements.java
@@ -1,6 +1,9 @@
 package org.upm.inesdata.countelements.sql.index.schema;
 
+import org.eclipse.edc.spi.query.QuerySpec;
 import org.eclipse.edc.sql.translation.SqlOperatorTranslator;
+import org.upm.inesdata.countelements.sql.index.schema.postgres.SqlDatasetMapping;
+import org.upm.inesdata.search.extension.InesdataSqlQueryStatement;
 
 import static java.lang.String.format;
 
@@ -15,31 +18,33 @@ public BaseSqlDialectStatements(SqlOperatorTranslator operatorTranslator) {
         this.operatorTranslator = operatorTranslator;
     }
 
+    /**
+     * {@inheritDoc}
+     *
+     * @see CountElementsStatements#getCount(String)
+     */
     @Override
     public String getCount(String entityType) {
-        String tableName = null;
-        switch (entityType) {
-            case "asset":
-                tableName = getAssetTable();
-                break;
-            case "policyDefinition":
-                tableName = getPolicyDefinitionTable();
-                break;
-            case "contractDefinition":
-                tableName = getContractDefinitionTable();
-                break;
-            case "contractAgreement":
-                tableName = getContractAgreementTable();
-                break;
-            case "transferProcess":
-                tableName = getTransferProcessTable();
-                break;
-            case "federatedCatalog":
-                tableName = getDatasetTable();
-                break;
-        }
+        String tableName = switch (entityType) {
+            case "asset" -> getAssetTable();
+            case "policyDefinition" -> getPolicyDefinitionTable();
+            case "contractDefinition" -> getContractDefinitionTable();
+            case "contractAgreement" -> getContractAgreementTable();
+            case "transferProcess" -> getTransferProcessTable();
+            case "federatedCatalog" -> getDatasetTable();
+            default -> null;
+        };
         return format("SELECT COUNT(*) FROM %s",
                 tableName);
     }
 
+    /**
+     * {@inheritDoc}
+     *
+     * @see CountElementsStatements#createDatasetQuery(QuerySpec)
+     */
+    @Override
+    public InesdataSqlQueryStatement createCountDatasetQuery(String entityType, QuerySpec querySpec) {
+        return new InesdataSqlQueryStatement(getCount(entityType), querySpec, new SqlDatasetMapping(this), operatorTranslator);
+    }
 }
diff --git a/...rc/main/java/org/upm/inesdata/countelements/sql/index/schema/CountElementsStatements.java b/...rc/main/java/org/upm/inesdata/countelements/sql/index/schema/CountElementsStatements.java
@@ -1,7 +1,9 @@
 package org.upm.inesdata.countelements.sql.index.schema;
 
 import org.eclipse.edc.runtime.metamodel.annotation.ExtensionPoint;
+import org.eclipse.edc.spi.query.QuerySpec;
 import org.eclipse.edc.sql.statement.SqlStatements;
+import org.upm.inesdata.search.extension.InesdataSqlQueryStatement;
 
 /**
  * Defines queries used by the SqlCountElementsIndexServiceExtension.
@@ -51,6 +53,51 @@ default String getDatasetTable() {
         return "edc_dataset";
     }
 
+    /**
+     * Retrieves the name of the column storing dataset IDs.
+     *
+     * @return the name of the dataset ID column.
+     */
+    default String getDatasetIdColumn() {
+        return "id";
+    }
+
+    /**
+     * Retrieves the name of the column storing offers associated with datasets.
+     *
+     * @return the name of the offers column.
+     */
+    default String getDatasetOffersColumn() {
+        return "offers";
+    }
+
+    /**
+     * Retrieves the name of the column storing properties of datasets as JSON.
+     *
+     * @return the name of the properties column.
+     */
+    default String getDatasetPropertiesColumn() {
+        return "properties";
+    }
+
+    /**
+     * Retrieves the name of the column storing the catalog ID associated with datasets.
+     *
+     * @return the name of the catalog ID column.
+     */
+    default String getDatasetCatalogIdColumn() {
+        return "catalog_id";
+    }
+
+    /**
+     * Creates an SQL query statement specifically for datasets based on the provided query specification.
+     *
+     * @param querySpec the query specification defining filters, sorting, and pagination for datasets.
+     * @param entityType the entity type (federatedCatalog)
+     * @return an SQL query statement for datasets.
+     */
+    InesdataSqlQueryStatement createCountDatasetQuery(String entityType, QuerySpec querySpec);
+
     /**
      * SELECT COUNT clause.
      */

diff --git a/...main/java/org/upm/inesdata/countelements/sql/index/schema/postgres/SqlDatasetMapping.java b/...main/java/org/upm/inesdata/countelements/sql/index/schema/postgres/SqlDatasetMapping.java
@@ -0,0 +1,25 @@
+package org.upm.inesdata.countelements.sql.index.schema.postgres;
+
+import org.eclipse.edc.sql.translation.JsonFieldTranslator;
+import org.eclipse.edc.sql.translation.TranslationMapping;
+import org.upm.inesdata.countelements.sql.index.schema.CountElementsStatements;
+
+/**
+ * Maps fields of a dataset of federated catalog  onto the
+ * corresponding SQL schema (= column names) enabling access through Postgres JSON operators where applicable
+ */
+public class SqlDatasetMapping extends TranslationMapping {
+    /**
+     * Constructs a mapping for SQL dataset columns using the provided SQL federated catalog statements.
+     * This mapping specifies how dataset fields correspond to database columns.
+     *
+     * @param statements the SQL statements specific to the federated catalog schema.
+     */
+    public SqlDatasetMapping(CountElementsStatements statements) {
+        add("id", statements.getDatasetIdColumn());
+        add("offers", new JsonFieldTranslator(statements.getDatasetOffersColumn()));
+        add("properties", new JsonFieldTranslator(statements.getDatasetPropertiesColumn()));
+        add("catalog_id", statements.getDatasetCatalogIdColumn());
+    }
+
+}
diff --git a/extensions/federated-catalog-cache-sql/build.gradle.kts b/extensions/federated-catalog-cache-sql/build.gradle.kts
@@ -8,6 +8,7 @@ dependencies {
     implementation(libs.edc.federated.catalog.api)
     api(libs.edc.spi.core)
     api(libs.edc.transaction.spi)
+    api(project(":extensions:inesdata-search-extension"))
     implementation(libs.edc.transaction.spi)
     implementation(libs.edc.transaction.datasource.spi)
     implementation(libs.edc.sql.core)

diff --git a/...l/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java b/...l/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java
@@ -5,6 +5,7 @@
 import org.eclipse.edc.sql.translation.SqlQueryStatement;
 import org.upm.inesdata.federated.sql.index.schema.postgres.SqlDatasetMapping;
 import org.upm.inesdata.federated.sql.index.schema.postgres.SqlFederatedCatalogMapping;
+import org.upm.inesdata.search.extension.InesdataSqlQueryStatement;
 
 import static java.lang.String.format;
 
@@ -133,8 +134,8 @@ public SqlQueryStatement createQuery(QuerySpec querySpec) {
      * @see SqlFederatedCatalogStatements#createDatasetQuery(QuerySpec)
      */
     @Override
-    public SqlQueryStatement createDatasetQuery(QuerySpec querySpec) {
-        return new SqlQueryStatement(getSelectDatasetTemplate(), querySpec, new SqlDatasetMapping(this), operatorTranslator);
+    public InesdataSqlQueryStatement createDatasetQuery(QuerySpec querySpec) {
+        return new InesdataSqlQueryStatement(getSelectDatasetTemplate(), querySpec, new SqlDatasetMapping(this), operatorTranslator);
     }
 
     /**

diff --git a/.../main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java b/.../main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java
@@ -4,6 +4,7 @@
 import org.eclipse.edc.spi.query.QuerySpec;
 import org.eclipse.edc.sql.statement.SqlStatements;
 import org.eclipse.edc.sql.translation.SqlQueryStatement;
+import org.upm.inesdata.search.extension.InesdataSqlQueryStatement;
 
 /**
  * SQL statements interface for managing federated catalog data. Extends {@link SqlStatements} and provides methods for
@@ -250,7 +251,7 @@ default String getDistributionDatasetIdColumn() {
    * @param querySpec the query specification defining filters, sorting, and pagination for datasets.
    * @return an SQL query statement for datasets.
    */
-  SqlQueryStatement createDatasetQuery(QuerySpec querySpec);
+  InesdataSqlQueryStatement createDatasetQuery(QuerySpec querySpec);
 
   /**
    * Retrieves the SQL template for deleting expired catalogs.