Merge pull request #91 from aodn/feature/query-real-data

query real data
aodn · Oct 29, 2024 · 0b4cf55 · 0b4cf55
2 parents 251146a + 13a7d09
commit 0b4cf55
Show file tree

Hide file tree

Showing 9 changed files with 197 additions and 11 deletions.
diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatasetModel.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatasetModel.java
@@ -0,0 +1,12 @@
+package au.org.aodn.ogcapi.server.core.model;
+
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.List;
+@Data
+@Builder
+public class DatasetModel {
+    private String uuid;
+    private List<DatumModel> data;
+}
diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatasetSearchResult.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatasetSearchResult.java
@@ -0,0 +1,57 @@
+package au.org.aodn.ogcapi.server.core.model;
+
+import au.org.aodn.ogcapi.features.model.*;
+import au.org.aodn.ogcapi.server.core.model.enumeration.FeatureProperty;
+import lombok.Getter;
+
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+@Getter
+public class DatasetSearchResult {
+
+    private final FeatureCollectionGeoJSON dataset;
+
+    public DatasetSearchResult() {
+        this.dataset = new FeatureCollectionGeoJSON();
+        initDataset();
+    }
+
+    private void initDataset() {
+        dataset.setType(FeatureCollectionGeoJSON.TypeEnum.FEATURECOLLECTION);
+        dataset.setFeatures(new ArrayList<>());
+    }
+
+    public void addDatum(DatumModel datum) {
+
+        if (datum == null) {
+            throw new IllegalArgumentException("Datum cannot be null");
+        }
+
+        var feature = new FeatureGeoJSON();
+        feature.setType(FeatureGeoJSON.TypeEnum.FEATURE);
+        var geometry = new PointGeoJSON();
+        geometry.setType(PointGeoJSON.TypeEnum.POINT);
+        var coordinates = new ArrayList<BigDecimal>();
+
+        // Don't use null checks here because it is a list and even if it is null,
+        // it still needs "null" to occupy the space
+        coordinates.add(datum.getLongitude());
+        coordinates.add(datum.getLatitude());
+
+        geometry.setCoordinates(coordinates);
+        feature.setGeometry(geometry);
+
+        // Please add more properties if needed
+        Map<String, Object> properties = new HashMap<>();
+        properties.put(FeatureProperty.TIME.getValue(), datum.getTime());
+        properties.put(FeatureProperty.DEPTH.getValue(), datum.getDepth());
+        properties.put(FeatureProperty.COUNT.getValue(), datum.getCount());
+
+        feature.setProperties(properties);
+        dataset.getFeatures().add(feature);
+    }
+
+}
diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatumModel.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/DatumModel.java
@@ -0,0 +1,18 @@
+package au.org.aodn.ogcapi.server.core.model;
+
+import lombok.Builder;
+import lombok.Data;
+
+import java.math.BigDecimal;
+
+@Data
+@Builder
+public class DatumModel {
+
+    private String time;
+    private BigDecimal latitude;
+    private BigDecimal longitude;
+    private BigDecimal depth;
+
+    private long count;
+}
diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureProperty.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureProperty.java
@@ -0,0 +1,19 @@
+package au.org.aodn.ogcapi.server.core.model.enumeration;
+
+import lombok.Getter;
+
+@Getter
+public enum FeatureProperty {
+    TIME("time"),
+    DEPTH("depth"),
+    COUNT("count"),
+    UUID("uuid")
+    ;
+
+    private final String value;
+
+    FeatureProperty(String value) {
+        this.value = value;
+    }
+
+}
diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java
@@ -1,5 +1,7 @@
 package au.org.aodn.ogcapi.server.core.service;
 
+import au.org.aodn.ogcapi.server.core.model.DatasetModel;
+import au.org.aodn.ogcapi.server.core.model.DatasetSearchResult;
 import au.org.aodn.ogcapi.server.core.model.dto.SearchSuggestionsDto;
 import au.org.aodn.ogcapi.server.core.model.enumeration.*;
 import au.org.aodn.ogcapi.server.core.parser.CQLToElasticFilterFactory;
@@ -8,15 +10,16 @@
 import co.elastic.clients.elasticsearch._types.FieldValue;
 import co.elastic.clients.elasticsearch._types.SortOptions;
 import co.elastic.clients.elasticsearch._types.SortOrder;
+import co.elastic.clients.elasticsearch._types.query_dsl.*;
+import co.elastic.clients.elasticsearch.core.SearchMvtRequest;
 import co.elastic.clients.elasticsearch.core.SearchRequest;
 import co.elastic.clients.elasticsearch.core.SearchResponse;
 import co.elastic.clients.elasticsearch.core.search.Hit;
-import co.elastic.clients.elasticsearch._types.query_dsl.*;
-import co.elastic.clients.elasticsearch.core.SearchMvtRequest;
 import co.elastic.clients.elasticsearch.core.search_mvt.GridType;
 import co.elastic.clients.transport.endpoints.BinaryResponse;
 import co.elastic.clients.util.ObjectBuilder;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import lombok.extern.slf4j.Slf4j;
 import org.geotools.filter.text.commons.CompilerUtil;
 import org.geotools.filter.text.commons.Language;
@@ -28,6 +31,7 @@
 
 import java.io.IOException;
 import java.util.*;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -48,9 +52,14 @@ public class ElasticSearch extends ElasticSearchBase implements Search {
     @Value("${elasticsearch.vocabs_index.name}")
     protected String vocabsIndexName;
 
+    @Value("${elasticsearch.dataset_index.name}")
+    protected String datasetIndexName;
+
     @Value("${elasticsearch.search_after.split_regex:\\|\\|}")
     protected String searchAfterSplitRegex;
 
+    private final int DATASET_ENTRY_MAX = 2000;
+
     public ElasticSearch(ElasticsearchClient client,
                          ObjectMapper mapper,
                          String indexName,
@@ -420,4 +429,42 @@ protected static FieldValue toFieldValue(String s) {
         // Assume it is string
         return FieldValue.of(s.trim());
     }
+
+    @Override
+    public DatasetSearchResult searchDataset(
+            String collectionId,
+            String startDate,
+            String endDate
+    ) {
+        List<Query> queries = new ArrayList<>();
+        queries.add(MatchQuery.of(query -> query
+                .field("uuid")
+                .query(collectionId))._toQuery());
+
+        Supplier<SearchRequest.Builder> builderSupplier = () -> {
+            SearchRequest.Builder builder = new SearchRequest.Builder();
+            builder.index(datasetIndexName)
+                    .size(DATASET_ENTRY_MAX)
+                    .query(query -> query.bool(createBoolQueryForProperties(queries, null, null)));
+
+            return builder;
+        };
+
+        try {
+            Iterable<Hit<ObjectNode>> response = pagableSearch(builderSupplier, ObjectNode.class, (long) DATASET_ENTRY_MAX);
+
+            DatasetSearchResult result = new DatasetSearchResult();
+
+            for (var node : response) {
+                if (node != null && node.source() != null) {
+                    var monthlyData = mapper.readValue(node.source().toPrettyString(), DatasetModel.class);
+                    monthlyData.getData().forEach(result::addDatum);
+                }
+            }
+            return result;
+        } catch (Exception e) {
+            log.error("Error while searching dataset.", e);
+        }
+        return null;
+    }
 }
diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/Search.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/Search.java
@@ -1,8 +1,7 @@
 package au.org.aodn.ogcapi.server.core.service;
 
-import au.org.aodn.ogcapi.server.core.model.StacCollectionModel;
+import au.org.aodn.ogcapi.server.core.model.DatasetSearchResult;
 import au.org.aodn.ogcapi.server.core.model.enumeration.CQLCrsType;
-import co.elastic.clients.elasticsearch._types.SortOptions;
 import co.elastic.clients.transport.endpoints.BinaryResponse;
 import org.springframework.http.ResponseEntity;
 
@@ -16,6 +15,7 @@ public interface Search {
 
     ElasticSearchBase.SearchResult searchCollections(List<String> ids, String sortBy);
     ElasticSearchBase.SearchResult searchAllCollections(String sortBy) throws Exception;
+    DatasetSearchResult searchDataset(String collectionId, String startDate, String endDate) throws Exception;
 
     ElasticSearchBase.SearchResult searchByParameters(
             List<String> targets,

diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java
@@ -9,8 +9,7 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.bind.annotation.*;
 
 import java.math.BigDecimal;
 import java.util.List;
@@ -29,9 +28,29 @@ public ResponseEntity<Collection> describeCollection(String collectionId) {
 
     @Override
     public ResponseEntity<FeatureGeoJSON> getFeature(String collectionId, String featureId) {
-        return null;
+        return ResponseEntity.status(HttpStatus.NOT_IMPLEMENTED).build();
+    }
+
+    @RequestMapping(
+            value = {"/collections/{collectionId}/items"},
+            produces = {"application/geo+json", "text/html", "application/json"},
+            method = {RequestMethod.GET}
+    )
+    public ResponseEntity<FeatureCollectionGeoJSON> getFeatures(
+
+            @PathVariable("collectionId") String collectionId,
+            @RequestParam(value = "start_datetime", required = false) String startDate,
+            @RequestParam(value = "end_datetime", required = false) String endDate
+    ) {
+        return  featuresService.getDataset(collectionId, startDate, endDate);
     }
 
+
+
+    /**
+     * Hidden because we want to have a more functional implementation
+     */
+    @Hidden
     @Override
     public ResponseEntity<FeatureCollectionGeoJSON> getFeatures(String collectionId, Integer limit, List<BigDecimal> bbox, String datetime) {
         return null;

diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java
@@ -1,20 +1,17 @@
 package au.org.aodn.ogcapi.server.features;
 
 import au.org.aodn.ogcapi.features.model.Collection;
+import au.org.aodn.ogcapi.features.model.FeatureCollectionGeoJSON;
 import au.org.aodn.ogcapi.server.core.mapper.StacToCollection;
-import au.org.aodn.ogcapi.server.core.model.ErrorResponse;
-import au.org.aodn.ogcapi.server.core.model.StacCollectionModel;
 import au.org.aodn.ogcapi.server.core.service.ElasticSearch;
 import au.org.aodn.ogcapi.server.core.service.OGCApiService;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Service;
 
 import java.util.List;
 import java.util.NoSuchElementException;
-import java.util.Optional;
 
 @Service("FeaturesRestService")
 @Slf4j
@@ -43,4 +40,19 @@ public ResponseEntity<Collection> getCollection(String id, String sortBy) throws
             return ResponseEntity.notFound().build();
         }
     }
+
+    public ResponseEntity<FeatureCollectionGeoJSON> getDataset(
+            String collectionId,
+            String startDate,
+            String endDate
+    ) {
+        try {
+            var result = search.searchDataset(collectionId, startDate, endDate);
+            return ResponseEntity.ok()
+                    .body(result.getDataset());
+        } catch (Exception e) {
+            log.error("Error while getting dataset", e);
+            return ResponseEntity.internalServerError().build();
+        }
+    }
 }
diff --git a/server/src/main/resources/application.yaml b/server/src/main/resources/application.yaml
@@ -10,6 +10,8 @@ elasticsearch:
     name: dev_portal_records
   vocabs_index:
     name: vocabs_index
+  dataset_index:
+    name: dataset_index
   serverUrl: http://localhost:9200
   apiKey: <sample-api-key>
   search_as_you_type: