Features/6092 improve schema dataset

geonetwork4-api/pom.xml

@@ -28,6 +28,11 @@
             <groupId>org.glassfish.jaxb</groupId>
             <artifactId>jaxb-runtime</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <optional>true</optional>
+        </dependency>
     </dependencies>
 
     <build>

indexer/src/main/java/au/org/aodn/esindexer/model/GeoNetworkField.java → geonetwork4-api/src/main/java/au/org/aodn/metadata/geonetwork/GeoNetworkField.java

@@ -1,4 +1,4 @@
-package au.org.aodn.esindexer.model;
+package au.org.aodn.metadata.geonetwork;
 
 import lombok.Getter;
 

indexer/src/main/java/au/org/aodn/esindexer/configuration/AppConstants.java

@@ -2,7 +2,7 @@
 
 public interface AppConstants {
     String PORTAL_RECORDS_MAPPING_JSON_FILE = "portal_records_index_schema.json";
-    String DATASET_INDEX_MAPPING_JSON_FILE = "dataset_index_schema.json";
+    String DATASET_INDEX_MAPPING_JSON_FILE = "data_index_schema.json";
 
     String FORMAT_XML = "xml";
     String FORMAT_ISO19115_3_2018 = "iso19115-3.2018";

indexer/src/main/java/au/org/aodn/esindexer/configuration/DatasetAccessConfig.java

@@ -1,17 +1,24 @@
 package au.org.aodn.esindexer.configuration;
 
+import au.org.aodn.esindexer.service.DataAccessService;
 import au.org.aodn.esindexer.service.DataAccessServiceImpl;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.web.client.RestTemplate;
 
 @Configuration
 public class DatasetAccessConfig {
 
-    @Bean(name = "DataAccessService")
+    @Bean
+    @ConditionalOnMissingBean(DataAccessService.class)
     public DataAccessServiceImpl createDataAccessService(
-        @Value("${dataaccess.host:defaultForTesting}") String serverUrl
-    ){
-        return new DataAccessServiceImpl(serverUrl);
+            @Value("${dataaccess.host:http://localhost:5000}") String serverUrl,
+            @Value("${dataaccess.baseUrl:/api/v1/das/}") String baseUrl,
+            @Autowired RestTemplate template){
+
+        return new DataAccessServiceImpl(serverUrl, baseUrl, template);
     }
 }

indexer/src/main/java/au/org/aodn/esindexer/controller/IndexerController.java

@@ -1,8 +1,10 @@
 package au.org.aodn.esindexer.controller;
 
+import au.org.aodn.esindexer.model.TemporalExtent;
 import au.org.aodn.esindexer.service.DataAccessService;
 import au.org.aodn.esindexer.service.GeoNetworkService;
-import au.org.aodn.esindexer.service.IndexerService;
+import au.org.aodn.esindexer.service.IndexCloudOptimizedService;
+import au.org.aodn.esindexer.service.IndexerMetadataService;
 import co.elastic.clients.elasticsearch.core.BulkResponse;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import io.swagger.v3.oas.annotations.Operation;
@@ -19,10 +21,10 @@
 import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
 
 import java.io.IOException;
+import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
 
 @RestController
 @RequestMapping(value = "/api/v1/indexer/index")
@@ -31,7 +33,10 @@
 public class IndexerController {
 
     @Autowired
-    IndexerService indexerService;
+    IndexerMetadataService indexerMetadata;
+
+    @Autowired
+    IndexCloudOptimizedService indexCloudOptimizedData;
 
     @Autowired
     GeoNetworkService geonetworkResourceService;
@@ -51,7 +56,7 @@ public ResponseEntity<String> getMetadataRecordFromGeoNetworkByUUID(@PathVariabl
     @Operation(description = "Get a document from portal index by UUID")
     public ResponseEntity<ObjectNode> getDocumentByUUID(@PathVariable("uuid") String uuid) throws IOException {
         log.info("getting a document form portal by UUID: {}", uuid);
-        ObjectNode response =  indexerService.getDocumentByUUID(uuid).source();
+        ObjectNode response =  indexerMetadata.getDocumentByUUID(uuid).source();
         return ResponseEntity.status(HttpStatus.OK).body(response);
     }
     /**
@@ -69,13 +74,12 @@ public ResponseEntity<String> indexAllMetadataRecords(
             @RequestParam(value = "confirm", defaultValue = "false") Boolean confirm,
             @RequestParam(value = "beginWithUuid", required=false) String beginWithUuid) throws IOException {
 
-        List<BulkResponse> responses = indexerService.indexAllMetadataRecordsFromGeoNetwork(beginWithUuid, confirm, null);
+        List<BulkResponse> responses = indexerMetadata.indexAllMetadataRecordsFromGeoNetwork(beginWithUuid, confirm, null);
         return ResponseEntity.ok(responses.toString());
     }
     /**
      * Emit result to FE so it will not result in gateway time-out. You need to run it with postman or whatever tools
      * support server side event, the content type needs to be text/event-stream in order to work
-     *
      * Noted: There is a bug in postman desktop, so either you run postman using web-browser with agent directly
      * or you need to have version 10.2 or above in order to get the emitted result
      *
@@ -91,7 +95,7 @@ public SseEmitter indexAllMetadataRecordsAsync(
 
         final SseEmitter emitter = new SseEmitter(0L); // 0L means no timeout;
 
-        IndexerService.Callback callback = new IndexerService.Callback() {
+        IndexerMetadataService.Callback callback = new IndexerMetadataService.Callback() {
             @Override
             public void onProgress(Object update) {
                 try {
@@ -130,7 +134,7 @@ public void onComplete(Object result) {
 
         new Thread(() -> {
             try {
-                indexerService.indexAllMetadataRecordsFromGeoNetwork(beginWithUuid, confirm, callback);
+                indexerMetadata.indexAllMetadataRecordsFromGeoNetwork(beginWithUuid, confirm, callback);
             }
             catch(IOException e) {
                 emitter.completeWithError(e);
@@ -142,36 +146,42 @@ public void onComplete(Object result) {
 
     @PostMapping(path="/{uuid}", produces = "application/json")
     @Operation(security = { @SecurityRequirement(name = "X-API-Key") }, description = "Index a metadata record by UUID")
-    public ResponseEntity<String> addDocumentByUUID(@PathVariable("uuid") String uuid) throws IOException, FactoryException, JAXBException, TransformException, ExecutionException, InterruptedException {
+    public ResponseEntity<String> addDocumentByUUID(@PathVariable("uuid") String uuid) throws IOException, FactoryException, JAXBException, TransformException {
         String metadataValues = geonetworkResourceService.searchRecordBy(uuid);
 
-        CompletableFuture<ResponseEntity<String>> f = indexerService.indexMetadata(metadataValues);
+        CompletableFuture<ResponseEntity<String>> f = indexerMetadata.indexMetadata(metadataValues);
         // Return when done make it back to sync instead of async
         return f.join();
     }
 
     @DeleteMapping(path="/{uuid}", produces = "application/json")
     @Operation(security = { @SecurityRequirement(name = "X-API-Key") }, description = "Delete a metadata record by UUID")
     public ResponseEntity<String> deleteDocumentByUUID(@PathVariable("uuid") String uuid) throws IOException {
-        return indexerService.deleteDocumentByUUID(uuid);
+        return indexerMetadata.deleteDocumentByUUID(uuid);
     }
 
     @PostMapping(path="/{uuid}/dataset", produces = "application/json")
     @Operation(security = {@SecurityRequirement(name = "X-API-Key") }, description = "Index a dataset by UUID")
-    public ResponseEntity<List<String>> indexDatasetByUUID(@PathVariable("uuid") String uuid)  {
+    public ResponseEntity<?> indexDatasetByUUID(@PathVariable("uuid") String uuid)  {
 
-        var temporalExtents = dataAccessService.getTemporalExtentOf(uuid);
-        var startDate = temporalExtents.getStartDate();
-        var endDate = temporalExtents.getEndDate();
-        log.info("Indexing dataset with UUID: {} from {} to {}", uuid, startDate, endDate);
+        List<TemporalExtent> temporalExtents = dataAccessService.getTemporalExtentOf(uuid);
+        if(!temporalExtents.isEmpty()) {
+            // Only first block works from dataservice api
+            LocalDate startDate = temporalExtents.get(0).getLocalStartDate();
+            LocalDate endDate = temporalExtents.get(0).getLocalEndDate();
+            log.info("Indexing dataset with UUID: {} from {} to {}", uuid, startDate, endDate);
 
-        var responses = indexerService.indexDataset(uuid, startDate, endDate);
+            var responses = indexCloudOptimizedData.indexCloudOptimizedData(uuid, startDate, endDate);
 
-        List<String> result = new ArrayList<>();
-        for (BulkResponse response : responses) {
-            result.add(response.toString());
-        }
+            List<String> result = new ArrayList<>();
+            for (BulkResponse response : responses) {
+                result.add(response.toString());
+            }
 
-        return ResponseEntity.ok(result);
+            return ResponseEntity.ok(result);
+        }
+        else {
+            return ResponseEntity.notFound().build();
+        }
     }
 }

indexer/src/main/java/au/org/aodn/esindexer/model/CloudOptimizedEntry.java

@@ -0,0 +1,118 @@
+package au.org.aodn.esindexer.model;
+
+import com.fasterxml.jackson.annotation.*;
+import lombok.Data;
+
+import java.math.BigDecimal;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.Temporal;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * A container that represent the data entry from cloud optimize, if you add fields, please
+ * update the hashCode() and equals()
+ */
+@Data
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class CloudOptimizedEntry {
+
+    static final BigDecimal MIN = new BigDecimal(Double.MIN_VALUE);
+
+    @JsonIgnore
+    protected DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+
+    @JsonIgnore
+    protected Temporal time;
+
+    @JsonIgnore
+    protected BigDecimal longitude;
+
+    @JsonIgnore
+    protected BigDecimal latitude;
+
+    @JsonIgnore
+    protected BigDecimal depth;
+
+    @JsonIgnore
+    public ZonedDateTime getZonedDateTime() {
+        return ((LocalDateTime)this.time).atZone(ZoneOffset.UTC);
+    }
+
+    @JsonProperty("DEPTH")
+    public void setDepth(Double v) {
+        this.depth = new BigDecimal(v);
+    }
+
+    @JsonProperty("LONGITUDE")
+    public void setLongitude(Double v) {
+        this.longitude = new BigDecimal(v);
+    }
+
+    @JsonProperty("LATITUDE")
+    public void setLatitude(Double v) {
+        this.latitude = new BigDecimal(v);
+    }
+
+    @JsonProperty("TIME")
+    public void setTime(String time) {
+        this.time = LocalDateTime.parse(time, DATE_FORMATTER);
+    }
+    /**
+     * Must use function as child class may override functions
+     * @return - The hashcode
+     */
+    @Override
+    public int hashCode() {
+        return Objects.hash(
+                this.getTime(),
+                this.getLongitude(),
+                this.getLatitude(),
+                this.getDepth()
+        );
+    }
+    /**
+     * Must use function as child class may override functions
+     * @param obj - Input
+     * @return - Compare result
+     */
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        else if (obj instanceof CloudOptimizedEntry that) {
+            int compareLong = Optional
+                    .ofNullable(that.getLongitude())
+                    .orElse(MIN)
+                    .compareTo(Optional.ofNullable(getLongitude())
+                            .orElse(MIN));
+
+            int compareLat = Optional
+                    .ofNullable(that.getLatitude())
+                    .orElse(MIN)
+                    .compareTo(Optional.ofNullable(getLatitude())
+                            .orElse(MIN));
+
+            int compareDepth = Optional
+                    .ofNullable(that.getDepth())
+                    .orElse(MIN)
+                    .compareTo(Optional.ofNullable(getDepth())
+                            .orElse(MIN));
+
+            boolean compareTime = Objects.equals(that.getTime(), getTime());
+
+            return compareLong == 0 &&
+                    compareLat == 0 &&
+                    compareDepth == 0 &&
+                    compareTime;
+        }
+        else {
+            return false;
+        }
+    }
+}

indexer/src/main/java/au/org/aodn/esindexer/model/CloudOptimizedEntryReducePrecision.java

@@ -0,0 +1,54 @@
+package au.org.aodn.esindexer.model;
+
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.time.LocalDateTime;
+import java.time.YearMonth;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+
+/**
+ * This class modified the time entry and consider the Year and Month only, and reduce precision it is used
+ * later to aggregate data based on year ond month only
+ */
+public class CloudOptimizedEntryReducePrecision extends CloudOptimizedEntry {
+
+    @Override
+    public void setTime(String time) {
+        LocalDateTime l = LocalDateTime.parse(time, DATE_FORMATTER);
+        this.time = YearMonth.of(l.getYear(), l.getMonth());
+    }
+
+    @Override
+    public void setLongitude(Double v) {
+        this.longitude = new BigDecimal(v).setScale(2, RoundingMode.HALF_UP);
+    }
+
+    @Override
+    public void setLatitude(Double v) {
+        this.latitude = new BigDecimal(v).setScale(2, RoundingMode.HALF_UP);
+    }
+    /**
+     * Round to 10th position, with two decimal place so later parse will generate .00
+     * @param v - Input
+     */
+    @Override
+    public void setDepth(Double v) {
+        this.depth = new BigDecimal(v)
+                .divide(BigDecimal.TEN, RoundingMode.HALF_UP)
+                .setScale(0, RoundingMode.HALF_UP)
+                .multiply(BigDecimal.TEN)
+                .setScale(2, RoundingMode.HALF_UP);
+    }
+    /**
+     * Return a zoned datetime in this case because we use YearMonth internally, we set the day to 1 and all time zero
+     * @return - A ZonedDateTime at day 1 time 0
+     */
+    @Override
+    public ZonedDateTime getZonedDateTime() {
+        return ((YearMonth)this.time)
+                .atDay(1)
+                .atTime(0,0,0)
+                .atZone(ZoneOffset.UTC);
+    }
+}

indexer/src/main/java/au/org/aodn/esindexer/model/DatasetEsEntry.java

@@ -5,6 +5,6 @@
 public record DatasetEsEntry(
         String uuid,
         String yearMonth,
-        List<Datum> data) {
+        List<CloudOptimizedEntry> data) {
 
 }