Merge pull request #173 from aodn/features/6092-improve-request-speed

Features/6092 improve request speed
aodn · Jan 13, 2025 · 49c6b77 · 49c6b77
2 parents 4b4d6f3 + 27f5810
commit 49c6b77
Show file tree

Hide file tree

Showing 47 changed files with 169,285 additions and 510 deletions.
diff --git a/geonetwork4-api/pom.xml b/geonetwork4-api/pom.xml
@@ -28,6 +28,11 @@
             <groupId>org.glassfish.jaxb</groupId>
             <artifactId>jaxb-runtime</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <optional>true</optional>
+        </dependency>
     </dependencies>
 
     <build>

diff --git a/...aodn/esindexer/model/GeoNetworkField.java → .../metadata/geonetwork/GeoNetworkField.java b/...aodn/esindexer/model/GeoNetworkField.java → .../metadata/geonetwork/GeoNetworkField.java
@@ -1,4 +1,4 @@
-package au.org.aodn.esindexer.model;
+package au.org.aodn.metadata.geonetwork;
 
 import lombok.Getter;
 

diff --git a/indexer/src/main/java/au/org/aodn/esindexer/configuration/AppConstants.java b/indexer/src/main/java/au/org/aodn/esindexer/configuration/AppConstants.java
@@ -2,7 +2,7 @@
 
 public interface AppConstants {
     String PORTAL_RECORDS_MAPPING_JSON_FILE = "portal_records_index_schema.json";
-    String DATASET_INDEX_MAPPING_JSON_FILE = "dataset_index_schema.json";
+    String DATASET_INDEX_MAPPING_JSON_FILE = "data_index_schema.json";
 
     String FORMAT_XML = "xml";
     String FORMAT_ISO19115_3_2018 = "iso19115-3.2018";

diff --git a/indexer/src/main/java/au/org/aodn/esindexer/configuration/DatasetAccessConfig.java b/indexer/src/main/java/au/org/aodn/esindexer/configuration/DatasetAccessConfig.java
@@ -1,17 +1,24 @@
 package au.org.aodn.esindexer.configuration;
 
+import au.org.aodn.esindexer.service.DataAccessService;
 import au.org.aodn.esindexer.service.DataAccessServiceImpl;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.web.client.RestTemplate;
 
 @Configuration
 public class DatasetAccessConfig {
 
-    @Bean(name = "DataAccessService")
+    @Bean
+    @ConditionalOnMissingBean(DataAccessService.class)
     public DataAccessServiceImpl createDataAccessService(
-        @Value("${dataaccess.host:defaultForTesting}") String serverUrl
-    ){
-        return new DataAccessServiceImpl(serverUrl);
+            @Value("${dataaccess.host:http://localhost:5000}") String serverUrl,
+            @Value("${dataaccess.baseUrl:/api/v1/das/}") String baseUrl,
+            @Autowired RestTemplate template){
+
+        return new DataAccessServiceImpl(serverUrl, baseUrl, template);
     }
 }
diff --git a/indexer/src/main/java/au/org/aodn/esindexer/configuration/ObjectMapperConfig.java b/indexer/src/main/java/au/org/aodn/esindexer/configuration/ObjectMapperConfig.java
@@ -1,24 +1,21 @@
 package au.org.aodn.esindexer.configuration;
 
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.PropertyNamingStrategy;
+import com.fasterxml.jackson.databind.*;
+import com.fasterxml.jackson.databind.json.JsonMapper;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
 @Configuration
 public class ObjectMapperConfig {
     @Bean("indexerObjectMapper")
     public static ObjectMapper objectMapper() {
-        ObjectMapper objectMapper = new ObjectMapper();
-
-        // Enable pretty printing for JSON output
-        objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
-
-        // Ignore unknown properties during deserialization
-        objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+        ObjectMapper objectMapper = JsonMapper.builder()
+                .enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES)
+                // Enable pretty printing for JSON output
+                .enable(SerializationFeature.INDENT_OUTPUT)
+                // Ignore unknown properties during deserialization
+                .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
+                .build();
 
         // Use a specific date format for serialization and deserialization (if needed)
         // objectMapper.setDateFormat(new SimpleDateFormat("yyyy-MM-dd"));

diff --git a/indexer/src/main/java/au/org/aodn/esindexer/controller/IndexerController.java b/indexer/src/main/java/au/org/aodn/esindexer/controller/IndexerController.java
@@ -1,8 +1,7 @@
 package au.org.aodn.esindexer.controller;
 
-import au.org.aodn.esindexer.service.DataAccessService;
-import au.org.aodn.esindexer.service.GeoNetworkService;
-import au.org.aodn.esindexer.service.IndexerService;
+import au.org.aodn.esindexer.model.TemporalExtent;
+import au.org.aodn.esindexer.service.*;
 import co.elastic.clients.elasticsearch.core.BulkResponse;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import io.swagger.v3.oas.annotations.Operation;
@@ -19,10 +18,9 @@
 import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
 
 import java.io.IOException;
-import java.util.ArrayList;
+import java.time.LocalDate;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
 
 @RestController
 @RequestMapping(value = "/api/v1/indexer/index")
@@ -31,7 +29,10 @@
 public class IndexerController {
 
     @Autowired
-    IndexerService indexerService;
+    IndexerMetadataService indexerMetadata;
+
+    @Autowired
+    IndexCloudOptimizedService indexCloudOptimizedData;
 
     @Autowired
     GeoNetworkService geonetworkResourceService;
@@ -51,7 +52,7 @@ public ResponseEntity<String> getMetadataRecordFromGeoNetworkByUUID(@PathVariabl
     @Operation(description = "Get a document from portal index by UUID")
     public ResponseEntity<ObjectNode> getDocumentByUUID(@PathVariable("uuid") String uuid) throws IOException {
         log.info("getting a document form portal by UUID: {}", uuid);
-        ObjectNode response =  indexerService.getDocumentByUUID(uuid).source();
+        ObjectNode response =  indexerMetadata.getDocumentByUUID(uuid).source();
         return ResponseEntity.status(HttpStatus.OK).body(response);
     }
     /**
@@ -69,13 +70,12 @@ public ResponseEntity<String> indexAllMetadataRecords(
             @RequestParam(value = "confirm", defaultValue = "false") Boolean confirm,
             @RequestParam(value = "beginWithUuid", required=false) String beginWithUuid) throws IOException {
 
-        List<BulkResponse> responses = indexerService.indexAllMetadataRecordsFromGeoNetwork(beginWithUuid, confirm, null);
+        List<BulkResponse> responses = indexerMetadata.indexAllMetadataRecordsFromGeoNetwork(beginWithUuid, confirm, null);
         return ResponseEntity.ok(responses.toString());
     }
     /**
      * Emit result to FE so it will not result in gateway time-out. You need to run it with postman or whatever tools
      * support server side event, the content type needs to be text/event-stream in order to work
-     *
      * Noted: There is a bug in postman desktop, so either you run postman using web-browser with agent directly
      * or you need to have version 10.2 or above in order to get the emitted result
      *
@@ -90,8 +90,65 @@ public SseEmitter indexAllMetadataRecordsAsync(
             @RequestParam(value = "beginWithUuid", required=false) String beginWithUuid) {
 
         final SseEmitter emitter = new SseEmitter(0L); // 0L means no timeout;
+        final IndexService.Callback callback = createCallback(emitter);
+
+        new Thread(() -> {
+            try {
+                indexerMetadata.indexAllMetadataRecordsFromGeoNetwork(beginWithUuid, confirm, callback);
+            }
+            catch(IOException e) {
+                emitter.completeWithError(e);
+            }
+        }).start();
+
+        return emitter;
+    }
+
+    @PostMapping(path="/{uuid}", produces = "application/json")
+    @Operation(security = { @SecurityRequirement(name = "X-API-Key") }, description = "Index a metadata record by UUID")
+    public ResponseEntity<String> addDocumentByUUID(@PathVariable("uuid") String uuid) throws IOException, FactoryException, JAXBException, TransformException {
+        String metadataValues = geonetworkResourceService.searchRecordBy(uuid);
+
+        CompletableFuture<ResponseEntity<String>> f = indexerMetadata.indexMetadata(metadataValues);
+        // Return when done make it back to sync instead of async
+        return f.join();
+    }
+
+    @DeleteMapping(path="/{uuid}", produces = "application/json")
+    @Operation(security = { @SecurityRequirement(name = "X-API-Key") }, description = "Delete a metadata record by UUID")
+    public ResponseEntity<String> deleteDocumentByUUID(@PathVariable("uuid") String uuid) throws IOException {
+        return indexerMetadata.deleteDocumentByUUID(uuid);
+    }
+
+    @PostMapping(path="/{uuid}/dataset", produces = "application/json")
+    @Operation(security = {@SecurityRequirement(name = "X-API-Key") }, description = "Index a dataset by UUID")
+    public SseEmitter indexDatasetByUUID(@PathVariable("uuid") String uuid)  {
+
+        final SseEmitter emitter = new SseEmitter(0L); // 0L means no timeout;
+        final IndexService.Callback callback = createCallback(emitter);
 
-        IndexerService.Callback callback = new IndexerService.Callback() {
+        new Thread(() -> {
+            try {
+                List<TemporalExtent> temporalExtents = dataAccessService.getTemporalExtentOf(uuid);
+                if (!temporalExtents.isEmpty()) {
+                    // Only first block works from dataservice api
+                    LocalDate startDate = temporalExtents.get(0).getLocalStartDate();
+                    LocalDate endDate = temporalExtents.get(0).getLocalEndDate();
+                    log.info("Indexing dataset with UUID: {} from {} to {}", uuid, startDate, endDate);
+
+                    indexCloudOptimizedData.indexCloudOptimizedData(uuid, startDate, endDate, callback);
+                }
+            }
+            finally {
+                emitter.complete();
+            }
+        }).start();
+
+        return emitter;
+    }
+
+    protected IndexerMetadataService.Callback createCallback(SseEmitter emitter) {
+        return new IndexService.Callback() {
             @Override
             public void onProgress(Object update) {
                 try {
@@ -102,8 +159,7 @@ public void onProgress(Object update) {
                             .name("Indexer update event");
 
                     emitter.send(event);
-                }
-                catch (IOException e) {
+                } catch (IOException e) {
                     // In case of fail, try close the stream, if it cannot be closed. (likely stream terminated
                     // already, the load error out and we need to result from a particular uuid.
                     emitter.completeWithError(e);
@@ -121,57 +177,10 @@ public void onComplete(Object result) {
 
                     emitter.send(event);
                     emitter.complete();
-                }
-                catch (IOException e) {
+                } catch (IOException e) {
                     emitter.completeWithError(e);
                 }
             }
         };
-
-        new Thread(() -> {
-            try {
-                indexerService.indexAllMetadataRecordsFromGeoNetwork(beginWithUuid, confirm, callback);
-            }
-            catch(IOException e) {
-                emitter.completeWithError(e);
-            }
-        }).start();
-
-        return emitter;
-    }
-
-    @PostMapping(path="/{uuid}", produces = "application/json")
-    @Operation(security = { @SecurityRequirement(name = "X-API-Key") }, description = "Index a metadata record by UUID")
-    public ResponseEntity<String> addDocumentByUUID(@PathVariable("uuid") String uuid) throws IOException, FactoryException, JAXBException, TransformException, ExecutionException, InterruptedException {
-        String metadataValues = geonetworkResourceService.searchRecordBy(uuid);
-
-        CompletableFuture<ResponseEntity<String>> f = indexerService.indexMetadata(metadataValues);
-        // Return when done make it back to sync instead of async
-        return f.join();
-    }
-
-    @DeleteMapping(path="/{uuid}", produces = "application/json")
-    @Operation(security = { @SecurityRequirement(name = "X-API-Key") }, description = "Delete a metadata record by UUID")
-    public ResponseEntity<String> deleteDocumentByUUID(@PathVariable("uuid") String uuid) throws IOException {
-        return indexerService.deleteDocumentByUUID(uuid);
-    }
-
-    @PostMapping(path="/{uuid}/dataset", produces = "application/json")
-    @Operation(security = {@SecurityRequirement(name = "X-API-Key") }, description = "Index a dataset by UUID")
-    public ResponseEntity<List<String>> indexDatasetByUUID(@PathVariable("uuid") String uuid)  {
-
-        var temporalExtents = dataAccessService.getTemporalExtentOf(uuid);
-        var startDate = temporalExtents.getStartDate();
-        var endDate = temporalExtents.getEndDate();
-        log.info("Indexing dataset with UUID: {} from {} to {}", uuid, startDate, endDate);
-
-        var responses = indexerService.indexDataset(uuid, startDate, endDate);
-
-        List<String> result = new ArrayList<>();
-        for (BulkResponse response : responses) {
-            result.add(response.toString());
-        }
-
-        return ResponseEntity.ok(result);
     }
 }