Merge pull request #81 from aodn/sync-with-new-es-schema

refactor to sync with latest es-indexer changes
aodn · Aug 25, 2024 · 0c68e69 · 0c68e69
2 parents 78f5851 + 3fdff18
commit 0c68e69
Show file tree

Hide file tree

Showing 23 changed files with 277 additions and 227 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -5,7 +5,7 @@ COPY ./server/target/ogcapi-java-server-*-exec.jar app.jar
 ENTRYPOINT [\
     "java",\
     "-Delasticsearch.index.name=${INDEX_NAME}",\
-    "-Delasticsearch.search_as_you_type.category_suggest.index_name=${CATEGORY_INDEX_NAME}",\
+    "-Delasticsearch.vocabs_index.name=${VOCABS_INDEX_NAME}",\
     "-Dapi.host=${HOST}:${PORT}",\
     "-Dserver.port=${PORT}",\
     "-Delasticsearch.serverUrl=${ELASTIC_URL}",\

diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/common/RestExtApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/common/RestExtApi.java
@@ -1,6 +1,6 @@
 package au.org.aodn.ogcapi.server.common;
 
-import au.org.aodn.ogcapi.server.core.model.CategoryVocabModel;
+import au.org.aodn.ogcapi.server.core.model.ParameterVocabModel;
 import au.org.aodn.ogcapi.server.core.model.enumeration.CQLCrsType;
 import au.org.aodn.ogcapi.server.core.service.Search;
 import io.swagger.v3.oas.annotations.Parameter;
@@ -52,19 +52,19 @@ public class RestExtApi {
     /**
      * Evict cache to allow reload
      */
-    @CacheEvict(value="parameter_categories", allEntries = true)
-    @Scheduled(fixedRateString = "${caching.parameter_category.ttl:43200000}")
-    public void emptyCachedParameterCategory() {
-        log.info("Evict parameter_category cache as TTL pass");
+    @CacheEvict(value="parameter_vocabs", allEntries = true)
+    @Scheduled(fixedRateString = "${caching.parameter_vocab.ttl:43200000}")
+    public void emptyCachedParameterVocab() {
+        log.info("Evict parameter_vocab cache as TTL pass");
     }
 
     /**
      * Value cached to avoid excessive load
      * @return
      */
-    @Cacheable("parameter_categories")
-    @GetMapping(path="/parameter/categories")
-    public ResponseEntity<List<CategoryVocabModel>> getParameterCategory() {
-        return ResponseEntity.ok(restExtService.getParameterCategory(vocabApi));
+    @Cacheable("parameter_vocabs")
+    @GetMapping(path="/parameter/vocabs")
+    public ResponseEntity<List<ParameterVocabModel>> getParameterVocab() {
+        return ResponseEntity.ok(restExtService.getParameterVocab(vocabApi));
     }
 }
diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/common/RestExtService.java b/server/src/main/java/au/org/aodn/ogcapi/server/common/RestExtService.java
@@ -1,6 +1,6 @@
 package au.org.aodn.ogcapi.server.common;
 
-import au.org.aodn.ogcapi.server.core.model.CategoryVocabModel;
+import au.org.aodn.ogcapi.server.core.model.ParameterVocabModel;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.databind.node.TextNode;
@@ -39,16 +39,16 @@ public class RestExtService {
 
     /**
      * We want to get the list of leaf node for the API, from there we need to query individual resources to get the broadMatch value
-     * this value is the link to the second level of the category
+     * this value is the link to the second level of the vocab
      *
      * API to the details to get the broadMatch
      * http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-discovery-parameter-vocabulary/version-1-6/resource.json?uri=http://vocab.aodn.org.au/def/discovery_parameter/891
      *
      * @param vocabApiBase
      * @return
      */
-    protected Map<String, List<CategoryVocabModel>> getLeafNodeOfParameterCategory(String vocabApiBase) {
-        Map<String, List<CategoryVocabModel>> result = new HashMap<>();
+    protected Map<String, List<ParameterVocabModel>> getLeafNodeOfParameterVocab(String vocabApiBase) {
+        Map<String, List<ParameterVocabModel>> result = new HashMap<>();
         String url = String.format(vocabApiBase + leafPath);
 
         while (url != null) {
@@ -69,7 +69,7 @@ protected Map<String, List<CategoryVocabModel>> getLeafNodeOfParameterCategory(S
                             if(isNodeValid.apply(d, "result") && isNodeValid.apply(d.get("result"), "primaryTopic")) {
                                 JsonNode target = d.get("result").get("primaryTopic");
 
-                                CategoryVocabModel model = CategoryVocabModel
+                                ParameterVocabModel model = ParameterVocabModel
                                         .builder()
                                         .label(label.apply(target).toLowerCase())
                                         .definition(definition.apply(target))
@@ -107,17 +107,17 @@ protected Map<String, List<CategoryVocabModel>> getLeafNodeOfParameterCategory(S
         return result;
     }
 
-    protected CategoryVocabModel buildCategoryVocabModel(JsonNode currentNode, JsonNode outerNode) {
+    protected ParameterVocabModel buildParameterVocabModel(JsonNode currentNode, JsonNode outerNode) {
         if (currentNode instanceof ObjectNode objectNode) {
             if (objectNode.has("prefLabel") && objectNode.has("_about")) {
-                return CategoryVocabModel.builder()
+                return ParameterVocabModel.builder()
                         .about(about.apply(currentNode))
                         .label(label.apply(currentNode).toLowerCase())
                         .build();
             }
         } else if (currentNode instanceof TextNode textNode) {
             if (textNode.asText().contains("parameter_classes")) {
-                return CategoryVocabModel.builder()
+                return ParameterVocabModel.builder()
                         .about(textNode.asText())
                         .label(this.findLabelByAbout(outerNode, textNode.asText()).toLowerCase())
                         .build();
@@ -135,9 +135,9 @@ protected String findLabelByAbout(JsonNode node, String c) {
         return null;
     }
 
-    public List<CategoryVocabModel> getParameterCategory(String vocabApiBase) {
-        Map<String, List<CategoryVocabModel>> leaves = getLeafNodeOfParameterCategory(vocabApiBase);
-        List<CategoryVocabModel> result = new ArrayList<>();
+    public List<ParameterVocabModel> getParameterVocab(String vocabApiBase) {
+        Map<String, List<ParameterVocabModel>> leaves = getLeafNodeOfParameterVocab(vocabApiBase);
+        List<ParameterVocabModel> result = new ArrayList<>();
 
         String url = String.format(vocabApiBase + path);
 
@@ -152,21 +152,21 @@ public List<CategoryVocabModel> getParameterCategory(String vocabApiBase) {
 
                     if (!node.isEmpty() && node.has("items") && !node.get("items").isEmpty()) {
                         for (JsonNode j : node.get("items")) {
-                            List<CategoryVocabModel> broader = new ArrayList<>();
-                            List<CategoryVocabModel> narrower = new ArrayList<>();
+                            List<ParameterVocabModel> broader = new ArrayList<>();
+                            List<ParameterVocabModel> narrower = new ArrayList<>();
 
 
                             log.debug("Processing label {}", label.apply(j));
 
                             if (j.has("broader")) {
                                 for (JsonNode b : j.get("broader")) {
-                                    broader.add(this.buildCategoryVocabModel(b, node));
+                                    broader.add(this.buildParameterVocabModel(b, node));
                                 }
                             }
 
                             if (j.has("narrower")) {
                                 for (JsonNode b : j.get("narrower")) {
-                                    CategoryVocabModel c = this.buildCategoryVocabModel(b, node);
+                                    ParameterVocabModel c = this.buildParameterVocabModel(b, node);
                                     // The record comes from ardc have two levels only, so the second level for sure
                                     // is empty, but the third level info comes form another link (aka the leaves)
                                     // and therefore we can attach it to the second level to for the third.
@@ -177,7 +177,7 @@ public List<CategoryVocabModel> getParameterCategory(String vocabApiBase) {
                                 }
                             }
 
-                            CategoryVocabModel model = CategoryVocabModel
+                            ParameterVocabModel model = ParameterVocabModel
                                     .builder()
                                     .label(label.apply(j).toLowerCase())
                                     .definition(definition.apply(j))
@@ -201,7 +201,7 @@ public List<CategoryVocabModel> getParameterCategory(String vocabApiBase) {
                     url = null;
                 }
             } catch (RestClientException e) {
-                log.error("Fail connect {}, category return likely outdated", url);
+                log.error("Fail connect {}, parameter vocab return likely outdated", url);
                 url = null;
             }
         }

diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ArdcVocabModel.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ArdcVocabModel.java
@@ -0,0 +1,15 @@
+package au.org.aodn.ogcapi.server.core.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Builder;
+import lombok.Data;
+
+@Data
+@Builder
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ArdcVocabModel {
+    // properties are extendable (e.g platformVocabs, organisationVocabs etc.), currently just parameterVocabs.
+    @JsonProperty("parameter_vocab")
+    ParameterVocabModel parameterVocabModel;
+}
diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/CategorySuggestDTO.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/CategorySuggestDTO.java
diff --git a/...server/core/model/CategoryVocabModel.java → ...erver/core/model/ParameterVocabModel.java b/...server/core/model/CategoryVocabModel.java → ...erver/core/model/ParameterVocabModel.java
@@ -1,23 +1,22 @@
 package au.org.aodn.ogcapi.server.core.model;
 
-import lombok.Builder;
-import lombok.Getter;
-import lombok.Setter;
+import lombok.*;
 
 import java.util.List;
-import java.util.Map;
 
 /**
  * This is the model class for http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/
  */
 @Builder
 @Getter
 @Setter
-public class CategoryVocabModel {
+@NoArgsConstructor
+@AllArgsConstructor
+public class ParameterVocabModel {
 
     protected String label;
     protected String definition;
     protected String about;
-    protected List<CategoryVocabModel> broader;
-    protected List<CategoryVocabModel> narrower;
+    protected List<ParameterVocabModel> broader;
+    protected List<ParameterVocabModel> narrower;
 }
diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java
@@ -111,9 +111,9 @@ public enum CQLFields implements CQLFieldsInterface {
             null,
             null
     ),
-    category(
-            StacBasicField.DiscoveryCategories.searchField,
-            StacBasicField.DiscoveryCategories.displayField,
+    parameter_vocab(
+            StacBasicField.ParameterVocabs.searchField,
+            StacBasicField.ParameterVocabs.displayField,
             null,
             null
     ),
@@ -123,9 +123,9 @@ public enum CQLFields implements CQLFieldsInterface {
             null,
             null
     ),
-    discovery_categories(
-            StacBasicField.DiscoveryCategories.searchField,
-            StacBasicField.DiscoveryCategories.displayField,
+    parameter_vocabs(
+            StacBasicField.ParameterVocabs.searchField,
+            StacBasicField.ParameterVocabs.displayField,
             null,
             null
     ),

diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/StacBasicField.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/StacBasicField.java
@@ -8,9 +8,9 @@ public enum StacBasicField {
             "providers",    // This result in the whole provider section return
             "providers.name"
     ),
-    DiscoveryCategories(
-            "discovery_categories", // This result in the whole themes section return
-            "summaries.discovery_categories"
+    ParameterVocabs(
+            "parameter_vocabs", // This result in the whole themes section return
+            "summaries.parameter_vocabs"
     ),
     Links("links", "links")
     ;

diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java
@@ -1,6 +1,6 @@
 package au.org.aodn.ogcapi.server.core.service;
 
-import au.org.aodn.ogcapi.server.core.model.CategorySuggestDTO;
+import au.org.aodn.ogcapi.server.core.model.ParameterVocabModel;
 import au.org.aodn.ogcapi.server.core.model.RecordSuggestDTO;
 import au.org.aodn.ogcapi.server.core.model.enumeration.*;
 import au.org.aodn.ogcapi.server.core.parser.CQLToElasticFilterFactory;
@@ -17,6 +17,7 @@
 import co.elastic.clients.elasticsearch.core.search_mvt.GridType;
 import co.elastic.clients.transport.endpoints.BinaryResponse;
 import co.elastic.clients.util.ObjectBuilder;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.extern.slf4j.Slf4j;
 import org.geotools.filter.text.commons.CompilerUtil;
@@ -46,19 +47,14 @@ public class ElasticSearch extends ElasticSearchBase implements Search {
     @Value("${elasticsearch.search_as_you_type.record_suggest.fields}")
     protected String[] searchAsYouTypeEnabledFields;
 
-    /*
-     * this secondLevelCategorySuggestFilters for accessing the search_as_you_type "label" field
-     * of the second level categories (discovery_category index) will never be changed unless the schema is changed,
-     * or the discovery_category index is no longer be used
-     */
-    protected Query secondLevelCategorySuggestFilters = Query.of(q -> q.bool(b -> b.filter(f -> f.nested(n -> n.path("broader")
-            .query(qq -> qq.exists(e -> e.field("broader")))))));
+    @Value("${elasticsearch.vocabs_index.name}")
+    protected String vocabsIndexName;
 
-    @Value("${elasticsearch.search_as_you_type.category_suggest.field}")
-    protected String secondLevelCategorySuggestField;
+    @Value("${elasticsearch.search_as_you_type.parameter_vocab.path}")
+    protected String parameterVocabsPath;
 
-    @Value("${elasticsearch.search_as_you_type.category_suggest.index_name}")
-    protected String categorySuggestIndex;
+    @Value("${elasticsearch.search_as_you_type.parameter_vocab.field}")
+    protected String parameterVocabsField;
 
     public ElasticSearch(ElasticsearchClient client,
                          ObjectMapper mapper,
@@ -108,11 +104,11 @@ protected List<Hit<RecordSuggestDTO>> getRecordSuggestions(String input, String
         ));
 
         /*
-            this is where the discovery categories filter is applied
-            use term query for exact match of the categories
+            this is where the discovery parameter vocabs filter is applied
+            use term query for exact match of the parameter vocabs
             (e.g you don't want "something", "something special" and "something secret" be returned when searching for "something")
             see more: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-terms-query.html#query-dsl-terms-query
-            this query uses AND operator for the categories (e.g "wave" AND "temperature")
+            this query uses AND operator for the parameter vocabs (e.g "wave" AND "temperature")
         */
         List<Query> filters;
         if (cql != null) {
@@ -145,32 +141,41 @@ this query uses AND operator for the categories (e.g "wave" AND "temperature")
         return response.hits().hits();
     }
 
-    protected List<Hit<CategorySuggestDTO>> getCategorySuggestions(String input) throws IOException {
+    protected List<Hit<JsonNode>> getParameterVocabSuggestions(String input) throws IOException {
         // create query
-        Query secondLevelCategorySuggestQuery = this.generateSearchAsYouTypeQuery(input, secondLevelCategorySuggestField);
+        Query secondLevelParameterVocabSuggestQuery = this.generateSearchAsYouTypeQuery(input, parameterVocabsPath + "." + parameterVocabsField);
+
+        /*
+         * this secondLevelParameterVocabSuggestFilters for accessing the search_as_you_type "label" field
+         * of the second level vocabs of the parameter_vocabs field from the vocabs_index index will never be changed unless the schema is changed,
+         * or the vocabs_index index is no longer be used
+         */
+        Query secondLevelParameterVocabSuggestFilters = Query.of(q -> q.bool(b -> b.filter(f -> f.nested(n -> n.path(parameterVocabsPath + ".broader")
+                .query(qq -> qq.exists(e -> e.field(parameterVocabsPath + ".broader")))))));
 
         // create request
         SearchRequest searchRequest = this.buildSearchAsYouTypeRequest(
-                List.of("label"),
-                categorySuggestIndex,
-                List.of(secondLevelCategorySuggestQuery),
-                List.of(secondLevelCategorySuggestFilters));
+                List.of(parameterVocabsPath + "." + parameterVocabsField),
+                vocabsIndexName,
+                List.of(secondLevelParameterVocabSuggestQuery),
+                List.of(secondLevelParameterVocabSuggestFilters));
 
         // execute
-        log.info("getCategorySuggestions | Elastic search payload {}", searchRequest.toString());
-        SearchResponse<CategorySuggestDTO> response = esClient.search(searchRequest, CategorySuggestDTO.class);
-        log.info("getCategorySuggestions | Elastic search response {}", response);
+        log.info("getParameterVocabSuggestions | Elastic search payload {}", searchRequest.toString());
+        SearchResponse<JsonNode> response = esClient.search(searchRequest, JsonNode.class);
+        log.info("getParameterVocabSuggestions | Elastic search response {}", response);
 
         // return
         return response.hits().hits();
     }
 
     public ResponseEntity<Map<String, ?>> getAutocompleteSuggestions(String input, String cql, CQLCrsType coor) throws IOException, CQLException {
-        // extract category suggestions
-        Set<String> categorySuggestions = new HashSet<>();
-        for (Hit<CategorySuggestDTO> item : this.getCategorySuggestions(input)) {
-            if (item.source() != null) {
-                categorySuggestions.add(item.source().getLabel());
+        // extract parameter vocab suggestions
+        Set<String> parameterVocabSuggestions = new HashSet<>();
+        for (Hit<JsonNode> item : this.getParameterVocabSuggestions(input)) {
+            if (item.source() != null && item.source().get(parameterVocabsPath) != null) {
+                ParameterVocabModel parameterVocab = mapper.readValue(item.source().get(parameterVocabsPath).toString(), ParameterVocabModel.class);
+                parameterVocabSuggestions.add(parameterVocab.getLabel());
             }
         }
 
@@ -183,7 +188,7 @@ protected List<Hit<CategorySuggestDTO>> getCategorySuggestions(String input) thr
                 .collect(Collectors.toSet());
 
         Map<String, Object> allSuggestions = new HashMap<>();
-        allSuggestions.put("category_suggestions", new ArrayList<>(categorySuggestions));
+        allSuggestions.put("parameter_vocab_suggestions", new ArrayList<>(parameterVocabSuggestions));
 
         Map<String, Set<String>> recordSuggestions = new HashMap<>();
         recordSuggestions.put("suggest_phrases", abstractPhrases);