From b3b5b73e8be4dd79a01afcc223a1cfbbf7e8f925 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Date: Tue, 16 Apr 2024 16:45:08 +1000 Subject: [PATCH 01/19] boilerplate for context suggester implementation --- .../AbstractRequestEntityCreator.java | 18 +++++++++++++ .../GeoNetworkRequestEntityCreator.java | 16 +++++++++++ .../abstracts/OgcApiRequestEntityCreator.java | 17 ++++++++++++ .../service/GeoNetworkServiceImpl.java | 22 ++++++--------- .../esindexer/service/IndexerServiceImpl.java | 27 +++++++++++++++++-- indexer/src/main/resources/application.yaml | 3 +++ 6 files changed, 87 insertions(+), 16 deletions(-) create mode 100644 indexer/src/main/java/au/org/aodn/esindexer/abstracts/AbstractRequestEntityCreator.java create mode 100644 indexer/src/main/java/au/org/aodn/esindexer/abstracts/GeoNetworkRequestEntityCreator.java create mode 100644 indexer/src/main/java/au/org/aodn/esindexer/abstracts/OgcApiRequestEntityCreator.java diff --git a/indexer/src/main/java/au/org/aodn/esindexer/abstracts/AbstractRequestEntityCreator.java b/indexer/src/main/java/au/org/aodn/esindexer/abstracts/AbstractRequestEntityCreator.java new file mode 100644 index 00000000..b7a4b652 --- /dev/null +++ b/indexer/src/main/java/au/org/aodn/esindexer/abstracts/AbstractRequestEntityCreator.java @@ -0,0 +1,18 @@ +package au.org.aodn.esindexer.abstracts; + +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; + +public abstract class AbstractRequestEntityCreator { + abstract HttpHeaders createHeaders(MediaType accept); + + public HttpEntity getRequestEntity(String body) { + return getRequestEntity(null, body); + } + + public HttpEntity getRequestEntity(MediaType accept, String body) { + HttpHeaders headers = createHeaders(accept); + return body == null ? new HttpEntity<>(headers) : new HttpEntity<>(body, headers); + } +} diff --git a/indexer/src/main/java/au/org/aodn/esindexer/abstracts/GeoNetworkRequestEntityCreator.java b/indexer/src/main/java/au/org/aodn/esindexer/abstracts/GeoNetworkRequestEntityCreator.java new file mode 100644 index 00000000..68989705 --- /dev/null +++ b/indexer/src/main/java/au/org/aodn/esindexer/abstracts/GeoNetworkRequestEntityCreator.java @@ -0,0 +1,16 @@ +package au.org.aodn.esindexer.abstracts; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Component; + +import java.util.Collections; + +@Component +public class GeoNetworkRequestEntityCreator extends AbstractRequestEntityCreator { + @Override + HttpHeaders createHeaders(MediaType accept) { + HttpHeaders headers = new HttpHeaders(); + headers.setAccept(Collections.singletonList(accept == null ? MediaType.APPLICATION_XML : accept)); + return headers; + } +} diff --git a/indexer/src/main/java/au/org/aodn/esindexer/abstracts/OgcApiRequestEntityCreator.java b/indexer/src/main/java/au/org/aodn/esindexer/abstracts/OgcApiRequestEntityCreator.java new file mode 100644 index 00000000..4ec50a49 --- /dev/null +++ b/indexer/src/main/java/au/org/aodn/esindexer/abstracts/OgcApiRequestEntityCreator.java @@ -0,0 +1,17 @@ +package au.org.aodn.esindexer.abstracts; + +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Component; + +import java.util.Collections; + +@Component +public class OgcApiRequestEntityCreator extends AbstractRequestEntityCreator { + @Override + HttpHeaders createHeaders(MediaType accept) { + HttpHeaders headers = new HttpHeaders(); + headers.setAccept(Collections.singletonList(accept == null ? MediaType.APPLICATION_JSON : accept)); + return headers; + } +} diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/GeoNetworkServiceImpl.java b/indexer/src/main/java/au/org/aodn/esindexer/service/GeoNetworkServiceImpl.java index a3dce472..a90822dc 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/GeoNetworkServiceImpl.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/GeoNetworkServiceImpl.java @@ -1,5 +1,6 @@ package au.org.aodn.esindexer.service; +import au.org.aodn.esindexer.abstracts.GeoNetworkRequestEntityCreator; import au.org.aodn.esindexer.exception.MetadataNotFoundException; import au.org.aodn.esindexer.utils.StringUtil; import au.org.aodn.esindexer.configuration.AppConstants; @@ -24,11 +25,14 @@ import java.util.*; @Service -public class GeoNetworkServiceImpl implements GeoNetworkService { +public class GeoNetworkServiceImpl implements GeoNetworkService { @Autowired RestTemplate restTemplate; + @Autowired + GeoNetworkRequestEntityCreator geoNetworkRequestEntityCreator; + @Autowired @Qualifier("gn4ElasticsearchClient") protected ElasticsearchClient gn4ElasticClient; @@ -41,16 +45,6 @@ public class GeoNetworkServiceImpl implements GeoNetworkService { protected String server; - protected HttpEntity getRequestEntity(String body) { - return getRequestEntity(null, body); - } - - protected HttpEntity getRequestEntity(MediaType accept, String body) { - HttpHeaders headers = new HttpHeaders(); - headers.setAccept(Collections.singletonList(accept == null ? MediaType.APPLICATION_XML : accept)); - return body == null ? new HttpEntity<>(headers) : new HttpEntity<>(body, headers); - } - protected final static String UUID = "uuid"; protected final static String GEONETWORK_GROUP = "groupOwner"; @@ -94,7 +88,7 @@ public String findGroupById(String uuid) throws IOException { Map params = new HashMap<>(); params.put("id", group); - HttpEntity requestEntity = getRequestEntity(MediaType.APPLICATION_JSON, null); + HttpEntity requestEntity = geoNetworkRequestEntityCreator.getRequestEntity(MediaType.APPLICATION_JSON, null); ResponseEntity responseEntity = restTemplate.exchange( getGeoNetworkGroupsEndpoint(), @@ -117,7 +111,7 @@ public String findGroupById(String uuid) throws IOException { protected String findFormatterId(String uuid) { try { - HttpEntity requestEntity = getRequestEntity(MediaType.APPLICATION_JSON, null); + HttpEntity requestEntity = geoNetworkRequestEntityCreator.getRequestEntity(MediaType.APPLICATION_JSON, null); Map params = new HashMap<>(); params.put("indexName", getIndexName()); @@ -148,7 +142,7 @@ protected String findFormatterId(String uuid) { public String searchRecordBy(String uuid) { try { - HttpEntity requestEntity = getRequestEntity(null); + HttpEntity requestEntity = geoNetworkRequestEntityCreator.getRequestEntity(null); Map params = new HashMap<>(); params.put("indexName", getIndexName()); diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java b/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java index 58773e12..ca7d1ca7 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java @@ -1,5 +1,6 @@ package au.org.aodn.esindexer.service; +import au.org.aodn.esindexer.abstracts.OgcApiRequestEntityCreator; import au.org.aodn.stac.model.StacCollectionModel; import au.org.aodn.esindexer.utils.StringUtil; import au.org.aodn.esindexer.configuration.AppConstants; @@ -17,6 +18,7 @@ import co.elastic.clients.elasticsearch.indices.CreateIndexResponse; import co.elastic.clients.json.JsonData; import co.elastic.clients.transport.endpoints.BooleanResponse; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import jakarta.xml.bind.JAXBException; @@ -27,10 +29,11 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.ClassPathResource; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; +import org.springframework.http.*; import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; +import javax.annotation.PostConstruct; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -49,6 +52,15 @@ public class IndexerServiceImpl implements IndexerService { @Value("${elasticsearch.index.name}") private String indexName; + @Value("${ogc-api.host}") + private String ogcApiHost; + + @Autowired + RestTemplate restTemplate; + + @Autowired + OgcApiRequestEntityCreator ogcApiRequestEntityCreator; + @Autowired ObjectMapper objectMapper; @@ -156,6 +168,17 @@ protected StacCollectionModel getMappedMetadataValues(String metadataValues) thr return stacCollectionModel; } + @PostConstruct + protected JsonNode fetchARDCVocabs() { + HttpEntity requestEntity = ogcApiRequestEntityCreator.getRequestEntity(MediaType.APPLICATION_JSON, null); + ResponseEntity responseEntity = restTemplate.exchange( + ogcApiHost + "/api/v1/ogc/ext/parameter/categories", + HttpMethod.GET, + requestEntity, + JsonNode.class); + return responseEntity.getBody(); + } + public ResponseEntity indexMetadata(String metadataValues) { try { StacCollectionModel mappedMetadataValues = this.getMappedMetadataValues(metadataValues); diff --git a/indexer/src/main/resources/application.yaml b/indexer/src/main/resources/application.yaml index 28568194..3f2f0203 100644 --- a/indexer/src/main/resources/application.yaml +++ b/indexer/src/main/resources/application.yaml @@ -38,6 +38,9 @@ geonetwork: index: "records" endpoint: /geonetwork/srv/api/search +ogc-api: + host: http://localhost:8082 + springdoc: api-docs: enabled: true From a43170ab37b541add5b9a042a5571e17b460b2e2 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Date: Tue, 16 Apr 2024 17:54:30 +1000 Subject: [PATCH 02/19] Extend Summaries class --- .gitignore | 3 +- .../AbstractRequestEntityCreator.java | 1 + .../GeoNetworkRequestEntityCreator.java | 1 + .../abstracts/OgcApiRequestEntityCreator.java | 1 + .../esindexer/service/IndexerServiceImpl.java | 34 +++-------- .../AodnDiscoveryParameterVocabUtils.java | 57 +++++++++++++++++++ .../org/aodn/stac/model/SummariesModel.java | 7 +++ 7 files changed, 77 insertions(+), 27 deletions(-) create mode 100644 indexer/src/main/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtils.java diff --git a/.gitignore b/.gitignore index 047d7a3c..f01ca230 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,8 @@ target/ !.mvn !**/src/main/**/target/ !**/src/test/**/target/ -src/main/generated/ +**/main/generated/ + ### STS ### .apt_generated diff --git a/indexer/src/main/java/au/org/aodn/esindexer/abstracts/AbstractRequestEntityCreator.java b/indexer/src/main/java/au/org/aodn/esindexer/abstracts/AbstractRequestEntityCreator.java index b7a4b652..ed453147 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/abstracts/AbstractRequestEntityCreator.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/abstracts/AbstractRequestEntityCreator.java @@ -5,6 +5,7 @@ import org.springframework.http.MediaType; public abstract class AbstractRequestEntityCreator { + // abstract method to be implemented differently by subclasses abstract HttpHeaders createHeaders(MediaType accept); public HttpEntity getRequestEntity(String body) { diff --git a/indexer/src/main/java/au/org/aodn/esindexer/abstracts/GeoNetworkRequestEntityCreator.java b/indexer/src/main/java/au/org/aodn/esindexer/abstracts/GeoNetworkRequestEntityCreator.java index 68989705..3acdef3b 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/abstracts/GeoNetworkRequestEntityCreator.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/abstracts/GeoNetworkRequestEntityCreator.java @@ -7,6 +7,7 @@ @Component public class GeoNetworkRequestEntityCreator extends AbstractRequestEntityCreator { + // used by GeoNetwork related requests @Override HttpHeaders createHeaders(MediaType accept) { HttpHeaders headers = new HttpHeaders(); diff --git a/indexer/src/main/java/au/org/aodn/esindexer/abstracts/OgcApiRequestEntityCreator.java b/indexer/src/main/java/au/org/aodn/esindexer/abstracts/OgcApiRequestEntityCreator.java index 4ec50a49..69630a26 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/abstracts/OgcApiRequestEntityCreator.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/abstracts/OgcApiRequestEntityCreator.java @@ -8,6 +8,7 @@ @Component public class OgcApiRequestEntityCreator extends AbstractRequestEntityCreator { + // used by OGC-API related requests @Override HttpHeaders createHeaders(MediaType accept) { HttpHeaders headers = new HttpHeaders(); diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java b/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java index ca7d1ca7..b51616eb 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java @@ -1,8 +1,7 @@ package au.org.aodn.esindexer.service; -import au.org.aodn.esindexer.abstracts.OgcApiRequestEntityCreator; +import au.org.aodn.esindexer.utils.AodnDiscoveryParameterVocabUtils; import au.org.aodn.stac.model.StacCollectionModel; -import au.org.aodn.esindexer.utils.StringUtil; import au.org.aodn.esindexer.configuration.AppConstants; import au.org.aodn.esindexer.exception.*; import au.org.aodn.esindexer.utils.JaxbUtils; @@ -18,7 +17,6 @@ import co.elastic.clients.elasticsearch.indices.CreateIndexResponse; import co.elastic.clients.json.JsonData; import co.elastic.clients.transport.endpoints.BooleanResponse; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import jakarta.xml.bind.JAXBException; @@ -31,9 +29,6 @@ import org.springframework.core.io.ClassPathResource; import org.springframework.http.*; import org.springframework.stereotype.Service; -import org.springframework.web.client.RestTemplate; - -import javax.annotation.PostConstruct; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -52,15 +47,6 @@ public class IndexerServiceImpl implements IndexerService { @Value("${elasticsearch.index.name}") private String indexName; - @Value("${ogc-api.host}") - private String ogcApiHost; - - @Autowired - RestTemplate restTemplate; - - @Autowired - OgcApiRequestEntityCreator ogcApiRequestEntityCreator; - @Autowired ObjectMapper objectMapper; @@ -73,6 +59,9 @@ public class IndexerServiceImpl implements IndexerService { @Autowired RankingService rankingService; + @Autowired + AodnDiscoveryParameterVocabUtils aodnDiscoveryParameterVocabUtils; + private static final Logger logger = LogManager.getLogger(IndexerServiceImpl.class); protected long getDocumentsCount() { @@ -165,18 +154,11 @@ protected StacCollectionModel getMappedMetadataValues(String metadataValues) thr stacCollectionModel.setTitleSuggest(stacCollectionModel.getTitle()); - return stacCollectionModel; - } + // set AODN Discovery Categories + List aodnDiscoveryCategories = aodnDiscoveryParameterVocabUtils.getAodnDiscoveryCategories(stacCollectionModel.getThemes()); + stacCollectionModel.getSummaries().setAodnDiscoveryCategories(aodnDiscoveryCategories); - @PostConstruct - protected JsonNode fetchARDCVocabs() { - HttpEntity requestEntity = ogcApiRequestEntityCreator.getRequestEntity(MediaType.APPLICATION_JSON, null); - ResponseEntity responseEntity = restTemplate.exchange( - ogcApiHost + "/api/v1/ogc/ext/parameter/categories", - HttpMethod.GET, - requestEntity, - JsonNode.class); - return responseEntity.getBody(); + return stacCollectionModel; } public ResponseEntity indexMetadata(String metadataValues) { diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtils.java new file mode 100644 index 00000000..f72c78e6 --- /dev/null +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtils.java @@ -0,0 +1,57 @@ +package au.org.aodn.esindexer.utils; + +import au.org.aodn.esindexer.abstracts.OgcApiRequestEntityCreator; +import au.org.aodn.stac.model.ThemesModel; +import com.fasterxml.jackson.databind.JsonNode; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; + +import javax.annotation.PostConstruct; +import java.util.ArrayList; +import java.util.List; + +@Component +public class AodnDiscoveryParameterVocabUtils { + + @Value("${ogc-api.host}") + private String ogcApiHost; + + @Autowired + RestTemplate restTemplate; + + @Autowired + OgcApiRequestEntityCreator ogcApiRequestEntityCreator; + + @PostConstruct + protected JsonNode fetchAodnDiscoveryParameterVocabs() { + HttpEntity requestEntity = ogcApiRequestEntityCreator.getRequestEntity(MediaType.APPLICATION_JSON, null); + ResponseEntity responseEntity = restTemplate.exchange( + ogcApiHost + "/api/v1/ogc/ext/parameter/categories", + HttpMethod.GET, + requestEntity, + JsonNode.class); + return responseEntity.getBody(); + } + + public List getAodnDiscoveryCategories(List themes) { + List result = new ArrayList<>(); + JsonNode aodnDiscoveryParameterVocabs = this.fetchAodnDiscoveryParameterVocabs(); + themes.forEach(theme -> { + if (theme.getTitle().equals("AODN Discovery Parameter Vocabulary")) { + theme.getConcepts().forEach(concept -> { + String id = concept.get("id"); + String url = concept.get("url"); + // TODO: do something with id and url + System.out.println("id: " + id + " | url: " + url); + }); + } + }); + return result; + } +} diff --git a/stacmodel/src/main/java/au/org/aodn/stac/model/SummariesModel.java b/stacmodel/src/main/java/au/org/aodn/stac/model/SummariesModel.java index 36b81f95..ad644666 100644 --- a/stacmodel/src/main/java/au/org/aodn/stac/model/SummariesModel.java +++ b/stacmodel/src/main/java/au/org/aodn/stac/model/SummariesModel.java @@ -14,6 +14,7 @@ public class SummariesModel { protected Integer score; protected String status; protected Map scope; + /** * Group info as setup in geonetwork */ @@ -39,4 +40,10 @@ public class SummariesModel { */ @JsonProperty("temporal") protected List> temporal; + + /** + * second-level AODN discovery parameter vocabularies + */ + @JsonProperty("aodn_discovery_categories") + protected List aodnDiscoveryCategories; } From 6a0e964ac1e48ff41bdc135842c77a77bd34ac8b Mon Sep 17 00:00:00 2001 From: Viet Nguyen Date: Wed, 17 Apr 2024 12:34:41 +1000 Subject: [PATCH 03/19] extract record's second-level aodn discovery vocabs --- .../service/StacCollectionMapperService.java | 15 ++-- .../AodnDiscoveryParameterVocabUtils.java | 75 +++++++++++++++---- .../au/org/aodn/stac/model/ConceptModel.java | 26 +++++++ .../au/org/aodn/stac/model/ThemesModel.java | 2 +- 4 files changed, 95 insertions(+), 23 deletions(-) create mode 100644 stacmodel/src/main/java/au/org/aodn/stac/model/ConceptModel.java diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java b/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java index 42691d73..4ea3060d 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java @@ -312,19 +312,21 @@ String mapGeoNetworkGroup(MDMetadataType source) { } } - protected List> mapThemesConcepts(MDKeywordsPropertyType descriptiveKeyword) { - List> keywords = new ArrayList<>(); + protected List mapThemesConcepts(MDKeywordsPropertyType descriptiveKeyword) { + List concepts = new ArrayList<>(); descriptiveKeyword.getMDKeywords().getKeyword().forEach(keyword -> { if (keyword != null) { + ConceptModel conceptModel = ConceptModel.builder().build(); if (keyword.getCharacterString().getValue() instanceof AnchorType value) { - keywords.add(Map.of("id", value.getValue(), - "url", value.getHref())); + conceptModel.setId(value.getValue()); + conceptModel.setUrl(value.getHref()); } else { - keywords.add(Map.of("id", keyword.getCharacterString().getValue().toString())); + conceptModel.setId(keyword.getCharacterString().getValue().toString()); } + concepts.add(conceptModel); } }); - return keywords; + return concepts; } protected String mapThemesTitle(MDKeywordsPropertyType descriptiveKeyword, String uuid) { @@ -389,7 +391,6 @@ List mapThemes(MDMetadataType source) { String uuid = this.mapUUID(source); themesModel.setConcepts(mapThemesConcepts(descriptiveKeyword)); - themesModel.setTitle(mapThemesTitle(descriptiveKeyword, uuid)); themesModel.setDescription(mapThemesDescription(descriptiveKeyword, uuid)); themesModel.setScheme(mapThemesScheme(descriptiveKeyword, uuid)); diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtils.java index f72c78e6..36426d00 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtils.java @@ -1,8 +1,10 @@ package au.org.aodn.esindexer.utils; import au.org.aodn.esindexer.abstracts.OgcApiRequestEntityCreator; +import au.org.aodn.stac.model.ConceptModel; import au.org.aodn.stac.model.ThemesModel; import com.fasterxml.jackson.databind.JsonNode; +import jakarta.annotation.PostConstruct; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpEntity; @@ -12,9 +14,9 @@ import org.springframework.stereotype.Component; import org.springframework.web.client.RestTemplate; -import javax.annotation.PostConstruct; import java.util.ArrayList; import java.util.List; +import java.util.Objects; @Component public class AodnDiscoveryParameterVocabUtils { @@ -28,30 +30,73 @@ public class AodnDiscoveryParameterVocabUtils { @Autowired OgcApiRequestEntityCreator ogcApiRequestEntityCreator; + /* run this method after the bean is created */ @PostConstruct - protected JsonNode fetchAodnDiscoveryParameterVocabs() { + protected List fetchAodnDiscoveryParameterVocabs() { HttpEntity requestEntity = ogcApiRequestEntityCreator.getRequestEntity(MediaType.APPLICATION_JSON, null); ResponseEntity responseEntity = restTemplate.exchange( ogcApiHost + "/api/v1/ogc/ext/parameter/categories", HttpMethod.GET, requestEntity, JsonNode.class); - return responseEntity.getBody(); + + List results = new ArrayList<>(); + for (JsonNode obj : Objects.requireNonNull(responseEntity.getBody())) { + /* + the json object contains all level vocabs (first -> second -> bottom) + has list of items inside "narrower" field but empty inside "broader" field, + the pattern can be verified by inspecting the response from OGCApi parameter categories endpoint + and from the current AODN portal's parameter menu + "results" length should be 4; + */ + if (!obj.get("narrower").isEmpty() && obj.get("broader").isEmpty()) { + results.add(obj); + } + } + + return results; + } + + protected boolean themesMatchConcept(List themes, ConceptModel concept) { + for (ThemesModel theme : themes) { + for (ConceptModel themeConcept : theme.getConcepts()) { + /* + comparing by hashcode (id and url) of the concept object + this will prevent cases where bottom-level vocabs are the same in text, but their parent vocabs are different + e.g "car -> parts" vs "bike -> parts" ("parts" is the same but different parent) + */ + if (themeConcept.equals(concept)) { + return true; + } + } + } + return false; } public List getAodnDiscoveryCategories(List themes) { - List result = new ArrayList<>(); - JsonNode aodnDiscoveryParameterVocabs = this.fetchAodnDiscoveryParameterVocabs(); - themes.forEach(theme -> { - if (theme.getTitle().equals("AODN Discovery Parameter Vocabulary")) { - theme.getConcepts().forEach(concept -> { - String id = concept.get("id"); - String url = concept.get("url"); - // TODO: do something with id and url - System.out.println("id: " + id + " | url: " + url); - }); + List results = new ArrayList<>(); + // Iterate over the top-level vocabularies + for (JsonNode topLevelVocab : this.fetchAodnDiscoveryParameterVocabs()) { + if (topLevelVocab.has("narrower") && !topLevelVocab.get("narrower").isEmpty()) { + for (JsonNode secondLevelVocab : topLevelVocab.get("narrower")) { + String secondLevelVocabLabel = secondLevelVocab.get("label").asText(); + if (secondLevelVocab.has("narrower") && !secondLevelVocab.get("narrower").isEmpty()) { + for (JsonNode bottomLevelVocab : secondLevelVocab.get("narrower")) { + // map the original values to a ConceptModel object for doing comparison + ConceptModel bottomConcept = ConceptModel.builder() + .id(bottomLevelVocab.get("label").asText()) + .url(bottomLevelVocab.get("about").asText()) + .build(); + // Compare with themes' concepts + if (themesMatchConcept(themes, bottomConcept)) { + results.add(secondLevelVocabLabel); + break; // To avoid duplicates because under the same second-level vocab there can be multiple bottom-level vocabs that pass the condition + } + } + } + } } - }); - return result; + } + return results; } } diff --git a/stacmodel/src/main/java/au/org/aodn/stac/model/ConceptModel.java b/stacmodel/src/main/java/au/org/aodn/stac/model/ConceptModel.java new file mode 100644 index 00000000..e63109ce --- /dev/null +++ b/stacmodel/src/main/java/au/org/aodn/stac/model/ConceptModel.java @@ -0,0 +1,26 @@ +package au.org.aodn.stac.model; + +import lombok.Builder; +import lombok.Data; + +import java.util.Objects; + + +@Data +@Builder +public class ConceptModel { + private String id; + private String url; + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ConceptModel that = (ConceptModel) o; + return Objects.equals(id, that.id) && + Objects.equals(url, that.url); + } + @Override + public int hashCode() { + return Objects.hash(id, url); + } +} diff --git a/stacmodel/src/main/java/au/org/aodn/stac/model/ThemesModel.java b/stacmodel/src/main/java/au/org/aodn/stac/model/ThemesModel.java index 291f2118..50189f43 100644 --- a/stacmodel/src/main/java/au/org/aodn/stac/model/ThemesModel.java +++ b/stacmodel/src/main/java/au/org/aodn/stac/model/ThemesModel.java @@ -9,7 +9,7 @@ @Data @Builder public class ThemesModel { - protected List> concepts; + protected List concepts; protected String scheme; protected String description; protected String title; From a6834ca5d324da6dfe272f715c91ef88a9d97bc5 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Date: Wed, 17 Apr 2024 15:07:42 +1000 Subject: [PATCH 04/19] Refactor and add test cases --- .../AodnDiscoveryParameterVocabUtils.java | 23 +- .../portal_records_index_schema.json | 1 + .../AodnDiscoveryParameterVocabUtilsTest.java | 89 ++ .../aodn_discovery_parameter_vocabs.json | 1400 +++++++++++++++++ .../au/org/aodn/stac/model/ConceptModel.java | 6 + .../au/org/aodn/stac/model/ThemesModel.java | 7 + 6 files changed, 1505 insertions(+), 21 deletions(-) create mode 100644 indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java create mode 100644 indexer/src/test/resources/canned/aodn_discovery_parameter_vocabs.json diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtils.java index 36426d00..11c54e69 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtils.java @@ -4,7 +4,6 @@ import au.org.aodn.stac.model.ConceptModel; import au.org.aodn.stac.model.ThemesModel; import com.fasterxml.jackson.databind.JsonNode; -import jakarta.annotation.PostConstruct; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpEntity; @@ -16,7 +15,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Objects; @Component public class AodnDiscoveryParameterVocabUtils { @@ -30,31 +28,14 @@ public class AodnDiscoveryParameterVocabUtils { @Autowired OgcApiRequestEntityCreator ogcApiRequestEntityCreator; - /* run this method after the bean is created */ - @PostConstruct - protected List fetchAodnDiscoveryParameterVocabs() { + protected JsonNode fetchAodnDiscoveryParameterVocabs() { HttpEntity requestEntity = ogcApiRequestEntityCreator.getRequestEntity(MediaType.APPLICATION_JSON, null); ResponseEntity responseEntity = restTemplate.exchange( ogcApiHost + "/api/v1/ogc/ext/parameter/categories", HttpMethod.GET, requestEntity, JsonNode.class); - - List results = new ArrayList<>(); - for (JsonNode obj : Objects.requireNonNull(responseEntity.getBody())) { - /* - the json object contains all level vocabs (first -> second -> bottom) - has list of items inside "narrower" field but empty inside "broader" field, - the pattern can be verified by inspecting the response from OGCApi parameter categories endpoint - and from the current AODN portal's parameter menu - "results" length should be 4; - */ - if (!obj.get("narrower").isEmpty() && obj.get("broader").isEmpty()) { - results.add(obj); - } - } - - return results; + return responseEntity.getBody(); } protected boolean themesMatchConcept(List themes, ConceptModel concept) { diff --git a/indexer/src/main/resources/config_files/portal_records_index_schema.json b/indexer/src/main/resources/config_files/portal_records_index_schema.json index 3968e4aa..1f9a5dfe 100644 --- a/indexer/src/main/resources/config_files/portal_records_index_schema.json +++ b/indexer/src/main/resources/config_files/portal_records_index_schema.json @@ -76,6 +76,7 @@ "properties" : { "score": { "type": "long" }, "status": { "type": "text" }, + "aodn_discovery_categories" : { "type": "text" }, "scope" : { "type": "nested", "properties" : { diff --git a/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java b/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java new file mode 100644 index 00000000..26bdbfb7 --- /dev/null +++ b/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java @@ -0,0 +1,89 @@ +package au.org.aodn.esindexer.utils; + +import au.org.aodn.esindexer.BaseTestClass; +import au.org.aodn.esindexer.abstracts.OgcApiRequestEntityCreator; +import au.org.aodn.stac.model.ConceptModel; +import au.org.aodn.stac.model.ThemesModel; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.*; +import org.mockito.Mock; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.web.client.RestTemplate; +import org.springframework.http.MediaType; +import org.springframework.http.HttpMethod; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles("test") +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class AodnDiscoveryParameterVocabUtilsTest extends BaseTestClass { + @Mock + private RestTemplate restTemplate; + + @Mock + private OgcApiRequestEntityCreator ogcApiRequestEntityCreator; + + @Autowired + AodnDiscoveryParameterVocabUtils aodnDiscoveryParameterVocabUtils; + + @BeforeEach + void setup() throws IOException { + String jsonString = readResourceFile("classpath:canned/aodn_discovery_parameter_vocabs.json"); + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode responseJsonNode = objectMapper.readTree(jsonString); + HttpEntity mockHttpEntity = new HttpEntity<>(MediaType.APPLICATION_JSON_VALUE); + ResponseEntity mockResponseEntity = ResponseEntity.ok(responseJsonNode); + + when(ogcApiRequestEntityCreator.getRequestEntity(MediaType.APPLICATION_JSON, null)).thenReturn(mockHttpEntity); + when(restTemplate.exchange( + anyString(), + eq(HttpMethod.GET), + eq(mockHttpEntity), + eq(JsonNode.class) + )).thenReturn(mockResponseEntity); + } + + @Test + void testFetchAodnDiscoveryParameterVocabs() { + JsonNode results = aodnDiscoveryParameterVocabUtils.fetchAodnDiscoveryParameterVocabs(); + // Verification + assertEquals(33, results.size()); + } + + @Test + void testGetAodnDiscoveryCategories() throws Exception { + // Prepare themes + List themes = List.of( + new ThemesModel(Arrays.asList( + new ConceptModel("Temperature of the water body", "http://vocab.nerc.ac.uk/collection/P01/current/TEMPPR01"), + new ConceptModel("Practical salinity of the water body", "http://vocab.nerc.ac.uk/collection/P01/current/PSLTZZ01"), + new ConceptModel("Concentration of carbon (total inorganic) per unit mass of the water body", "http://vocab.aodn.org.au/def/discovery_parameter/entity/1"), + new ConceptModel("Total alkalinity per unit mass of the water body", "http://vocab.nerc.ac.uk/collection/P01/current/MDMAP014"), + new ConceptModel("Saturation state of aragonite in the water body", "http://vocab.aodn.org.au/def/discovery_parameter/entity/24"), + new ConceptModel("Saturation state of aragonite in the water body", "http://vocab.aodn.org.au/def/discovery_parameter/entity/24"), + new ConceptModel("pH (total scale) of the water body", "http://vocab.aodn.org.au/def/discovery_parameter/entity/27") + ), "theme", null, "AODN Discovery Parameter Vocabulary") + ); + + // Perform the test + List categories = aodnDiscoveryParameterVocabUtils.getAodnDiscoveryCategories(themes); + + // Assertions + assertNotNull(categories); + assertTrue(categories.contains("Alkalinity")); + assertTrue(categories.contains("Temperature")); + assertTrue(categories.contains("Salinity")); + assertEquals(3, categories.size()); + } +} diff --git a/indexer/src/test/resources/canned/aodn_discovery_parameter_vocabs.json b/indexer/src/test/resources/canned/aodn_discovery_parameter_vocabs.json new file mode 100644 index 00000000..c9d8718d --- /dev/null +++ b/indexer/src/test/resources/canned/aodn_discovery_parameter_vocabs.json @@ -0,0 +1,1400 @@ +[ + { + "label": "Air pressure", + "definition": "This category contains vocabulary terms describing air pressure parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/10", + "broader": [ + { + "label": "Physical-Atmosphere", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/53" + } + ], + "narrower": [] + }, + { + "label": "Oxygen", + "definition": "This category contains vocabulary terms describing dissolved oxygen parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/11", + "broader": [ + { + "label": "Chemical", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/54" + } + ], + "narrower": [] + }, + { + "label": "Chlorophyll", + "definition": "This category contains vocabulary terms describing chlorophyll concentration parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/19", + "broader": [ + { + "label": "Biological", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/55" + } + ], + "narrower": [] + }, + { + "label": "Wave", + "definition": "This category contains vocabulary terms describing waves parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/2", + "broader": [ + { + "label": "Physical-Water", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/56" + } + ], + "narrower": [] + }, + { + "label": "Visibility", + "definition": "This category contains vocabulary terms describing atmospheric visibility parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/22", + "broader": [ + { + "label": "Physical-Atmosphere", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/53" + } + ], + "narrower": [] + }, + { + "label": "Density", + "definition": "This category contains vocabulary terms describing density parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/23", + "broader": [ + { + "label": "Physical-Water", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/56" + } + ], + "narrower": [] + }, + { + "label": "Water pressure", + "definition": "This category contains vocabulary terms describing water pressure parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/25", + "broader": [ + { + "label": "Physical-Water", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/56" + } + ], + "narrower": [] + }, + { + "label": "Other Pigment", + "definition": "This category contains vocabulary terms describing pigments concentration parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/26", + "broader": [ + { + "label": "Biological", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/55" + } + ], + "narrower": [] + }, + { + "label": "Alkalinity", + "definition": "This category contains vocabulary terms describing alkalinity parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/27", + "broader": [ + { + "label": "Chemical", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/54" + } + ], + "narrower": [] + }, + { + "label": "Turbidity", + "definition": "This category contains vocabulary terms describing turbidity parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/28", + "broader": [ + { + "label": "Physical-Water", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/56" + } + ], + "narrower": [] + }, + { + "label": "Current", + "definition": "This category contains vocabulary terms describing currents parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/3", + "broader": [ + { + "label": "Physical-Water", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/56" + } + ], + "narrower": [] + }, + { + "label": "Air-Sea Fluxes", + "definition": "This category contains vocabulary terms describing Air-Sea Fluxes parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/30", + "broader": [ + { + "label": "Physical-Atmosphere", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/53" + }, + { + "label": "Physical-Water", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/56" + } + ], + "narrower": [] + }, + { + "label": "Ocean Biota", + "definition": "This category contains vocabulary terms describing ocean biota parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/4", + "broader": [ + { + "label": "Biological", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/55" + } + ], + "narrower": [] + }, + { + "label": "Optical properties", + "definition": "This category contains vocabulary terms describing optical properties parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/45", + "broader": [ + { + "label": "Physical-Water", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/56" + } + ], + "narrower": [] + }, + { + "label": "Suspended particulate material", + "definition": "This category contains vocabulary terms describing suspended particulate material parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/46", + "broader": [ + { + "label": "Chemical", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/54" + }, + { + "label": "Biological", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/55" + } + ], + "narrower": [] + }, + { + "label": "Backscattering", + "definition": "This category contains vocabulary terms describing backscattering parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/47", + "broader": [ + { + "label": "Physical-Water", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/56" + } + ], + "narrower": [] + }, + { + "label": "UV radiation", + "definition": "This category contains vocabulary terms describing UV radiation parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/48", + "broader": [ + { + "label": "Physical-Atmosphere", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/53" + } + ], + "narrower": [] + }, + { + "label": "Temperature", + "definition": "This category contains vocabulary terms describing temperature parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/49", + "broader": [ + { + "label": "Physical-Water", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/56" + } + ], + "narrower": [] + }, + { + "label": "Acoustics", + "definition": "This category contains vocabulary terms describing acoustics parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/5", + "broader": [ + { + "label": "Physical-Water", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/56" + } + ], + "narrower": [] + }, + { + "label": "Salinity", + "definition": "This category contains vocabulary terms describing salinity parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/50", + "broader": [ + { + "label": "Physical-Water", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/56" + } + ], + "narrower": [] + }, + { + "label": "Nutrient", + "definition": "This category contains vocabulary terms describing nutrient parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/51", + "broader": [], + "narrower": [] + }, + { + "label": "Carbon", + "definition": "This category contains vocabulary terms describing carbon parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/52", + "broader": [], + "narrower": [] + }, + { + "label": "Physical-Atmosphere", + "definition": "This category contains vocabulary terms describing physical atmosphere parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/53", + "broader": [], + "narrower": [ + { + "label": "Air pressure", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/10", + "narrower": [ + { + "label": "Pressure (measured variable) exerted by the atmosphere and correction to sea level", + "definition": "Measurement as a phenomenon (as opposed to a co-ordinate) of the force per unit area exerted by the atmosphere determined in-situ at a known altitude and converted to the value at sea level assuming an isothermal layer.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/77" + }, + { + "label": "Pressure (measured variable) exerted by the atmosphere", + "definition": "Measurement as a phenomenon (as opposed to a co-ordinate) of the force per unit area exerted by the atmosphere determined in-situ at a known altitude.", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/CAPHZZ01" + } + ] + }, + { + "label": "Visibility", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/22", + "narrower": [ + { + "label": "Horizontal visibility in the atmosphere", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/390" + }, + { + "label": "Present weather in the atmosphere", + "definition": "Visual observation using WMO code table 4677 or 4501", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/397" + } + ] + }, + { + "label": "Air-Sea Fluxes", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/30", + "narrower": [ + { + "label": "Net shortwave heat flux", + "definition": "The net shortwave heat flux is the difference between the downwelling radiation from the sun as modified by the gases and clouds of the atmosphere and the radiation returned from the sea surface, which is either reflected by the surface or backscattered from within the water column.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/10" + }, + { + "label": "Net heat flux", + "definition": "Net surface heat flux across the air-sea interface.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/11" + }, + { + "label": "Net mass flux", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/12" + }, + { + "label": "Downwelling vector irradiance as photons (PAR wavelengths) in the atmosphere", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/572" + }, + { + "label": "Downwelling vector irradiance as energy (solar (300-3000nm) wavelengths) in the atmosphere", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/576" + }, + { + "label": "Sensible heat flux", + "definition": "Heat exchange due to difference in temperature between air and water.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/6" + }, + { + "label": "Downwelling vector irradiance as energy in the atmosphere", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/651" + }, + { + "label": "Latent heat flux", + "definition": "Heat loss due to evaporation", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/7" + }, + { + "label": "Sensible heat flux due to precipitation", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/8" + }, + { + "label": "Net longwave heat flux", + "definition": "The net longwave heat flux is the net flux of the greybody emissions from the sea surface, cloud layers and the gases of the atmosphere.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/9" + }, + { + "label": "Downwelling vector irradiance as energy (longwave) in the atmosphere", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/LWRDZZ01" + } + ] + }, + { + "label": "UV radiation", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/48", + "narrower": [ + { + "label": "Downwelling vector irradiance as energy (ultra-violet wavelengths) in the atmosphere", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/733" + } + ] + }, + { + "label": "Humidity", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/8", + "narrower": [ + { + "label": "Dew point temperature of the atmosphere", + "definition": "The temperature to which air must cool to become saturated with water vapour", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/CDEWZZ01" + }, + { + "label": "Specific humidity of the atmosphere", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/CHUMSS01" + }, + { + "label": "Relative humidity of the atmosphere", + "definition": "The ratio of the amount of water vapour in the air compared to the maximum amount of water vapour that can theoretically be held at the air's temperature", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/CRELZZ01" + }, + { + "label": "Wet bulb temperature of the atmosphere", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/CWETZZ01" + } + ] + }, + { + "label": "Precipitation and evaporation", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/9", + "narrower": [ + { + "label": "Thickness of precipitation amount (liquid water equivalent) in the atmosphere", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/386" + }, + { + "label": "Precipitation duration (liquid water equivalent) in the atmosphere", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/405" + }, + { + "label": "Hail duration in the atmosphere", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/637" + }, + { + "label": "Thickness of hail amount in the atmosphere", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/638" + }, + { + "label": "Hail intensity in the atmosphere", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/639" + }, + { + "label": "Precipitation rate (liquid water equivalent) in the atmosphere", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/94" + } + ] + } + ] + }, + { + "label": "Chemical", + "definition": "This category contains vocabulary terms describing chemicalparameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/54", + "broader": [], + "narrower": [ + { + "label": "Oxygen", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/11", + "narrower": [ + { + "label": "Concentration of oxygen {O2} per unit volume of the water body", + "definition": "Concentration of dissolved oxygen per unit volume of the water column. Oxygen may be expressed in terms of mass, volume or quantity of substance.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/372" + }, + { + "label": "Concentration of oxygen {O2} per unit mass of the water body", + "definition": "Concentration of dissolved oxygen per unit mass of the water column. Oxygen may be expressed in terms of mass, volume or quantity of substance.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/373" + }, + { + "label": "Saturation of oxygen {O2} in the water body [dissolved phase]", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/84" + } + ] + }, + { + "label": "Alkalinity", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/27", + "narrower": [ + { + "label": "Total alkalinity per unit mass of the water body", + "definition": "Concentration (moles or mass) of acid-neutralising bases per unit mass of the water column", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/MDMAP014" + }, + { + "label": "Concentration of carbonate ions per unit mass of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/26" + } + ] + }, + { + "label": "Suspended particulate material", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/46", + "narrower": [ + { + "label": "Concentration of suspended particulate material per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/654" + }, + { + "label": "Concentration of suspended particulate material (organic) per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/655" + }, + { + "label": "Concentration of suspended particulate material (inorganic) per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/656" + } + ] + } + ] + }, + { + "label": "Biological", + "definition": "This category contains vocabulary terms describing biologicalparameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/55", + "broader": [], + "narrower": [ + { + "label": "Chlorophyll", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/19", + "narrower": [ + { + "label": "Concentration of inferred chlorophyll from relative fluorescence per unit mass of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/893" + }, + { + "label": "Concentration of inferred chlorophyll from relative fluorescence per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/894" + }, + { + "label": "Concentration of total chlorophyll-a per unit volume of the water body", + "definition": "The amount (mass or moles) of the following pigments (divinyl chlorophyll-a, chlorophyll-a, chlorophyllide-a, chlorophyll-a allomers and chlorophyll-a prime) in a known volume of any water body.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/13" + }, + { + "label": "Concentration of total chlorophyll-c per unit volume of the water body", + "definition": "The amount (mass or moles) of the following pigments (chlorophyll-c1, chlorophyll-c2, chlorophyll-c1c2, chlorophyll-c3) in a known volume of any water body.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/14" + }, + { + "label": "Concentration of total chlorophyll-a plus chlorophyll-b plus chlorophyll-c per unit volume of the water body", + "definition": "The amount (mass or moles) of total chlorophyll-a (divinyl chlorophyll-a, chlorophyll-a, chlorophyllide-a, chlorophyll-a allomers and chlorophyll-a prime) plus total chlorophyll-b (divinyl chlorophyll-b, chlorophyll-b) plus total chlorophyll-c (chlorophyll-c1, chlorophyll-c2, chlorophyll-c1c2, chlorophyll-c3) in a known volume of any water body.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/15" + }, + { + "label": "Concentration of chlorophyll-b per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/745" + }, + { + "label": "Concentration of divinyl chlorophyll-b per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/746" + }, + { + "label": "Concentration of chlorophyll-a per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/750" + }, + { + "label": "Concentration of chlorophyll-a plus divinyl chlorophyll-a per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/752" + }, + { + "label": "Concentration of chlorophyll-b plus divinyl chlorophyll-b per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/753" + }, + { + "label": "Concentration of divinyl chlorophyll-a per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/754" + }, + { + "label": "Concentration of chlorophyllide-a per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/757" + }, + { + "label": "Concentration of chlorophyll-c2 per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/758" + }, + { + "label": "Concentration of chlorophyll-c1 per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/759" + }, + { + "label": "Concentration of chlorophyll-c1c2 per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/760" + }, + { + "label": "Concentration of chlorophyll-c3 per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/761" + }, + { + "label": "Concentration of chlorophyll per unit mass of the water body", + "definition": "The mass of all types of chlorophyll per unit mass of the water column held in particles of any size", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/CHLTMASS" + }, + { + "label": "Concentration of chlorophyll per unit volume of the water body", + "definition": "The mass of all types of chlorophyll per unit volume of the water column held in particles of any size", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/CHLTVOLU" + }, + { + "label": "Fluorescence of the water body", + "definition": "The amount of radiation generated in the water column in response to higher energy radiation transmission expressed relative to an unspecified baseline", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/FLUOZZZZ" + }, + { + "label": "Concentration of total chlorophyll-b per unit volume of the water body", + "definition": "The amount (mass or moles) of the following pigments (divinyl chlorophyll-b and chlorophyll-b) in a known volume of any water body.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/52" + } + ] + }, + { + "label": "Other Pigment", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/26", + "narrower": [ + { + "label": "Light absorption coefficient of dissolved organic matter", + "definition": "The proportion of the incident light that is absorbed by the dissolved component of a sample from any body of fresh or salt water.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/890" + }, + { + "label": "Light absorption coefficient of phytoplankton", + "definition": "The proportion of the incident light that is absorbed by the algal (phytoplankton) component of a sample from any body of fresh or salt water.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/891" + }, + { + "label": "Light absorption coefficient of nonalgal particles", + "definition": "The proportion of the incident light that is absorbed by the detrital or non-algal component of a sample from any body of fresh or salt water.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/892" + }, + { + "label": "Concentration of photoprotective carotenoids per unit volume of the water body", + "definition": "The amount (mass or moles) of the following pigments (alloxanthin, diadinoxanthin, diatoxanthin, Zeaxanthin, alpha carotene, beta carotene) in a known volume of any water body.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/16" + }, + { + "label": "Concentration of photosynthetic carotenoids per unit volume of the water body", + "definition": "The amount (mass or moles) of the following pigments (19-butanoyloxyfucoxanthin, fucoxanthin, 19-hexanoyloxyfucoxanthin, peridinin) in a known volume of any water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/17" + }, + { + "label": "Concentration of photosynthetic pigments per unit volume of the water body", + "definition": "The amount (mass or moles) of photosynthetic carotenoids (19-butanoyloxyfucoxanthin, fucoxanthin, 19-hexanoyloxyfucoxanthin, peridinin) plus total chlorophyll-a pigments (divinyl chlorophyll-a, chlorophyll-a, chlorophyllide-a, chlorophyll-a allomers and chlorophyll-a prime) plus total chlorophyll-b pigments (divinyl chlorophyll-b, chlorophyll-b) plus total chlorophyll-c pigments (chlorophyll-c1, chlorophyll-c2, chlorophyll-c1c2, chlorophyll-c3) in a known volume of any water body.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/18" + }, + { + "label": "Concentration of total accessory pigments per unit volume of the water body", + "definition": "The amount (mass or moles) of photosynthetic carotenoids (19-butanoyloxyfucoxanthin, fucoxanthin, 19-hexanoyloxyfucoxanthin, peridinin) plus photoprotective carotenoids (alloxanthin, diadinoxanthin, diatoxanthin, Zeaxanthin, alpha carotene, beta carotene) plus total chlorophyll-b pigments (divinyl chlorophyll-b, chlorophyll-b) plus total chlorophyll-c pigments (chlorophyll-c1, chlorophyll-c2, chlorophyll-c1c2, chlorophyll-c3) in a known volume of any water body.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/19" + }, + { + "label": "Concentration of total pigment per unit volume of the water body", + "definition": "The amount (mass or moles) of all pigments in a known volume of any water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/20" + }, + { + "label": "Concentration of total diagnostic pigments per unit volume of the water body", + "definition": "The amount (mass or moles) of photosynthetic carotenoids (19-butanoyloxyfucoxanthin, fucoxanthin, 19-hexanoyloxyfucoxanthin, peridinin) plus total chlorophyll-b pigments (divinyl chlorophyll-b, chlorophyll-b) plus alloxanthin and zeaxanthin in a known volume of any water body.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/21" + }, + { + "label": "Concentration of echinenone per unit mass of the water body", + "definition": "The amount (mass or moles) of the specified pigment determined by HPLC assay of a sample collected by dissolution in acteone of the residue collected by GF/F filtration of a known volume of any water body. The quoted value either results from a single determination or the average of replicate determinations", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/478" + }, + { + "label": "Concentration of antheraxanthin per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/658" + }, + { + "label": "Concentration of canthaxanthin per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/659" + }, + { + "label": "Concentration of lutein per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/660" + }, + { + "label": "Concentration of neoxanthin per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/734" + }, + { + "label": "Concentration of prasinoxanthin per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/735" + }, + { + "label": "Concentration of violaxanthin per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/736" + }, + { + "label": "Concentration of astaxanthin per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/737" + }, + { + "label": "Concentration of dinoxanthin per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/738" + }, + { + "label": "Concentration of diatoxanthin per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/739" + }, + { + "label": "Concentration of zeaxanthin per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/740" + }, + { + "label": "Concentration of lycopene per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/741" + }, + { + "label": "Concentration of diadinoxanthin per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/742" + }, + { + "label": "Concentration of gyroxanthin diester per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/743" + }, + { + "label": "Concentration of diadinochrome per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/744" + }, + { + "label": "Concentration of 19-hexanoyloxyfucoxanthin per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/747" + }, + { + "label": "Concentration of pheophytin-a per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/748" + }, + { + "label": "Concentration of pheophytin-b per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/749" + }, + { + "label": "Concentration of pyropheophytin-a per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/751" + }, + { + "label": "Concentration of pyropheophorbide-a per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/755" + }, + { + "label": "Concentration of pheophorbide-a per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/756" + }, + { + "label": "Concentration of 19-butanoyloxyfucoxanthin per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/762" + }, + { + "label": "Concentration of 4-keto-19-hexanoyloxyfucoxanthin per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/763" + }, + { + "label": "Concentration of alpha-carotene per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/764" + }, + { + "label": "Concentration of beta-carotene per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/765" + }, + { + "label": "Concentration of alpha-carotene and beta-carotene per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/766" + }, + { + "label": "Concentration of Mg-2,4-divinyl pheoporphyrin a5 monomethyl ester per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/767" + }, + { + "label": "Concentration of alloxanthin per unit volume of the water body", + "definition": "Concentration of the carotenoid pigment alloxanthin per unit volume of a water body. The pigment is contained in suspended particles of unspecified size", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/ALLOXXPX" + }, + { + "label": "Concentration of fucoxanthin per unit volume of the water body", + "definition": "Concentration of the carotenoid pigment fucoxanthin per unit volume of a water body. The pigment is contained in suspended particles of unspecified size", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/FUCXZZZZ" + }, + { + "label": "Concentration of peridinin per unit volume of the water body", + "definition": "Concentration of the carotenoid pigment peridinin per unit volume of a water body. The pigment is contained in suspended particles of unspecified size", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/PERDXXXX" + }, + { + "label": "Concentration of phycoerythrin per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/51" + } + ] + }, + { + "label": "Ocean Biota", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/4", + "narrower": [ + { + "label": "Biovolume", + "definition": "The volume of a biological object described elsewhere in the metadata (e.g.: single-species populations, all species within a phytoplankton sample …) occurring in a given volume of water.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/22" + }, + { + "label": "Mean unit biovolume", + "definition": "The average volume of individual organisms (cells or colonies).", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/23" + }, + { + "label": "Phytoplankton Colour Index", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/407" + }, + { + "label": "Abundance of biota", + "definition": "The relative representation of flora and fauna in a particular sample or region.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/488" + }, + { + "label": "Biotic taxonomic identification", + "definition": "The identification of the biotic elements to the relevant scientific classification group (taxa), e.g. genus or species.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/489" + }, + { + "label": "Dry weight biomass per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/570" + }, + { + "label": "Wet weight biomass per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/571" + }, + { + "label": "Detection of acoustic tag", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/596" + } + ] + }, + { + "label": "Suspended particulate material", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/46", + "narrower": [ + { + "label": "Concentration of suspended particulate material per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/654" + }, + { + "label": "Concentration of suspended particulate material (organic) per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/655" + }, + { + "label": "Concentration of suspended particulate material (inorganic) per unit volume of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/656" + } + ] + } + ] + }, + { + "label": "Physical-Water", + "definition": "This category contains vocabulary terms describing physical water parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/56", + "broader": [], + "narrower": [ + { + "label": "Bathymetry", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/1", + "narrower": [ + { + "label": "Sea-floor depth below surface of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/574" + }, + { + "label": "Height above surface of the water body", + "definition": "Surface references mean sea level, which means the time mean of sea surface elevation at a given location over an arbitrary period sufficient to eliminate the tidal signals.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/44" + } + ] + }, + { + "label": "Wave", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/2", + "narrower": [ + { + "label": "Spectral significant height of waves on the water body", + "definition": "Significant wave height approximated as four times the square root of the first moment of the wave power spectrum.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/2" + }, + { + "label": "Period at the peak spectral energy of waves on the water body", + "definition": "Period of the peak of the energy spectrum analysed by spectral method.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/3" + }, + { + "label": "Direction at spectral maximum of waves on the water body", + "definition": "Direction (related to true north) from which the peak period waves are coming from analysed by spectral method.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/4" + }, + { + "label": "Significant height of waves on the water body", + "definition": "The average height of the highest one third of the waves, where the height is defined as the vertical distance from a wave trough to the following wave crest.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/411" + }, + { + "label": "Average height of waves on the water body", + "definition": "The mean height of waves on the water body analysed by zero crossing method.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/412" + }, + { + "label": "Root mean square height of waves on the water body", + "definition": "The root mean square wave height analysed by zero crossing method.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/413" + }, + { + "label": "Average height of the highest 1/10th of waves on the water body", + "definition": "The mean of the highest ten per cent of trough to crest distances measured during the observation period.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/414" + }, + { + "label": "Average zero crossing period of the highest 1/3rd of waves on the water body", + "definition": "The average period of the highest 1/3 of waves measured during a recording burst", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/417" + }, + { + "label": "Maximum height of waves on the water body", + "definition": "The maximum vertical distance between a wave crest and the immediately preceding or following through during a specified observation period.", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/GCMXZZ01" + }, + { + "label": "Directional spreading of waves on the water body", + "definition": "Sea surface wave mean directional spread analysed by spectral method.", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/GSPRZZ01" + }, + { + "label": "Period of swell waves on the water body", + "definition": "Period between successive swell maxima.", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/GSZZXXXX" + }, + { + "label": "Average crest period of waves on the water body", + "definition": "Sea surface wave mean crest period analysed by zero crossing method.", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/GTCAZZ01" + }, + { + "label": "Direction of waves on the water body", + "definition": "Direction from which waves are coming relative to True North.", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/GWDRZZ01" + }, + { + "label": "Significant height of swell waves on the water body", + "definition": "The average height of the highest one third of the swell waves, where the height is defined as the vertical distance from a wave trough to the following wave crest.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/28" + }, + { + "label": "Period at spectral maximum of swell waves first component on the water body", + "definition": "Period of the highest swell wave peak analysed by spectral method.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/29" + }, + { + "label": "Direction at spectral maximum of swell waves first component on the water body", + "definition": "Direction from which the most energetic waves are coming in the swell wave component of a sea.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/30" + }, + { + "label": "Significant height of wind sea waves on the water body", + "definition": "The average height of the highest one third of the wind sea waves, where the height is defined as the vertical distance from a wave trough to the following wave crest.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/31" + }, + { + "label": "Period at spectral maximum of wind sea waves on the water body", + "definition": "Period of the highest wind sea wave peak analysed by spectral method.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/32" + }, + { + "label": "Direction at spectral maximum of wind sea waves on the water body", + "definition": "Direction (related to true north) from which the peak period wind sea waves are coming from analysed by spectral method.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/33" + }, + { + "label": "Maximum zero crossing period of waves on the water body", + "definition": "The maximum period of the waves analysed by zero crossing method.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/34" + }, + { + "label": "Average height of the highest 1/3rd of waves on the water body", + "definition": "The mean of the highest third of waves on the water body analysed by zero crossing method.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/35" + }, + { + "label": "Average zero crossing period of waves on the water body", + "definition": "The period of the significant wave analysed by zero crossing method.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/36" + }, + { + "label": "Average zero crossing period of the highest 1/10th waves on the water body", + "definition": "The average period of the highest tenth of waves measured during a recording burst.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/37" + }, + { + "label": "Variance spectral density of waves on the water body", + "definition": "Fourier Analysis on directional sea level elevation, also named 'Directional Spectral density' or 'Directional Power spectral density'.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/38" + }, + { + "label": "Directional variance spectral density of waves on the water body", + "definition": "Sea surface wave directional variance spectral density is the variance of the amplitude of the waves within given ranges of direction and wave frequency.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/39" + }, + { + "label": "Period at second highest energy spectrum peak of waves on the water body", + "definition": "Period of the second highest peak of the energy spectrum analysed by spectral method.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/40" + }, + { + "label": "Zeroth spectral moment of waves on the water body", + "definition": "The variance spectral density of the zeroth frequency moment.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/41" + }, + { + "label": "Direction of wind sea waves on the water body", + "definition": "Direction from which wind sea waves are coming relative to True North.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/42" + }, + { + "label": "Spectral directional spread of wave peak on the water body", + "definition": "Sea surface wave peak directional spread analysed by spectral method.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/61" + }, + { + "label": "Period of the highest wave on the water body", + "definition": "Period corresponding to the wave with maximum height in the record analysed by zero crossing method.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/60" + }, + { + "label": "Mean period at spectral second frequency moment of waves on the water body", + "definition": "The square root of ratio of the zeroth and second-order moment of the non-directional wave spectrum analysed by spectral method.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/62" + }, + { + "label": "Mean period at spectral first frequency moment of waves on the water body", + "definition": "The square root of ratio of the zeroth and first-order moment of the non-directional wave spectrum analysed by spectral method.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/63" + }, + { + "label": "Average direction at spectral maximum of waves on the water body", + "definition": "Average direction (related to true north) from which the peak period waves are coming from analysed by spectral method.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/64" + }, + { + "label": "Direction at spectral maximum of swell waves on the water body", + "definition": "Direction (related to true north) towards which the peak period swell waves are heading, analysed by spectral method.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/65" + }, + { + "label": "Wavelength at spectral maximum of swell waves on the water body", + "definition": "The wavelength of the peak period swell waves analysed by spectral method. The wavelength is the horizontal distance between repeated features on the waveform such as crests, troughs or upward passes through the mean level.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/66" + } + ] + }, + { + "label": "Density", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/23", + "narrower": [ + { + "label": "Density of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/401" + } + ] + }, + { + "label": "Water pressure", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/25", + "narrower": [ + { + "label": "Pressure (measured variable) in the water body exerted by overlying sea water and any medium above it", + "definition": "The force per unit area exerted by the overlying sea water, sea ice, air and any other medium that may be present on a sensor located in the water column.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/565" + }, + { + "label": "Pressure (measured variable) in the water body exerted by overlying sea water only", + "definition": "The force per unit area exerted by the overlying sea water on a sensor located in the water column. Excludes the pressure due to sea ice, air and any other medium that may be present.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/566" + }, + { + "label": "Pressure (measured variable) exerted by the atmosphere", + "definition": "Measurement as a phenomenon (as opposed to a co-ordinate) of the force per unit area exerted by the atmosphere determined in-situ at a known altitude.", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/CAPHZZ01" + } + ] + }, + { + "label": "Turbidity", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/28", + "narrower": [ + { + "label": "Turbidity of the water body", + "definition": "Estimate of suspended sediment concentration based on the proportion of a light transmission in the water column that is reflected back to a co-located receiver.", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/TURBXXXX" + } + ] + }, + { + "label": "Current", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/3", + "narrower": [ + { + "label": "Current speed in the water body", + "definition": "The speed of Eulerian flow in the water column", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/383" + }, + { + "label": "Eastward geostrophic current velocity in the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/640" + }, + { + "label": "Northward geostrophic current velocity in the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/641" + }, + { + "label": "Current direction in the water body", + "definition": "The direction towards which Eulerian current in the water column is flowing.", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/LCDAZZ01" + }, + { + "label": "Eastward current velocity in the water body", + "definition": "Speed of Eulerian current flow towards due east.", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/LCEWZZ01" + }, + { + "label": "Northward current velocity in the water body", + "definition": "Speed of Eulerian current flow towards true north", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/LCNSZZ01" + }, + { + "label": "Upward current velocity in the water body", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/LRZAZZZZ" + } + ] + }, + { + "label": "Air-Sea Fluxes", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/30", + "narrower": [ + { + "label": "Net shortwave heat flux", + "definition": "The net shortwave heat flux is the difference between the downwelling radiation from the sun as modified by the gases and clouds of the atmosphere and the radiation returned from the sea surface, which is either reflected by the surface or backscattered from within the water column.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/10" + }, + { + "label": "Net heat flux", + "definition": "Net surface heat flux across the air-sea interface.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/11" + }, + { + "label": "Net mass flux", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/12" + }, + { + "label": "Downwelling vector irradiance as photons (PAR wavelengths) in the atmosphere", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/572" + }, + { + "label": "Downwelling vector irradiance as energy (solar (300-3000nm) wavelengths) in the atmosphere", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/576" + }, + { + "label": "Sensible heat flux", + "definition": "Heat exchange due to difference in temperature between air and water.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/6" + }, + { + "label": "Downwelling vector irradiance as energy in the atmosphere", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/651" + }, + { + "label": "Latent heat flux", + "definition": "Heat loss due to evaporation", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/7" + }, + { + "label": "Sensible heat flux due to precipitation", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/8" + }, + { + "label": "Net longwave heat flux", + "definition": "The net longwave heat flux is the net flux of the greybody emissions from the sea surface, cloud layers and the gases of the atmosphere.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/9" + }, + { + "label": "Downwelling vector irradiance as energy (longwave) in the atmosphere", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/LWRDZZ01" + } + ] + }, + { + "label": "Optical properties", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/45", + "narrower": [ + { + "label": "Light absorption coefficient of dissolved organic matter", + "definition": "The proportion of the incident light that is absorbed by the dissolved component of a sample from any body of fresh or salt water.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/890" + }, + { + "label": "Light absorption coefficient of phytoplankton", + "definition": "The proportion of the incident light that is absorbed by the algal (phytoplankton) component of a sample from any body of fresh or salt water.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/891" + }, + { + "label": "Light absorption coefficient of nonalgal particles", + "definition": "The proportion of the incident light that is absorbed by the detrital or non-algal component of a sample from any body of fresh or salt water.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/892" + }, + { + "label": "Light extinction coefficient (490nm wavelength) in the water body", + "definition": "Light extinction coefficient (490nm wavelength) in the water body by radiometer and computation from variation of light intensity with depth.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/5" + }, + { + "label": "Downwelling vector irradiance as photons (PAR wavelengths) in the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/573" + }, + { + "label": "Colored Dissolved Organic Matter", + "definition": "The amount of colored organic compounds (sometimes called yellow substance or gelbstoff) dissolved in a specified amount of water from any body of fresh or salt water", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/629" + }, + { + "label": "Water-leaving radiance from the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/646" + }, + { + "label": "Water-leaving radiance from the water body corrected for viewing angle dependance", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/647" + }, + { + "label": "Normalised water-leaving radiance from the water body determined from Lw", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/648" + }, + { + "label": "Normalised water-leaving radiance from the water body determined from Lw_Q", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/649" + }, + { + "label": "Downwelling vector irradiance as energy in the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/650" + }, + { + "label": "Sky radiance as energy in the atmosphere", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/652" + }, + { + "label": "Upwelling radiance as energy in the atmosphere", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/653" + }, + { + "label": "Secchi depth", + "definition": "Estimate of the transparency of the surface water based upon the distance below the ship at which a standard black and white disk becomes visible.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/729" + }, + { + "label": "Total absorption coefficient in the water body", + "definition": "Inherent optical property of natural waters. Describes the intensity loss of a light beam through a unit volume of water due to absorption.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/59" + }, + { + "label": "Total attenuation coefficient in the water body", + "definition": "Inherent optical property of natural waters. Describes the intensity loss of a light beam through a unit volume of water due to scattering and absorption.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/58" + }, + { + "label": "Total backscattering coefficient in the water body", + "definition": "Inherent optical property of natural waters. Describes the reflection of light from a unit volume of water back to the incident direction. Integral of the volume scattering function in the backward direction from 90 to 180 degrees.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/54" + }, + { + "label": "Particle backscattering coefficient in the water body", + "definition": "Inherent optical property of natural waters. Describes the reflection of light by particles from a unit volume of water back to the incident direction. Integral of the particle volume scattering function in the backward direction from 90 to 180 degrees.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/55" + }, + { + "label": "Total volume scattering function in the water body", + "definition": "Inherent optical property of natural waters. Describes the angular dependence of scattered light from an incident light beam.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/56" + }, + { + "label": "Particle volume scattering function in the water body", + "definition": "Inherent optical property of natural waters. Describes the angular dependence of scattered light by particles from an incident light beam.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/57" + } + ] + }, + { + "label": "Backscattering", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/47" + }, + { + "label": "Temperature", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/49", + "narrower": [ + { + "label": "Temperature anomaly of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/590" + }, + { + "label": "Ocean heat content", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/631" + }, + { + "label": "Skin temperature of the water body", + "definition": "Skin temperature of the water body measure by Advanced Very High Resolution Radiometer (AVHRR)", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/97" + }, + { + "label": "Temperature of the water body", + "definition": "The degree of hotness of the water column expressed against a standard scale. Includes both IPTS-68 and ITS-90 scales.", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/TEMPPR01" + } + ] + }, + { + "label": "Acoustics", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/5", + "narrower": [ + { + "label": "Acoustic signal return amplitude from the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/399" + }, + { + "label": "Sound recorded in the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/597" + }, + { + "label": "Volume backscattering coefficient", + "definition": "Echosounder provides two basic measurements: (1) the time delay between transmission of a sound pulse and reception of a return echo from an acoustic target (for example fish), and (2) the intensity of the returning echo. The intensity of returning echo from the fish is dependent on the intensity of sound transmitted by the echosounder, the loss of intensity as the sound wave travels through water (both to the fish and back to the echosounder), and the backscattering cross-section (or target strength) of the fish. Backscattering cross-section determines what proportion of incident sound intensity is reflected from the fish, back to the echosounder. The derived parameter ‘volume backscattering coefficient’ is the summation of backscattering cross-section from all acoustic targets present within the sampling volume scaled to 1 m3.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/728" + }, + { + "label": "Sound velocity in the water body", + "definition": "The rate at which sound travels through the water column", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/SVELXXXX" + } + ] + }, + { + "label": "Salinity", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/50", + "narrower": [ + { + "label": "Practical salinity anomaly of the water body", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/591" + }, + { + "label": "Specific electrical conductivity of the water body", + "definition": "Conductance is a temperature corrected value, and approximates what the AC (Actual Conductivity) of a solution would be at 25 degrees Celsisus.", + "about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/730" + }, + { + "label": "Electrical conductivity of the water body", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/CNDCZZ01" + }, + { + "label": "Practical salinity of the water body", + "definition": "The quantity of dissolved ions (predominantly salt in seawater) expressed on a scale (PSS-78) based on the conductivity ratio of a seawater sample to a standard KCl solution.", + "about": "http://vocab.nerc.ac.uk/collection/P01/current/PSLTZZ01" + } + ] + } + ] + }, + { + "label": "Sea surface height", + "definition": "This category contains vocabulary terms describing sea surface height parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/57", + "broader": [], + "narrower": [] + }, + { + "label": "Depth", + "definition": "This category contains vocabulary terms describing depth parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/58", + "broader": [], + "narrower": [] + }, + { + "label": "Wind", + "definition": "This category contains vocabulary terms describing wind parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/6", + "broader": [], + "narrower": [] + }, + { + "label": "Air temperature", + "definition": "This category contains vocabulary terms describing air temperature parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/7", + "broader": [], + "narrower": [] + }, + { + "label": "Humidity", + "definition": "This category contains vocabulary terms describing atmospheric humidity parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/8", + "broader": [ + { + "label": "Physical-Atmosphere", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/53" + } + ], + "narrower": [] + }, + { + "label": "Precipitation and evaporation", + "definition": "This category contains vocabulary terms describing precipitation and evaporation parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/9", + "broader": [ + { + "label": "Physical-Atmosphere", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/53" + } + ], + "narrower": [] + }, + { + "label": "Bathymetry", + "definition": "This category contains vocabulary terms describing bathymetry parameters", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/1", + "broader": [ + { + "label": "Physical-Water", + "about": "http://vocab.aodn.org.au/def/parameter_classes/category/56" + } + ], + "narrower": [] + } +] diff --git a/stacmodel/src/main/java/au/org/aodn/stac/model/ConceptModel.java b/stacmodel/src/main/java/au/org/aodn/stac/model/ConceptModel.java index e63109ce..8fd6afda 100644 --- a/stacmodel/src/main/java/au/org/aodn/stac/model/ConceptModel.java +++ b/stacmodel/src/main/java/au/org/aodn/stac/model/ConceptModel.java @@ -11,6 +11,12 @@ public class ConceptModel { private String id; private String url; + + public ConceptModel(String id, String url) { + this.id = id; + this.url = url; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/stacmodel/src/main/java/au/org/aodn/stac/model/ThemesModel.java b/stacmodel/src/main/java/au/org/aodn/stac/model/ThemesModel.java index 50189f43..8738d716 100644 --- a/stacmodel/src/main/java/au/org/aodn/stac/model/ThemesModel.java +++ b/stacmodel/src/main/java/au/org/aodn/stac/model/ThemesModel.java @@ -13,4 +13,11 @@ public class ThemesModel { protected String scheme; protected String description; protected String title; + + public ThemesModel(List concepts, String scheme, String description, String title) { + this.concepts = concepts; + this.scheme = scheme; + this.description = description; + this.title = title; + } } From 99bb29722b27a929acbb426f81f6b185dfab5972 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Date: Wed, 17 Apr 2024 17:07:02 +1000 Subject: [PATCH 05/19] Update tests --- .../org/aodn/esindexer/utils/StringUtil.java | 7 - .../service/IndexerServiceTests.java | 8 +- .../AodnDiscoveryParameterVocabUtilsTest.java | 25 +- .../test/resources/canned/sample3_stac.json | 342 ------------------ .../test/resources/canned/sample4_stac.json | 253 +++++++++++++ .../au/org/aodn/stac/model/ThemesModel.java | 1 - 6 files changed, 270 insertions(+), 366 deletions(-) delete mode 100644 indexer/src/test/resources/canned/sample3_stac.json create mode 100644 indexer/src/test/resources/canned/sample4_stac.json diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/StringUtil.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/StringUtil.java index 5d1899fe..044541aa 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/StringUtil.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/StringUtil.java @@ -1,13 +1,6 @@ package au.org.aodn.esindexer.utils; -import au.org.aodn.esindexer.configuration.AppConstants; - import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; public class StringUtil { // Static method to convert to UTF-8 String diff --git a/indexer/src/test/java/au/org/aodn/esindexer/service/IndexerServiceTests.java b/indexer/src/test/java/au/org/aodn/esindexer/service/IndexerServiceTests.java index f4ea5dfc..87fbc607 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/service/IndexerServiceTests.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/service/IndexerServiceTests.java @@ -102,17 +102,17 @@ public void verifyDeleteDocumentByUUID() throws IOException { @Test public void verifyGetDocumentByUUID() throws IOException { - String expected = readResourceFile("classpath:canned/sample3_stac.json"); + String expected = readResourceFile("classpath:canned/sample4_stac.json"); - insertMetadataRecords("06b09398-d3d0-47dc-a54a-a745319fbece", "classpath:canned/sample3.xml"); + insertMetadataRecords("7709f541-fc0c-4318-b5b9-9053aa474e0e", "classpath:canned/sample4.xml"); indexerService.indexAllMetadataRecordsFromGeoNetwork(true); - Hit objectNodeHit = indexerService.getDocumentByUUID("06b09398-d3d0-47dc-a54a-a745319fbece"); + Hit objectNodeHit = indexerService.getDocumentByUUID("7709f541-fc0c-4318-b5b9-9053aa474e0e"); String test = objectNodeHit.source().toPrettyString(); assertEquals("Stac equals", objectMapper.readTree(expected), objectMapper.readTree(test)); - deleteRecord("06b09398-d3d0-47dc-a54a-a745319fbece"); + deleteRecord("7709f541-fc0c-4318-b5b9-9053aa474e0e"); } } diff --git a/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java b/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java index 26bdbfb7..1cbf7673 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java @@ -1,45 +1,46 @@ package au.org.aodn.esindexer.utils; -import au.org.aodn.esindexer.BaseTestClass; import au.org.aodn.esindexer.abstracts.OgcApiRequestEntityCreator; import au.org.aodn.stac.model.ConceptModel; import au.org.aodn.stac.model.ThemesModel; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.*; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; import org.mockito.Mock; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; +import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.http.HttpEntity; import org.springframework.http.ResponseEntity; -import org.springframework.test.context.ActiveProfiles; +import org.springframework.util.ResourceUtils; import org.springframework.web.client.RestTemplate; import org.springframework.http.MediaType; import org.springframework.http.HttpMethod; + +import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@ActiveProfiles("test") -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class AodnDiscoveryParameterVocabUtilsTest extends BaseTestClass { +@ExtendWith(MockitoExtension.class) +public class AodnDiscoveryParameterVocabUtilsTest { @Mock private RestTemplate restTemplate; @Mock private OgcApiRequestEntityCreator ogcApiRequestEntityCreator; - @Autowired + @InjectMocks AodnDiscoveryParameterVocabUtils aodnDiscoveryParameterVocabUtils; @BeforeEach void setup() throws IOException { - String jsonString = readResourceFile("classpath:canned/aodn_discovery_parameter_vocabs.json"); + File f = ResourceUtils.getFile("classpath:canned/aodn_discovery_parameter_vocabs.json"); + String jsonString = new String(Files.readAllBytes(f.toPath())); ObjectMapper objectMapper = new ObjectMapper(); JsonNode responseJsonNode = objectMapper.readTree(jsonString); HttpEntity mockHttpEntity = new HttpEntity<>(MediaType.APPLICATION_JSON_VALUE); @@ -62,7 +63,7 @@ void testFetchAodnDiscoveryParameterVocabs() { } @Test - void testGetAodnDiscoveryCategories() throws Exception { + void testGetAodnDiscoveryCategories() { // Prepare themes List themes = List.of( new ThemesModel(Arrays.asList( diff --git a/indexer/src/test/resources/canned/sample3_stac.json b/indexer/src/test/resources/canned/sample3_stac.json deleted file mode 100644 index 2a8fbef2..00000000 --- a/indexer/src/test/resources/canned/sample3_stac.json +++ /dev/null @@ -1,342 +0,0 @@ -{ - "title": "IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data °", - "description": "The Animal Tracking Facility (formerly known as the Australian Animal Tracking And Monitoring System (AATAMS)) is a coordinated marine animal tagging project. Satellite Relay Data Loggers (SRDL) (most with CTDs, and some also with fluorometers) are used to explore how marine mammal behaviour relates to their oceanic environment. Loggers developed at the University of St Andrews Sea Mammal Research Unit transmit data in near real time via the Argo satellite system. The Satellite Relay Data Loggers are deployed on marine mammals, including Elephant Seals, Weddell Seals, Australian Fur Seals, Australian Sea Lions, New Zealand Fur Seals. Data is being collected in the Southern Ocean, the Great Australian Bight, and off the South-East Coast of Australia.\n This metadata record, represents several different datasets listed hereafter, which can all be accessed through a multi-WFS service. The data represented by this record are presented in delayed mode.\n CTD - parameters measured by the instruments include time, conductivity (salinity), temperature, speed, fluorescence (available in the future) and depth.\n Diving - parameters measured by the instruments include start and end time and longitude/latitude of each individual dive, post-dive surface duration, dive duration, maximum dive depth, intermediate dive depths and times.\n Haulout - a haulout begins when the SRDL has been continuously dry for a specified length of time (usually 10 minutes). It ends when continuously wet for another interval (usually 40 seconds). Haulout data parameters measured by the instruments include haulout start and end dates and longitude/latitude, and haulout number.\n Argos locations - location data parameters measured by the instruments include time, longitude, latitude, location quality, along with other diagnostic information provided by Argos (http://www.argos-system.org/).\n Summary Statistics - as well as sending records of individual events such as dives and haulouts, the SRDL also calculates summary statistics of those events over a specified time period (usually 3, 4 or 6 hours). Summary statistics computed by the instruments include the proportion of time spent diving, at the surface and hauled-out, the number of dives, and the average, standard deviation and maximum dive duration and dive depth during each summary period. These statistics are based on all the data recorded by the SRDL and so are not prone to distortion by variations in the efficiency of transmission via Argos.\n\n ** For data after October 2018, please consult IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations (https://catalogue-imos.aodn.org.au:443/geonetwork/srv/api/records/70f148b1-7040-4fad-944a-456413c95472), to access data with improved satellite locations. In the near future all historical delayed mode data will be reprocessed with the new quality control (QC) process, to improve the accuracy of the satellite location data, and this dataset will be replaced by the new QC’d one. **", - "extent": { - "temporal": [ - [ - "2007-03-26T06:00:00Z", - "2019-11-29T13:00:00Z" - ], - [ - "2007-03-26T06:00:00Z", - "2019-11-29T13:00:00Z" - ] - ] - }, - "summaries": { - "score": 90, - "status": "completed", - "scope": { - "code": "dataset", - "name": "IMOS dataset level record" - }, - "dataset_provider": "IMOS", - "dataset_group":"sample", - "temporal": [ - { - "start": "2007-03-26T06:00:00Z", - "end": "2019-11-29T13:00:00Z" - } - ] - }, - "contacts": [ - { - "emails": [ - "robert.harcourt@mq.edu.au" - ], - "addresses": [ - { - "deliveryPoint": [ - "Marine Predator Research Group, Department of Biological Sciences", - "Macquarie University" - ], - "country": "Australia", - "city": "North Ryde", - "postalCode": "2109", - "administrativeArea": "New South Wales" - } - ], - "roles": "principalInvestigator", - "organization": "Department of Biological Sciences, Macquarie University", - "name": "Harcourt, Rob", - "phones": [], - "links": [ - { - "href": "http://web.science.mq.edu.au/directory/listing/person.htm?id=rharcour", - "title": "Maquarie University staff profile for Rob Harcourt", - "type": "WWW:LINK-1.0-http--link" - } - ], - "position": "Animal Tracking Facility Leader" - } - ], - "languages": [ - { - "code": "eng", - "name": "English" - } - ], - "links": [ - { - "href": "http://imos.org.au/animaltracking.html", - "rel": "self", - "type": "text/html", - "title": "Animal Tracking Facility page on IMOS website" - }, - { - "href": "https://portal.aodn.org.au/search?uuid=06b09398-d3d0-47dc-a54a-a745319fbece", - "rel": "self", - "type": "text/html", - "title": "View and download data though the AODN Portal" - }, - { - "href": "http://oceancurrent.imos.org.au/aatams.php", - "rel": "self", - "type": "", - "title": "View profile plots from Oceancurrent SealCTDs page" - }, - { - "href": "http://geoserver-123.aodn.org.au/geoserver/wms", - "rel": "self", - "type": "", - "title": "imos:aatams_sattag_dm_profile_map" - }, - { - "href": "http://geoserver-123.aodn.org.au/geoserver/ows", - "rel": "self", - "type": "", - "title": "aatams_sattag_dm_profile_data" - }, - { - "href": "http://geoserver-123.aodn.org.au/geoserver/ows", - "rel": "self", - "type": "", - "title": "aatams_sattag_dm_location_data" - }, - { - "href": "http://geoserver-123.aodn.org.au/geoserver/ows", - "rel": "self", - "type": "", - "title": "aatams_sattag_dm_dive_profile_data" - }, - { - "href": "http://geoserver-123.aodn.org.au/geoserver/ows", - "rel": "self", - "type": "", - "title": "aatams_sattag_dm_haulout_data" - }, - { - "href": "http://geoserver-123.aodn.org.au/geoserver/ows", - "rel": "self", - "type": "", - "title": "aatams_sattag_dm_summary_data" - }, - { - "href": "https://help.aodn.org.au/web-services/ogc-wfs/", - "rel": "self", - "type": "text/html", - "title": "OGC WFS help documentation" - } - ], - "license": "Creative Commons Attribution 4.0 International License", - "providers": [ - { - "name": "Integrated Marine Observing System (IMOS)", - "roles": [ - "distributor" - ], - "url": "http://imos.org.au/aodn.html" - } - ], - "themes": [ - { - "concepts": [ - { - "id": "Oceans | Ocean Temperature | Water Temperature" - }, - { - "id": "Oceans | Salinity/density | Conductivity" - }, - { - "id": "Oceans | Marine Biology | Marine Mammals" - }, - { - "id": "Oceans | Marine Biology | Marine Birds" - } - ], - "scheme": "", - "description": "GCMD", - "title": "NASA/Global Change Master Directory Earth Science Keywords Version 5.3.8" - }, - { - "concepts": [ - { - "id": "Satellite Tag" - }, - { - "id": "CTD Satellite Relay Data Logger" - }, - { - "id": "Agency | SMRU | Sea Mammal Research Unit" - } - ], - "scheme": "", - "description": "IMOS", - "title": "IMOS Keywords Thesaurus" - }, - { - "concepts": [ - { - "id": "Tags and Tracking Devices" - } - ], - "scheme": "theme", - "description": "", - "title": "Marine Community Profile of ISO19115 v1.4 Collection Methods Vocabulary (Annex C.1.3)" - }, - { - "concepts": [ - { - "id": "Global / Oceans | Atlantic Ocean" - }, - { - "id": "Global / Oceans | Southern Ocean" - }, - { - "id": "Global / Oceans | Indian Ocean" - }, - { - "id": "Global / Oceans | Pacific Ocean" - }, - { - "id": "Marine Features (Australia) | Great Australian Bight, SA/WA" - }, - { - "id": "Marine Features (Australia) | Bass Strait, TAS/VIC" - }, - { - "id": "Regional Seas | Coral Sea" - }, - { - "id": "Regional Seas | Tasman Sea" - } - ], - "scheme": "place", - "description": "", - "title": "AODN Geographic Extents Vocabulary" - }, - { - "concepts": [ - { - "id": "Continents | Antarctica" - }, - { - "id": "Countries | Australia" - }, - { - "id": "Countries | New Zealand" - }, - { - "id": "Offshore Islands (Australia) | Heard & McDonald Islands" - }, - { - "id": "Offshore Islands (Australia) | Macquarie Island" - }, - { - "id": "States, Territories (Australia) | Western Australia" - }, - { - "id": "States, Territories (Australia) | South Australia" - }, - { - "id": "States, Territories (Australia) | Victoria" - }, - { - "id": "States, Territories (Australia) | Tasmania" - }, - { - "id": "States, Territories (Australia) | New South Wales" - } - ], - "scheme": "place", - "description": "", - "title": "AODN Geographic Extents Vocabulary" - }, - { - "concepts": [ - { - "id": "land-sea mammals", - "url": "http://vocab.nerc.ac.uk/collection/L06/current/77" - } - ], - "scheme": "theme", - "description": "", - "title": "AODN Platform Vocabulary" - }, - { - "concepts": [ - { - "id": "Temperature of the water body", - "url": "http://vocab.nerc.ac.uk/collection/P01/current/TEMPPR01" - }, - { - "id": "Practical salinity of the water body", - "url": "http://vocab.nerc.ac.uk/collection/P01/current/PSLTZZ01" - }, - { - "id": "Pressure (measured variable) in the water body exerted by overlying sea water and any medium above it", - "url": "http://vocab.aodn.org.au/def/discovery_parameter/entity/565" - } - ], - "scheme": "theme", - "description": "", - "title": "AODN Discovery Parameter Vocabulary" - }, - { - "concepts": [ - { - "id": "Latitude north", - "url": "http://vocab.nerc.ac.uk/collection/P01/current/ALATZZ01" - }, - { - "id": "Longitude east", - "url": "http://vocab.nerc.ac.uk/collection/P01/current/ALONZZ01" - } - ], - "scheme": "theme", - "description": "", - "title": "AODN Sampling Parameter Vocabulary" - }, - { - "concepts": [ - { - "id": "water temperature sensor", - "url": "http://vocab.nerc.ac.uk/collection/L05/current/134" - }, - { - "id": "CTD", - "url": "http://vocab.nerc.ac.uk/collection/L05/current/130" - }, - { - "id": "SMRU Ltd Satellite Relay Data Logger 9000x", - "url": "http://vocab.nerc.ac.uk/collection/L22/current/TOOL0719" - } - ], - "scheme": "theme", - "description": "", - "title": "AODN Instrument Vocabulary" - }, - { - "concepts": [ - { - "id": "Animal Tagging Sub-Facility, Integrated Marine Observing System (IMOS)", - "url": "http://vocab.aodn.org.au/def/organisation/entity/497" - } - ], - "scheme": "discipline", - "description": "", - "title": "AODN Organisation Vocabulary" - } - ], - "id": "06b09398-d3d0-47dc-a54a-a745319fbece", - "title_suggest": "IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data °", - "type": "Collection", - "stac_version": "1.0.0", - "stac_extensions": [ - "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", - "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", - "https://stac-extensions.github.io/projection/v1.1.0/schema.json", - "https://stac-extensions.github.io/language/v1.0.0/schema.json", - "https://stac-extensions.github.io/themes/v1.0.0/schema.json" - ] -} diff --git a/indexer/src/test/resources/canned/sample4_stac.json b/indexer/src/test/resources/canned/sample4_stac.json new file mode 100644 index 00000000..bf489d11 --- /dev/null +++ b/indexer/src/test/resources/canned/sample4_stac.json @@ -0,0 +1,253 @@ +{ + "title" : "Ocean acidification historical reconstruction", + "description" : "This dataset contains the reconstructed time series of monthly mean aragonite, calcite and pH together with distribution of dissolved inorganic carbon (DIC), total alkalinity (ALK), sea surface temperature and salinity in the Australian region at a 1 degree resolution over the period 1870-2013.", + "extent" : { + "bbox" : [ [ 95.5, -44.5, 169.5, -0.5 ], [ 95.5, -44.5, 169.5, -44.5, 169.5, -0.5, 95.5, -0.5, 95.5, -44.5 ] ], + "temporal" : [ [ "1870-07-16T14:10:44Z", "2013-06-16T14:00:00Z" ], [ "1870-07-16T14:10:44Z", "2013-06-16T14:00:00Z" ] ] + }, + "summaries" : { + "score" : 95, + "status" : "completed", + "scope" : { + "code" : "dataset", + "name" : "" + }, + "dataset_group" : "sample", + "dataset_provider" : "IMOS", + "proj:geometry" : { + "coordinates" : [ [ [ [ 95.5, -44.5 ], [ 169.5, -44.5 ], [ 169.5, -0.5 ], [ 95.5, -0.5 ], [ 95.5, -44.5 ] ] ] ], + "type" : "MultiPolygon" + }, + "temporal" : [ { + "start" : "1870-07-16T14:10:44Z", + "end" : "2013-06-16T14:00:00Z" + } ], + "aodn_discovery_categories" : [ "Alkalinity", "Temperature", "Salinity" ] + }, + "contacts" : [ { + "emails" : [ "Andrew.Lenton@csiro.au" ], + "addresses" : [ { + "deliveryPoint" : [ "GPO Box 1538" ], + "country" : "Australia", + "city" : "Hobart", + "postalCode" : "7000", + "administrativeArea" : "Tasmania" + } ], + "roles" : "principalInvestigator", + "organization" : "CSIRO Oceans and Atmosphere - Hobart", + "name" : "Lenton, Andrew", + "phones" : [ ], + "links" : [ ], + "position" : "" + }, { + "emails" : [ "info@aodn.org.au" ], + "addresses" : [ { + "deliveryPoint" : [ "University of Tasmania", "Private Bag 110" ], + "country" : "Australia", + "city" : "Hobart", + "postalCode" : "7001", + "administrativeArea" : "Tasmania" + } ], + "roles" : "pointOfContact", + "organization" : "Integrated Marine Observing System (IMOS)", + "name" : "", + "phones" : [ { + "roles" : "voice", + "value" : "61 3 6226 7488" + }, { + "roles" : "facsimile", + "value" : "61 3 6226 2701" + } ], + "links" : [ { + "href" : "http://imos.org.au/aodn.html", + "title" : "Website of the Australian Ocean Data Network (AODN)", + "type" : "WWW:LINK-1.0-http--link" + } ], + "position" : "Data Officer" + } ], + "languages" : [ { + "code" : "eng", + "name" : "English" + } ], + "links" : [ { + "href" : "http://www.biogeosciences.net/13/1753/2016", + "rel" : "self", + "type" : "text/html", + "title" : "Biogeosciences Journal article" + }, { + "href" : "https://portal.aodn.org.au/search?uuid=7709f541-fc0c-4318-b5b9-9053aa474e0e", + "rel" : "self", + "type" : "text/html", + "title" : "View and download data though the AODN Portal" + }, { + "href" : "http://thredds.aodn.org.au/thredds/dodsC/CSIRO/Climatology/Ocean_Acidification/OA_Reconstruction.nc.html", + "rel" : "self", + "type" : "text/html", + "title" : "NetCDF files via THREDDS catalog" + }, { + "href" : "http://geoserver-123.aodn.org.au/geoserver/ncwms", + "rel" : "self", + "type" : "", + "title" : "csiro_oa_reconstruction_url/SST" + }, { + "href" : "http://geoserver-123.aodn.org.au/geoserver/ncwms", + "rel" : "self", + "type" : "", + "title" : "csiro_oa_reconstruction_url/SSS" + }, { + "href" : "http://geoserver-123.aodn.org.au/geoserver/ncwms", + "rel" : "self", + "type" : "", + "title" : "csiro_oa_reconstruction_url/ALK" + }, { + "href" : "http://geoserver-123.aodn.org.au/geoserver/ncwms", + "rel" : "self", + "type" : "", + "title" : "csiro_oa_reconstruction_url/DIC" + }, { + "href" : "http://geoserver-123.aodn.org.au/geoserver/ncwms", + "rel" : "self", + "type" : "", + "title" : "csiro_oa_reconstruction_url/OMEGA_A" + }, { + "href" : "http://geoserver-123.aodn.org.au/geoserver/ncwms", + "rel" : "self", + "type" : "", + "title" : "csiro_oa_reconstruction_url/OMEGA_C" + }, { + "href" : "http://geoserver-123.aodn.org.au/geoserver/ncwms", + "rel" : "self", + "type" : "", + "title" : "csiro_oa_reconstruction_url/pH_T" + }, { + "href" : "https://processes.aodn.org.au/wps", + "rel" : "self", + "type" : "", + "title" : "csiro_oa_reconstruction_url" + }, { + "href" : "https://help.aodn.org.au/web-services/gogoduck-aggregator/", + "rel" : "self", + "type" : "text/html", + "title" : "GoGoDuck help documentation" + } ], + "license" : "Creative Commons Attribution 4.0 International License", + "providers" : [ { + "name" : "Integrated Marine Observing System (IMOS)", + "roles" : [ "distributor" ], + "url" : "http://imos.org.au/aodn.html" + } ], + "themes" : [ { + "concepts" : [ { + "id" : "Oceans | Ocean Temperature | Sea Surface Temperature" + }, { + "id" : "Oceans | Salinity/density | Salinity" + }, { + "id" : "Oceans | Ocean Chemistry | Carbon Dioxide" + }, { + "id" : "Oceans | Ocean Chemistry | Carbonate" + }, { + "id" : "Oceans | Ocean Chemistry | pH" + } ], + "scheme" : "theme", + "description" : "GCMD", + "title" : "NASA/Global Change Master Directory Earth Science Keywords Version 5.3.8" + }, { + "concepts" : [ { + "id" : "Ocean Biogeochemistry" + }, { + "id" : "pCO2" + } ], + "scheme" : "theme", + "description" : "IMOS", + "title" : "IMOS Keywords Thesaurus" + }, { + "concepts" : [ { + "id" : "Global / Oceans | Pacific Ocean" + }, { + "id" : "Global / Oceans | Southern Ocean" + }, { + "id" : "Global / Oceans | Indian Ocean" + }, { + "id" : "Marine Features (Australia) | Great Australian Bight, SA/WA" + }, { + "id" : "Marine Features (Australia) | Bass Strait, TAS/VIC" + }, { + "id" : "Regional Seas | Coral Sea" + }, { + "id" : "Regional Seas | Tasman Sea" + }, { + "id" : "Regional Seas | Timor Sea" + }, { + "id" : "Regional Seas | Arafura Sea" + }, { + "id" : "Regional Seas | Banda Sea" + }, { + "id" : "Regional Seas | Java Sea" + }, { + "id" : "Regional Seas | Solomon Sea" + } ], + "scheme" : "place", + "description" : "", + "title" : "AODN Geographic Extents Vocabulary" + }, { + "concepts" : [ { + "id" : "Countries | Australia" + }, { + "id" : "Countries | Papua New Guinea" + }, { + "id" : "Countries | Indonesia" + }, { + "id" : "Countries | Timor-Leste" + }, { + "id" : "Countries | New Zealand" + }, { + "id" : "States, Territories (Australia) | Western Australia" + }, { + "id" : "States, Territories (Australia) | Queensland" + }, { + "id" : "States, Territories (Australia) | South Australia" + }, { + "id" : "States, Territories (Australia) | Tasmania" + }, { + "id" : "States, Territories (Australia) | Victoria" + }, { + "id" : "States, Territories (Australia) | New South Wales" + }, { + "id" : "States, Territories (Australia) | Northern Territory" + } ], + "scheme" : "place", + "description" : "", + "title" : "AODN Geographic Extents Vocabulary" + }, { + "concepts" : [ { + "id" : "Temperature of the water body", + "url" : "http://vocab.nerc.ac.uk/collection/P01/current/TEMPPR01" + }, { + "id" : "Practical salinity of the water body", + "url" : "http://vocab.nerc.ac.uk/collection/P01/current/PSLTZZ01" + }, { + "id" : "Concentration of carbon (total inorganic) per unit mass of the water body", + "url" : "http://vocab.aodn.org.au/def/discovery_parameter/entity/1" + }, { + "id" : "Total alkalinity per unit mass of the water body", + "url" : "http://vocab.nerc.ac.uk/collection/P01/current/MDMAP014" + }, { + "id" : "Saturation state of aragonite in the water body", + "url" : "http://vocab.aodn.org.au/def/discovery_parameter/entity/24" + }, { + "id" : "Saturation state of calcite in the water body", + "url" : "http://vocab.aodn.org.au/def/discovery_parameter/entity/25" + }, { + "id" : "pH (total scale) of the water body", + "url" : "http://vocab.aodn.org.au/def/discovery_parameter/entity/27" + } ], + "scheme" : "theme", + "description" : "", + "title" : "AODN Discovery Parameter Vocabulary" + } ], + "id" : "7709f541-fc0c-4318-b5b9-9053aa474e0e", + "title_suggest" : "Ocean acidification historical reconstruction", + "type" : "Collection", + "stac_version" : "1.0.0", + "stac_extensions" : [ "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", "https://stac-extensions.github.io/projection/v1.1.0/schema.json", "https://stac-extensions.github.io/language/v1.0.0/schema.json", "https://stac-extensions.github.io/themes/v1.0.0/schema.json" ] +} diff --git a/stacmodel/src/main/java/au/org/aodn/stac/model/ThemesModel.java b/stacmodel/src/main/java/au/org/aodn/stac/model/ThemesModel.java index 8738d716..686cc303 100644 --- a/stacmodel/src/main/java/au/org/aodn/stac/model/ThemesModel.java +++ b/stacmodel/src/main/java/au/org/aodn/stac/model/ThemesModel.java @@ -3,7 +3,6 @@ import lombok.Builder; import lombok.Data; -import java.util.Map; import java.util.List; @Data From 34118217e91ee437e6226b4a7be129dccac310c1 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Date: Wed, 17 Apr 2024 23:13:54 +1000 Subject: [PATCH 06/19] Process ARDC APIs, refactor, and add tests --- ardcvocabs/pom.xml | 70 ++++ .../au/org/aodn/ardcvocabs/ArdcVocabs.java | 20 ++ .../aodn/ardcvocabs/configuration/Config.java | 31 ++ .../ardcvocabs/model/CategoryVocabModel.java | 23 ++ .../ardcvocabs/service/ArdcVocabsService.java | 194 ++++++++++ .../service/ArdcVocabsServiceTest.java | 81 +++++ .../src/test/resources/databag/vocab0.json | 212 +++++++++++ .../src/test/resources/databag/vocab1.json | 228 ++++++++++++ .../src/test/resources/databag/vocab2.json | 333 ++++++++++++++++++ .../src/test/resources/databag/vocab3.json | 80 +++++ .../resources/databag/vocab_discovery0.json | 32 ++ .../resources/databag/vocab_entity_390.json | 54 +++ .../AodnDiscoveryParameterVocabService.java} | 42 +-- .../esindexer/utils/CacheArdcVocabsUtils.java | 4 + .../AodnDiscoveryParameterVocabUtilsTest.java | 3 +- 15 files changed, 1377 insertions(+), 30 deletions(-) create mode 100644 ardcvocabs/pom.xml create mode 100644 ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/ArdcVocabs.java create mode 100644 ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/Config.java create mode 100644 ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/model/CategoryVocabModel.java create mode 100644 ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabsService.java create mode 100644 ardcvocabs/src/test/java/au/org/aodn/ardcvocabs/service/ArdcVocabsServiceTest.java create mode 100644 ardcvocabs/src/test/resources/databag/vocab0.json create mode 100644 ardcvocabs/src/test/resources/databag/vocab1.json create mode 100644 ardcvocabs/src/test/resources/databag/vocab2.json create mode 100644 ardcvocabs/src/test/resources/databag/vocab3.json create mode 100644 ardcvocabs/src/test/resources/databag/vocab_discovery0.json create mode 100644 ardcvocabs/src/test/resources/databag/vocab_entity_390.json rename indexer/src/main/java/au/org/aodn/esindexer/{utils/AodnDiscoveryParameterVocabUtils.java => service/AodnDiscoveryParameterVocabService.java} (58%) create mode 100644 indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java diff --git a/ardcvocabs/pom.xml b/ardcvocabs/pom.xml new file mode 100644 index 00000000..cc31feb6 --- /dev/null +++ b/ardcvocabs/pom.xml @@ -0,0 +1,70 @@ + + + + es-indexer + au.org.aodn + 0.0.0 + ../pom.xml + + 4.0.0 + + ardcvocabs + jar + + + 17 + 17 + + + + + org.springframework.boot + spring-boot-starter-web + + + javax.annotation + javax.annotation-api + + + javax.servlet + javax.servlet-api + 4.0.1 + provided + + + org.projectlombok + lombok + true + + + org.springframework.boot + spring-boot-autoconfigure + + + org.springframework.boot + spring-boot-starter-test + + + + + + + org.apache.maven.plugins + maven-source-plugin + ${maven.sourceplugin.version} + + + attach-sources + verify + + jar-no-fork + + + + + + + + diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/ArdcVocabs.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/ArdcVocabs.java new file mode 100644 index 00000000..b1fd729d --- /dev/null +++ b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/ArdcVocabs.java @@ -0,0 +1,20 @@ +package au.org.aodn.ardcvocabs; + +import jakarta.annotation.PostConstruct; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +import java.util.TimeZone; + +@SpringBootApplication + +public class ArdcVocabs { + @PostConstruct + void init() { + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + } + + public static void main(String[] args) { + SpringApplication.run(ArdcVocabs.class, args); + } +} diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/Config.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/Config.java new file mode 100644 index 00000000..751b220e --- /dev/null +++ b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/Config.java @@ -0,0 +1,31 @@ +package au.org.aodn.ardcvocabs.configuration; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.annotation.PostConstruct; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.web.client.RestTemplate; + +@Configuration +@EnableCaching +@EnableScheduling +public class Config { + + @Autowired + ObjectMapper mapper; + + @PostConstruct + public void init() { + // configure ObjectMapper to exclude null fields while serializing + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + } + + @Bean + public RestTemplate ardcVocabRestTemplate() { + return new RestTemplate(); + } +} diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/model/CategoryVocabModel.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/model/CategoryVocabModel.java new file mode 100644 index 00000000..f30c495f --- /dev/null +++ b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/model/CategoryVocabModel.java @@ -0,0 +1,23 @@ +package au.org.aodn.ardcvocabs.model; + +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; + +import java.util.List; +import java.util.Map; + +/** + * This is the model class for http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/ + */ +@Builder +@Getter +@Setter +public class CategoryVocabModel { + + protected String label; + protected String definition; + protected String about; + protected List broader; + protected List narrower; +} diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabsService.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabsService.java new file mode 100644 index 00000000..9b2517d5 --- /dev/null +++ b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabsService.java @@ -0,0 +1,194 @@ +package au.org.aodn.ardcvocabs.service; + +import au.org.aodn.ardcvocabs.model.CategoryVocabModel; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestTemplate; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Function; + + +@Slf4j +@Service +public class ArdcVocabsService { + @Autowired + @Qualifier("ardcVocabRestTemplate") + protected RestTemplate template; + + protected static String path = "/aodn-parameter-category-vocabulary/version-2-1/concept.json"; + protected static String leafPath = "/aodn-discovery-parameter-vocabulary/version-1-6/concept.json"; + + protected static String details = "/aodn-discovery-parameter-vocabulary/version-1-6/resource.json?uri=%s"; + + protected Function label = (node) -> node.get("prefLabel").get("_value").asText(); + protected Function about = (node) -> node.has("_about") ? node.get("_about").asText() : null; + protected Function definition = (node) -> node.has("definition") ? node.get("definition").asText() : null; + + protected BiFunction isNodeValid = (node, item) -> node != null && !node.isEmpty() && node.has(item) && !node.get(item).isEmpty(); + + /** + * We want to get the list of leaf node for the API, from there we need to query individual resources to get the broadMatch value + * this value is the link to the second level of the category + * + * API to the details to get the broadMatch + * http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-discovery-parameter-vocabulary/version-1-6/resource.json?uri=http://vocab.aodn.org.au/def/discovery_parameter/891 + * + * @param vocabApiBase + * @return + */ + protected Map> getLeafNodeOfParameterCategory(String vocabApiBase) { + Map> result = new HashMap<>(); + String url = String.format(vocabApiBase + leafPath); + + while (url != null) { + log.debug("Query api -> {}", url); + + ObjectNode r = template.getForObject(url, ObjectNode.class); + if (r != null && !r.isEmpty()) { + JsonNode node = r.get("result"); + + if (isNodeValid.apply(node, "items")) { + for (JsonNode j : node.get("items")) { + // Now we need to construct link to detail resources + String dl = String.format(vocabApiBase + details, about.apply(j)); + try { + log.debug("Query api -> {}", dl); + ObjectNode d = template.getForObject(dl, ObjectNode.class); + + if(isNodeValid.apply(d, "result") && isNodeValid.apply(d.get("result"), "primaryTopic")) { + JsonNode target = d.get("result").get("primaryTopic"); + + CategoryVocabModel model = CategoryVocabModel + .builder() + .label(label.apply(target)) + .definition(definition.apply(target)) + .about(about.apply(target)) + .build(); + + if(target.has("broadMatch") && !target.get("broadMatch").isEmpty()) { + for(JsonNode bm : target.get("broadMatch")) { + if (!result.containsKey(bm.asText())) { + result.put(bm.asText(), new ArrayList<>()); + } + // We will have multiple cat under the same parent + result.get(bm.asText()).add(model); + } + } + } + } + catch(Exception e) { + log.error("Item not found in resource {}", dl); + } + } + } + + if (!node.isEmpty() && node.has("next")) { + url = node.get("next").asText(); + } + else { + url = null; + } + } + else { + url = null; + } + } + return result; + } + + public List getParameterCategory(String vocabApiBase) { + Map> leaves = getLeafNodeOfParameterCategory(vocabApiBase); + List result = new ArrayList<>(); + + String url = String.format(vocabApiBase + path); + + while (url != null) { + try { + log.debug("Query api -> {}", url); + + ObjectNode r = template.getForObject(url, ObjectNode.class); + + if (r != null && !r.isEmpty()) { + JsonNode node = r.get("result"); + + if (!node.isEmpty() && node.has("items") && !node.get("items").isEmpty()) { + for (JsonNode j : node.get("items")) { + List broader = new ArrayList<>(); + List narrower = new ArrayList<>(); + + log.debug("Processing label {}", label.apply(j)); + if (j.has("broader")) { + for (JsonNode b : j.get("broader")) { + if (b.has("prefLabel") && b.has("_about")) { + CategoryVocabModel c = CategoryVocabModel + .builder() + .about(about.apply(b)) + .label(label.apply(b)) + .build(); + broader.add(c); + } + } + } + + if (j.has("narrower")) { + for (JsonNode b : j.get("narrower")) { + if (b.has("prefLabel") && b.has("_about")) { + CategoryVocabModel c = CategoryVocabModel + .builder() + .about(about.apply(b)) + .label(label.apply(b)) + .build(); + + narrower.add(c); + + // The record comes from ardc have two levels only, so the second level for sure + // is empty, but the third level info comes form another link (aka the leaves) + // and therefore we can attach it to the second level to for the third. + if(leaves.containsKey(about.apply(b))) { + c.setNarrower(leaves.get(about.apply(b))); + } + } + } + } + + CategoryVocabModel model = CategoryVocabModel + .builder() + .label(label.apply(j)) + .definition(definition.apply(j)) + .about(about.apply(j)) + .broader(broader) + .narrower(narrower) + .build(); + + result.add(model); + } + } + + if (!node.isEmpty() && node.has("next")) { + url = node.get("next").asText(); + } + else { + url = null; + } + } + else { + url = null; + } + } catch (RestClientException e) { + log.error("Fail connect {}, category return likely outdated", url); + url = null; + } + } + return result; + } +} diff --git a/ardcvocabs/src/test/java/au/org/aodn/ardcvocabs/service/ArdcVocabsServiceTest.java b/ardcvocabs/src/test/java/au/org/aodn/ardcvocabs/service/ArdcVocabsServiceTest.java new file mode 100644 index 00000000..1b16f5e4 --- /dev/null +++ b/ardcvocabs/src/test/java/au/org/aodn/ardcvocabs/service/ArdcVocabsServiceTest.java @@ -0,0 +1,81 @@ +package au.org.aodn.ardcvocabs.service; + +import au.org.aodn.ardcvocabs.model.CategoryVocabModel; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.util.ResourceUtils; +import org.springframework.web.client.RestTemplate; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.endsWith; +import static org.springframework.test.util.AssertionErrors.assertEquals; +import static org.springframework.test.util.AssertionErrors.assertTrue; + +@ExtendWith(MockitoExtension.class) +public class ArdcVocabsServiceTest { + + @Mock + RestTemplate mockRestTemplate; + + @InjectMocks + protected ArdcVocabsService ardcVocabsService; + + ObjectMapper objectMapper = new ObjectMapper(); + + @Test + public void verifyGetParameterCategory() throws IOException { + // Create expect result + Mockito.when(mockRestTemplate.getForObject(endsWith("/aodn-parameter-category-vocabulary/version-2-1/concept.json"), any(), any(Object[].class))) + .thenReturn((ObjectNode)objectMapper.readTree(ResourceUtils.getFile("classpath:databag/vocab0.json"))); + + Mockito.when(mockRestTemplate.getForObject(endsWith("/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=1"), any(), any(Object[].class))) + .thenReturn((ObjectNode)objectMapper.readTree(ResourceUtils.getFile("classpath:databag/vocab1.json"))); + + Mockito.when(mockRestTemplate.getForObject(endsWith("/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=2"), any(), any(Object[].class))) + .thenReturn((ObjectNode)objectMapper.readTree(ResourceUtils.getFile("classpath:databag/vocab2.json"))); + + Mockito.when(mockRestTemplate.getForObject(endsWith("/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=3"), any(), any(Object[].class))) + .thenReturn((ObjectNode)objectMapper.readTree(ResourceUtils.getFile("classpath:databag/vocab3.json"))); + + Mockito.when(mockRestTemplate.getForObject(endsWith("/aodn-discovery-parameter-vocabulary/version-1-6/concept.json"), any(), any(Object[].class))) + .thenReturn((ObjectNode)objectMapper.readTree(ResourceUtils.getFile("classpath:databag/vocab_discovery0.json"))); + + Mockito.when(mockRestTemplate.getForObject(endsWith("/aodn-discovery-parameter-vocabulary/version-1-6/resource.json?uri=http://vocab.aodn.org.au/def/discovery_parameter/entity/390"), any(), any(Object[].class))) + .thenReturn((ObjectNode)objectMapper.readTree(ResourceUtils.getFile("classpath:databag/vocab_entity_390.json"))); + + List categoryVocabModelList = ardcVocabsService.getParameterCategory(""); + assertEquals("Total equals", 33, categoryVocabModelList.size()); + + Optional m = categoryVocabModelList + .stream() + .filter(p -> !p.getNarrower().isEmpty() && p.getLabel().equals("Physical-Atmosphere")) + .findFirst(); + + assertTrue("Find target Physical-Atmosphere", m.isPresent()); + assertEquals("Have narrower equals", 6, m.get().getNarrower().size()); + + Optional visibility = m.get().getNarrower() + .stream() + .filter(p -> p.getLabel().equals("Visibility")) + .findFirst(); + + assertTrue("Find target Visibility", visibility.isPresent()); + + Optional airSeaLevel = visibility.get().getNarrower() + .stream() + .filter(p -> p.getLabel().equals("Horizontal visibility in the atmosphere")) + .findFirst(); + + assertTrue("Horizontal visibility in the atmosphere found", airSeaLevel.isPresent()); + } +} diff --git a/ardcvocabs/src/test/resources/databag/vocab0.json b/ardcvocabs/src/test/resources/databag/vocab0.json new file mode 100644 index 00000000..60e06c55 --- /dev/null +++ b/ardcvocabs/src/test/resources/databag/vocab0.json @@ -0,0 +1,212 @@ +{ + "format": "linked-data-api", + "version": "0.2", + "result": { + "_about": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json", + "definition": "http://vocabs.ardc.edu.au/repository/api/lda/meta/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json", + "extendedMetadataVersion": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json?_metadata=all", + "first": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=0", + "hasPart": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json", + "isPartOf": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json", + "items": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/10", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/53", + "prefLabel": { + "_value": "Physical-Atmosphere", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2018-03-14T22:42:26Z", + "definition": "This category contains vocabulary terms describing air pressure parameters", + "prefLabel": { + "_value": "Air pressure", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/11", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/54", + "prefLabel": { + "_value": "Chemical", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2018-03-14T22:42:27Z", + "definition": "This category contains vocabulary terms describing dissolved oxygen parameters", + "prefLabel": { + "_value": "Oxygen", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/19", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/55", + "prefLabel": { + "_value": "Biological", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2020-10-13T06:00:26Z", + "definition": "This category contains vocabulary terms describing chlorophyll concentration parameters", + "prefLabel": { + "_value": "Chlorophyll", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/2", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/56", + "prefLabel": { + "_value": "Physical-Water", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2023-07-27T06:19:42Z", + "definition": "This category contains vocabulary terms describing waves parameters", + "prefLabel": { + "_value": "Wave", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/22", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/53", + "prefLabel": { + "_value": "Physical-Atmosphere", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2018-03-14T22:42:26Z", + "definition": "This category contains vocabulary terms describing atmospheric visibility parameters", + "prefLabel": { + "_value": "Visibility", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/23", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/56", + "prefLabel": { + "_value": "Physical-Water", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2018-03-14T22:42:28Z", + "definition": "This category contains vocabulary terms describing density parameters", + "prefLabel": { + "_value": "Density", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/25", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/56", + "prefLabel": { + "_value": "Physical-Water", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2018-03-14T22:42:28Z", + "definition": "This category contains vocabulary terms describing water pressure parameters", + "prefLabel": { + "_value": "Water pressure", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/26", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/55", + "prefLabel": { + "_value": "Biological", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2020-10-13T05:50:01Z", + "definition": "This category contains vocabulary terms describing pigments concentration parameters", + "prefLabel": { + "_value": "Other Pigment", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/27", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/54", + "prefLabel": { + "_value": "Chemical", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2021-07-20T04:40:55Z", + "definition": "This category contains vocabulary terms describing alkalinity parameters", + "prefLabel": { + "_value": "Alkalinity", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/28", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/56", + "prefLabel": { + "_value": "Physical-Water", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2018-03-14T22:42:28Z", + "definition": "This category contains vocabulary terms describing turbidity parameters", + "prefLabel": { + "_value": "Turbidity", + "_lang": "en" + } + } + ], + "itemsPerPage": 10, + "next": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=1", + "page": 0, + "startIndex": 1, + "type": [ + "http://purl.org/linked-data/api/vocab#ListEndpoint", + "http://purl.org/linked-data/api/vocab#Page" + ] + } +} diff --git a/ardcvocabs/src/test/resources/databag/vocab1.json b/ardcvocabs/src/test/resources/databag/vocab1.json new file mode 100644 index 00000000..c8384499 --- /dev/null +++ b/ardcvocabs/src/test/resources/databag/vocab1.json @@ -0,0 +1,228 @@ +{ + "format": "linked-data-api", + "version": "0.2", + "result": { + "_about": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=1", + "definition": "http://vocabs.ardc.edu.au/repository/api/lda/meta/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json", + "extendedMetadataVersion": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=1&_metadata=all", + "first": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=0", + "isPartOf": { + "_about": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json", + "definition": "http://vocabs.ardc.edu.au/repository/api/lda/meta/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json", + "hasPart": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=1", + "type": "http://purl.org/linked-data/api/vocab#ListEndpoint" + }, + "items": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/3", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/56", + "prefLabel": { + "_value": "Physical-Water", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2018-03-14T22:42:28Z", + "definition": "This category contains vocabulary terms describing currents parameters", + "prefLabel": { + "_value": "Current", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/30", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/53", + "prefLabel": { + "_value": "Physical-Atmosphere", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/56", + "prefLabel": { + "_value": "Physical-Water", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2020-09-08T04:25:53Z", + "definition": "This category contains vocabulary terms describing Air-Sea Fluxes parameters", + "prefLabel": { + "_value": "Air-Sea Fluxes", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/4", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/55", + "prefLabel": { + "_value": "Biological", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2018-03-14T22:42:27Z", + "definition": "This category contains vocabulary terms describing ocean biota parameters", + "prefLabel": { + "_value": "Ocean Biota", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/45", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/56", + "prefLabel": { + "_value": "Physical-Water", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2021-07-20T05:59:26Z", + "definition": "This category contains vocabulary terms describing optical properties parameters", + "prefLabel": { + "_value": "Optical properties", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/46", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/54", + "prefLabel": { + "_value": "Chemical", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/55", + "prefLabel": { + "_value": "Biological", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2022-10-19T22:22:24Z", + "definition": "This category contains vocabulary terms describing suspended particulate material parameters", + "prefLabel": { + "_value": "Suspended particulate material", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/47", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/56", + "prefLabel": { + "_value": "Physical-Water", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2020-12-08T05:22:35Z", + "definition": "This category contains vocabulary terms describing backscattering parameters", + "prefLabel": { + "_value": "Backscattering", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/48", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/53", + "prefLabel": { + "_value": "Physical-Atmosphere", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2018-03-14T22:42:26Z", + "definition": "This category contains vocabulary terms describing UV radiation parameters", + "prefLabel": { + "_value": "UV radiation", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/49", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/56", + "prefLabel": { + "_value": "Physical-Water", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2018-03-14T22:42:28Z", + "definition": "This category contains vocabulary terms describing temperature parameters", + "prefLabel": { + "_value": "Temperature", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/5", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/56", + "prefLabel": { + "_value": "Physical-Water", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2020-12-08T05:22:23Z", + "definition": "This category contains vocabulary terms describing acoustics parameters", + "prefLabel": { + "_value": "Acoustics", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/50", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/56", + "prefLabel": { + "_value": "Physical-Water", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2018-03-14T22:42:28Z", + "definition": "This category contains vocabulary terms describing salinity parameters", + "prefLabel": { + "_value": "Salinity", + "_lang": "en" + } + } + ], + "itemsPerPage": 10, + "next": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=2", + "page": 1, + "prev": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=0", + "startIndex": 11, + "type": "http://purl.org/linked-data/api/vocab#Page" + } +} diff --git a/ardcvocabs/src/test/resources/databag/vocab2.json b/ardcvocabs/src/test/resources/databag/vocab2.json new file mode 100644 index 00000000..ed2a17de --- /dev/null +++ b/ardcvocabs/src/test/resources/databag/vocab2.json @@ -0,0 +1,333 @@ +{ + "format": "linked-data-api", + "version": "0.2", + "result": { + "_about": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=2", + "definition": "http://vocabs.ardc.edu.au/repository/api/lda/meta/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json", + "extendedMetadataVersion": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=2&_metadata=all", + "first": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=0", + "isPartOf": { + "_about": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json", + "definition": "http://vocabs.ardc.edu.au/repository/api/lda/meta/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json", + "hasPart": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=2", + "type": "http://purl.org/linked-data/api/vocab#ListEndpoint" + }, + "items": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/51", + "broader": [ + "http://vocab.aodn.org.au/def/parameter_classes/category/54", + "http://vocab.aodn.org.au/def/parameter_classes/category/55" + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2020-10-13T03:05:11Z", + "definition": "This category contains vocabulary terms describing nutrient parameters", + "prefLabel": { + "_value": "Nutrient", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/52", + "broader": [ + "http://vocab.aodn.org.au/def/parameter_classes/category/54" + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2021-07-20T04:40:45Z", + "definition": "This category contains vocabulary terms describing carbon parameters", + "prefLabel": { + "_value": "Carbon", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/53", + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2018-03-14T22:42:26Z", + "definition": "This category contains vocabulary terms describing physical atmosphere parameters", + "narrower": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/10", + "prefLabel": { + "_value": "Air pressure", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/22", + "prefLabel": { + "_value": "Visibility", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/30", + "prefLabel": { + "_value": "Air-Sea Fluxes", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/48", + "prefLabel": { + "_value": "UV radiation", + "_lang": "en" + } + }, + "http://vocab.aodn.org.au/def/parameter_classes/category/6", + "http://vocab.aodn.org.au/def/parameter_classes/category/7", + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/8", + "prefLabel": { + "_value": "Humidity", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/9", + "prefLabel": { + "_value": "Precipitation and evaporation", + "_lang": "en" + } + } + ], + "prefLabel": { + "_value": "Physical-Atmosphere", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/54", + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2022-10-19T22:22:24Z", + "definition": "This category contains vocabulary terms describing chemicalparameters", + "narrower": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/11", + "prefLabel": { + "_value": "Oxygen", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/27", + "prefLabel": { + "_value": "Alkalinity", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/46", + "prefLabel": { + "_value": "Suspended particulate material", + "_lang": "en" + } + }, + "http://vocab.aodn.org.au/def/parameter_classes/category/51", + "http://vocab.aodn.org.au/def/parameter_classes/category/52" + ], + "prefLabel": { + "_value": "Chemical", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/55", + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2018-03-14T22:42:27Z", + "definition": "This category contains vocabulary terms describing biologicalparameters", + "narrower": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/19", + "prefLabel": { + "_value": "Chlorophyll", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/26", + "prefLabel": { + "_value": "Other Pigment", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/4", + "prefLabel": { + "_value": "Ocean Biota", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/46", + "prefLabel": { + "_value": "Suspended particulate material", + "_lang": "en" + } + }, + "http://vocab.aodn.org.au/def/parameter_classes/category/51" + ], + "prefLabel": { + "_value": "Biological", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/56", + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2018-03-14T22:42:28Z", + "definition": "This category contains vocabulary terms describing physical water parameters", + "narrower": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/1", + "prefLabel": { + "_value": "Bathymetry", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/2", + "prefLabel": { + "_value": "Wave", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/23", + "prefLabel": { + "_value": "Density", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/25", + "prefLabel": { + "_value": "Water pressure", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/28", + "prefLabel": { + "_value": "Turbidity", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/3", + "prefLabel": { + "_value": "Current", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/30", + "prefLabel": { + "_value": "Air-Sea Fluxes", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/45", + "prefLabel": { + "_value": "Optical properties", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/47", + "prefLabel": { + "_value": "Backscattering", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/49", + "prefLabel": { + "_value": "Temperature", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/5", + "prefLabel": { + "_value": "Acoustics", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/50", + "prefLabel": { + "_value": "Salinity", + "_lang": "en" + } + }, + "http://vocab.aodn.org.au/def/parameter_classes/category/57", + "http://vocab.aodn.org.au/def/parameter_classes/category/58" + ], + "prefLabel": { + "_value": "Physical-Water", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/57", + "broader": [ + "http://vocab.aodn.org.au/def/parameter_classes/category/56" + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2018-10-30T00:49:29Z", + "definition": "This category contains vocabulary terms describing sea surface height parameters", + "prefLabel": { + "_value": "Sea surface height", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/58", + "broader": [ + "http://vocab.aodn.org.au/def/parameter_classes/category/56" + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2018-12-11T04:39:56Z", + "definition": "This category contains vocabulary terms describing depth parameters", + "prefLabel": { + "_value": "Depth", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/6", + "broader": [ + "http://vocab.aodn.org.au/def/parameter_classes/category/53" + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2018-03-14T22:42:26Z", + "definition": "This category contains vocabulary terms describing wind parameters", + "prefLabel": { + "_value": "Wind", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/7", + "broader": [ + "http://vocab.aodn.org.au/def/parameter_classes/category/53" + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2018-03-14T22:42:26Z", + "definition": "This category contains vocabulary terms describing air temperature parameters", + "prefLabel": { + "_value": "Air temperature", + "_lang": "en" + } + } + ], + "itemsPerPage": 10, + "next": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=3", + "page": 2, + "prev": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=1", + "startIndex": 21, + "type": "http://purl.org/linked-data/api/vocab#Page" + } +} diff --git a/ardcvocabs/src/test/resources/databag/vocab3.json b/ardcvocabs/src/test/resources/databag/vocab3.json new file mode 100644 index 00000000..e1e54110 --- /dev/null +++ b/ardcvocabs/src/test/resources/databag/vocab3.json @@ -0,0 +1,80 @@ +{ + "format": "linked-data-api", + "version": "0.2", + "result": { + "_about": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=3", + "definition": "http://vocabs.ardc.edu.au/repository/api/lda/meta/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json", + "extendedMetadataVersion": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=3&_metadata=all", + "first": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=0", + "isPartOf": { + "_about": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json", + "definition": "http://vocabs.ardc.edu.au/repository/api/lda/meta/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json", + "hasPart": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=3", + "type": "http://purl.org/linked-data/api/vocab#ListEndpoint" + }, + "items": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/8", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/53", + "prefLabel": { + "_value": "Physical-Atmosphere", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2018-03-14T22:42:26Z", + "definition": "This category contains vocabulary terms describing atmospheric humidity parameters", + "prefLabel": { + "_value": "Humidity", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/9", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/53", + "prefLabel": { + "_value": "Physical-Atmosphere", + "_lang": "en" + } + } + ], + "dctermsCreated": "2014-06-01T00:00:00Z", + "dctermsModified": "2018-03-14T22:42:27Z", + "definition": "This category contains vocabulary terms describing precipitation and evaporation parameters", + "prefLabel": { + "_value": "Precipitation and evaporation", + "_lang": "en" + } + }, + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/1", + "broader": [ + { + "_about": "http://vocab.aodn.org.au/def/parameter_classes/category/56", + "prefLabel": { + "_value": "Physical-Water", + "_lang": "en" + } + } + ], + "dctermsCreated": "2018-10-31T23:07:48Z", + "dctermsModified": "2018-12-11T04:39:23Z", + "definition": "This category contains vocabulary terms describing bathymetry parameters", + "prefLabel": { + "_value": "Bathymetry", + "_lang": "en" + } + } + ], + "itemsPerPage": 10, + "page": 3, + "prev": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/version-2-1/concept.json?_page=2", + "startIndex": 31, + "type": "http://purl.org/linked-data/api/vocab#Page" + } +} diff --git a/ardcvocabs/src/test/resources/databag/vocab_discovery0.json b/ardcvocabs/src/test/resources/databag/vocab_discovery0.json new file mode 100644 index 00000000..4508e49a --- /dev/null +++ b/ardcvocabs/src/test/resources/databag/vocab_discovery0.json @@ -0,0 +1,32 @@ +{ + "format": "linked-data-api", + "version": "0.2", + "result": { + "_about": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-discovery-parameter-vocabulary/version-1-6/concept.json?_page=1", + "definition": "http://vocabs.ardc.edu.au/repository/api/lda/meta/aodn/aodn-discovery-parameter-vocabulary/version-1-6/concept.json", + "extendedMetadataVersion": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-discovery-parameter-vocabulary/version-1-6/concept.json?_page=1&_metadata=all", + "first": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-discovery-parameter-vocabulary/version-1-6/concept.json?_page=0", + "isPartOf": { + "_about": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-discovery-parameter-vocabulary/version-1-6/concept.json", + "definition": "http://vocabs.ardc.edu.au/repository/api/lda/meta/aodn/aodn-discovery-parameter-vocabulary/version-1-6/concept.json", + "hasPart": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-discovery-parameter-vocabulary/version-1-6/concept.json?_page=1", + "type": "http://purl.org/linked-data/api/vocab#ListEndpoint" + }, + "items": [ + { + "_about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/390", + "dctermsCreated": "2015-08-07T00:57:51Z", + "dctermsModified": "2018-02-23T00:23:31Z", + "prefLabel": { + "_value": "Horizontal visibility in the atmosphere", + "_lang": "en" + } + } + ], + "itemsPerPage": 1, + "page": 1, + "prev": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-discovery-parameter-vocabulary/version-1-6/concept.json?_page=0", + "startIndex": 11, + "type": "http://purl.org/linked-data/api/vocab#Page" + } +} diff --git a/ardcvocabs/src/test/resources/databag/vocab_entity_390.json b/ardcvocabs/src/test/resources/databag/vocab_entity_390.json new file mode 100644 index 00000000..8513f8eb --- /dev/null +++ b/ardcvocabs/src/test/resources/databag/vocab_entity_390.json @@ -0,0 +1,54 @@ +{ + "format": "linked-data-api", + "version": "0.2", + "result": { + "_about": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-discovery-parameter-vocabulary/version-1-6/resource.json?uri=http://vocab.aodn.org.au/def/discovery_parameter/entity/390", + "definition": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-discovery-parameter-vocabulary/version-1-6/resource.json?uri=http://vocab.aodn.org.au/def/discovery_parameter/entity/390", + "extendedMetadataVersion": "http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-discovery-parameter-vocabulary/version-1-6/resource.json?_metadata=all&uri=http://vocab.aodn.org.au/def/discovery_parameter/entity/390", + "primaryTopic": { + "_about": "http://vocab.aodn.org.au/def/discovery_parameter/entity/390", + "broadMatch": [ + "http://vocab.aodn.org.au/def/parameter_classes/category/22" + ], + "contributor": "http://editor.vocabs.ands.org.au/user/eMII_Atkins.Natalia", + "creator": "http://editor.vocabs.ands.org.au/user/Sebastien-Mancini", + "dcPublisher": "eMarine Information Infrastructure (eMII)", + "dcSource": "Australian Ocean Data Network discovery parameter register", + "dctermsCreated": "2015-05-11T00:00:00Z", + "dctermsModified": "2018-10-24T02:34:33Z", + "hiddenLabel": [ + { + "_value": "VIS", + "_lang": "en" + } + ], + "inScheme": { + "_about": "http://vocab.aodn.org.au/def/discovery_parameter/1", + "hasTopConcept": [ + "http://vocab.aodn.org.au/def/discovery_parameter/entity/390" + ] + }, + "prefLabel": { + "_value": "Horizontal visibility in the atmosphere", + "_lang": "en" + }, + "relatedMatch": [ + "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0061/" + ], + "topConceptOf": { + "_about": "http://vocab.aodn.org.au/def/discovery_parameter/1", + "hasTopConcept": [ + "http://vocab.aodn.org.au/def/discovery_parameter/entity/390" + ] + }, + "type": [ + "http://www.w3.org/2000/01/rdf-schema#Resource", + "http://www.w3.org/2004/02/skos/core#Concept" + ] + }, + "type": [ + "http://purl.org/linked-data/api/vocab#ItemEndpoint", + "http://purl.org/linked-data/api/vocab#Page" + ] + } +} diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/service/AodnDiscoveryParameterVocabService.java similarity index 58% rename from indexer/src/main/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtils.java rename to indexer/src/main/java/au/org/aodn/esindexer/service/AodnDiscoveryParameterVocabService.java index 11c54e69..2c356f9e 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/AodnDiscoveryParameterVocabService.java @@ -1,15 +1,11 @@ -package au.org.aodn.esindexer.utils; +package au.org.aodn.esindexer.service; +import au.org.aodn.ardcvocabs.model.CategoryVocabModel; import au.org.aodn.esindexer.abstracts.OgcApiRequestEntityCreator; +import au.org.aodn.esindexer.utils.CacheArdcVocabsUtils; import au.org.aodn.stac.model.ConceptModel; import au.org.aodn.stac.model.ThemesModel; -import com.fasterxml.jackson.databind.JsonNode; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpMethod; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; import org.springframework.web.client.RestTemplate; @@ -17,26 +13,14 @@ import java.util.List; @Component -public class AodnDiscoveryParameterVocabUtils { - - @Value("${ogc-api.host}") - private String ogcApiHost; - +public class AodnDiscoveryParameterVocabService { @Autowired RestTemplate restTemplate; @Autowired OgcApiRequestEntityCreator ogcApiRequestEntityCreator; - protected JsonNode fetchAodnDiscoveryParameterVocabs() { - HttpEntity requestEntity = ogcApiRequestEntityCreator.getRequestEntity(MediaType.APPLICATION_JSON, null); - ResponseEntity responseEntity = restTemplate.exchange( - ogcApiHost + "/api/v1/ogc/ext/parameter/categories", - HttpMethod.GET, - requestEntity, - JsonNode.class); - return responseEntity.getBody(); - } + CacheArdcVocabsUtils cacheArdcVocabsUtils; protected boolean themesMatchConcept(List themes, ConceptModel concept) { for (ThemesModel theme : themes) { @@ -57,16 +41,16 @@ comparing by hashcode (id and url) of the concept object public List getAodnDiscoveryCategories(List themes) { List results = new ArrayList<>(); // Iterate over the top-level vocabularies - for (JsonNode topLevelVocab : this.fetchAodnDiscoveryParameterVocabs()) { - if (topLevelVocab.has("narrower") && !topLevelVocab.get("narrower").isEmpty()) { - for (JsonNode secondLevelVocab : topLevelVocab.get("narrower")) { - String secondLevelVocabLabel = secondLevelVocab.get("label").asText(); - if (secondLevelVocab.has("narrower") && !secondLevelVocab.get("narrower").isEmpty()) { - for (JsonNode bottomLevelVocab : secondLevelVocab.get("narrower")) { + for (CategoryVocabModel topLevelVocab : cacheArdcVocabsUtils.getCachedData()) { + if (!topLevelVocab.getNarrower().isEmpty()) { + for (CategoryVocabModel secondLevelVocab : topLevelVocab.getNarrower()) { + String secondLevelVocabLabel = secondLevelVocab.getLabel(); + if (!secondLevelVocab.getNarrower().isEmpty()) { + for (CategoryVocabModel bottomLevelVocab : secondLevelVocab.getNarrower()) { // map the original values to a ConceptModel object for doing comparison ConceptModel bottomConcept = ConceptModel.builder() - .id(bottomLevelVocab.get("label").asText()) - .url(bottomLevelVocab.get("about").asText()) + .id(bottomLevelVocab.getLabel()) + .url(bottomLevelVocab.getAbout()) .build(); // Compare with themes' concepts if (themesMatchConcept(themes, bottomConcept)) { diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java new file mode 100644 index 00000000..7fe25165 --- /dev/null +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java @@ -0,0 +1,4 @@ +package au.org.aodn.esindexer.utils; + +public class CacheArdcVocabsUtils { +} diff --git a/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java b/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java index 1cbf7673..9fe2583a 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java @@ -1,5 +1,6 @@ package au.org.aodn.esindexer.utils; +import au.org.aodn.ardcvocabs.model.CategoryVocabModel; import au.org.aodn.esindexer.abstracts.OgcApiRequestEntityCreator; import au.org.aodn.stac.model.ConceptModel; import au.org.aodn.stac.model.ThemesModel; @@ -57,7 +58,7 @@ void setup() throws IOException { @Test void testFetchAodnDiscoveryParameterVocabs() { - JsonNode results = aodnDiscoveryParameterVocabUtils.fetchAodnDiscoveryParameterVocabs(); + List results = aodnDiscoveryParameterVocabUtils.fetchAodnDiscoveryParameterVocabs(); // Verification assertEquals(33, results.size()); } From 9db18734b710beaaaac08032c3662a01345c7ade Mon Sep 17 00:00:00 2001 From: Viet Nguyen Date: Wed, 17 Apr 2024 23:15:31 +1000 Subject: [PATCH 07/19] Process ARDC APIs, refactor, and add tests --- indexer/pom.xml | 5 ++ .../esindexer/configuration/AppConstants.java | 3 + .../esindexer/configuration/WebMvcConfig.java | 16 +++++ .../AodnDiscoveryParameterVocabService.java | 10 +-- .../esindexer/service/IndexerServiceImpl.java | 5 +- .../esindexer/utils/CacheArdcVocabsUtils.java | 47 ++++++++++++++ indexer/src/main/resources/application.yaml | 3 - .../AodnDiscoveryParameterVocabUtilsTest.java | 63 +++++-------------- pom.xml | 1 + 9 files changed, 94 insertions(+), 59 deletions(-) diff --git a/indexer/pom.xml b/indexer/pom.xml index 9c6354c6..70365e96 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -197,6 +197,11 @@ geonetwork ${parent.version} + + au.org.aodn + ardcvocabs + ${parent.version} + au.org.aodn stacmodel diff --git a/indexer/src/main/java/au/org/aodn/esindexer/configuration/AppConstants.java b/indexer/src/main/java/au/org/aodn/esindexer/configuration/AppConstants.java index 0916e35c..869a6834 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/configuration/AppConstants.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/configuration/AppConstants.java @@ -15,4 +15,7 @@ public interface AppConstants { "a", "an", "and", "are", "as", "at", "be", "by", "for", "from", "has", "he", "in", "is", "it", "its", "of", "on", "that", "the", "to", "was", "were", "will", "with" ); + + String AODN_DISCOVERY_PARAMETER_VOCAB_API = "https://vocabs.ardc.edu.au/repository/api/lda/aodn"; + String AODN_DISCOVERY_CATEGORIES_CACHE = "parameter_categories"; } diff --git a/indexer/src/main/java/au/org/aodn/esindexer/configuration/WebMvcConfig.java b/indexer/src/main/java/au/org/aodn/esindexer/configuration/WebMvcConfig.java index d59ce19f..d8eed21a 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/configuration/WebMvcConfig.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/configuration/WebMvcConfig.java @@ -1,25 +1,41 @@ package au.org.aodn.esindexer.configuration; +import au.org.aodn.ardcvocabs.service.ArdcVocabsService; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.cache.concurrent.ConcurrentMapCacheManager; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.http.MediaType; import org.springframework.http.converter.HttpMessageConverter; import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; +import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.web.client.RestTemplate; import javax.annotation.PostConstruct; import java.util.Arrays; @Configuration +@ComponentScan("au.org.aodn.ardcvocabs") +@EnableCaching +@EnableScheduling public class WebMvcConfig { @Autowired protected ObjectMapper objectMapper; + @Autowired + protected ArdcVocabsService ardcVocabsService; + + @Bean + public ConcurrentMapCacheManager cacheManager() { + return new ConcurrentMapCacheManager(AppConstants.AODN_DISCOVERY_CATEGORIES_CACHE); + } + @PostConstruct public void init() { JavaTimeModule module = new JavaTimeModule(); diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/AodnDiscoveryParameterVocabService.java b/indexer/src/main/java/au/org/aodn/esindexer/service/AodnDiscoveryParameterVocabService.java index 2c356f9e..e208e646 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/AodnDiscoveryParameterVocabService.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/AodnDiscoveryParameterVocabService.java @@ -6,13 +6,14 @@ import au.org.aodn.stac.model.ConceptModel; import au.org.aodn.stac.model.ThemesModel; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; +import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; import java.util.ArrayList; import java.util.List; -@Component + +@Service public class AodnDiscoveryParameterVocabService { @Autowired RestTemplate restTemplate; @@ -20,6 +21,7 @@ public class AodnDiscoveryParameterVocabService { @Autowired OgcApiRequestEntityCreator ogcApiRequestEntityCreator; + @Autowired CacheArdcVocabsUtils cacheArdcVocabsUtils; protected boolean themesMatchConcept(List themes, ConceptModel concept) { @@ -42,10 +44,10 @@ public List getAodnDiscoveryCategories(List themes) { List results = new ArrayList<>(); // Iterate over the top-level vocabularies for (CategoryVocabModel topLevelVocab : cacheArdcVocabsUtils.getCachedData()) { - if (!topLevelVocab.getNarrower().isEmpty()) { + if (topLevelVocab.getNarrower() != null && !topLevelVocab.getNarrower().isEmpty()) { for (CategoryVocabModel secondLevelVocab : topLevelVocab.getNarrower()) { String secondLevelVocabLabel = secondLevelVocab.getLabel(); - if (!secondLevelVocab.getNarrower().isEmpty()) { + if (secondLevelVocab.getNarrower() != null && !secondLevelVocab.getNarrower().isEmpty()) { for (CategoryVocabModel bottomLevelVocab : secondLevelVocab.getNarrower()) { // map the original values to a ConceptModel object for doing comparison ConceptModel bottomConcept = ConceptModel.builder() diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java b/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java index b51616eb..005f42c6 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java @@ -1,6 +1,5 @@ package au.org.aodn.esindexer.service; -import au.org.aodn.esindexer.utils.AodnDiscoveryParameterVocabUtils; import au.org.aodn.stac.model.StacCollectionModel; import au.org.aodn.esindexer.configuration.AppConstants; import au.org.aodn.esindexer.exception.*; @@ -60,7 +59,7 @@ public class IndexerServiceImpl implements IndexerService { RankingService rankingService; @Autowired - AodnDiscoveryParameterVocabUtils aodnDiscoveryParameterVocabUtils; + AodnDiscoveryParameterVocabService aodnDiscoveryParameterVocabService; private static final Logger logger = LogManager.getLogger(IndexerServiceImpl.class); @@ -155,7 +154,7 @@ protected StacCollectionModel getMappedMetadataValues(String metadataValues) thr stacCollectionModel.setTitleSuggest(stacCollectionModel.getTitle()); // set AODN Discovery Categories - List aodnDiscoveryCategories = aodnDiscoveryParameterVocabUtils.getAodnDiscoveryCategories(stacCollectionModel.getThemes()); + List aodnDiscoveryCategories = aodnDiscoveryParameterVocabService.getAodnDiscoveryCategories(stacCollectionModel.getThemes()); stacCollectionModel.getSummaries().setAodnDiscoveryCategories(aodnDiscoveryCategories); return stacCollectionModel; diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java index 7fe25165..66780dd5 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java @@ -1,4 +1,51 @@ package au.org.aodn.esindexer.utils; +import au.org.aodn.ardcvocabs.model.CategoryVocabModel; +import au.org.aodn.ardcvocabs.service.ArdcVocabsService; +import au.org.aodn.esindexer.configuration.AppConstants; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.List; + +@Slf4j +@Component public class CacheArdcVocabsUtils { + @Value(AppConstants.AODN_DISCOVERY_PARAMETER_VOCAB_API) + protected String vocabApi; + + @Autowired + ArdcVocabsService ardcVocabsService; + + /* + call it after the bean is created to populate the cache + */ + @PostConstruct + @Cacheable(AppConstants.AODN_DISCOVERY_CATEGORIES_CACHE) + public List getCachedData() { + return fetchData(); + } + + protected List fetchData() { + log.info("Fetching AODN Discovery Parameter Vocabularies"); + return ardcVocabsService.getParameterCategory(vocabApi); + } + + // refresh every day at midnight + @Scheduled(cron = "0 0 0 * * *") + public void refreshCache() { + log.info("Refreshing AODN Discovery Parameter Vocabularies cache"); + clearCache(); + } + + @CacheEvict(value = AppConstants.AODN_DISCOVERY_CATEGORIES_CACHE, allEntries = true) + public void clearCache() { + // Intentionally empty; the annotation does the job + } } diff --git a/indexer/src/main/resources/application.yaml b/indexer/src/main/resources/application.yaml index 3f2f0203..28568194 100644 --- a/indexer/src/main/resources/application.yaml +++ b/indexer/src/main/resources/application.yaml @@ -38,9 +38,6 @@ geonetwork: index: "records" endpoint: /geonetwork/srv/api/search -ogc-api: - host: http://localhost:8082 - springdoc: api-docs: enabled: true diff --git a/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java b/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java index 9fe2583a..c5ef0c4a 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java @@ -1,67 +1,32 @@ package au.org.aodn.esindexer.utils; -import au.org.aodn.ardcvocabs.model.CategoryVocabModel; -import au.org.aodn.esindexer.abstracts.OgcApiRequestEntityCreator; +import au.org.aodn.ardcvocabs.service.ArdcVocabsService; +import au.org.aodn.esindexer.service.AodnDiscoveryParameterVocabService; import au.org.aodn.stac.model.ConceptModel; import au.org.aodn.stac.model.ThemesModel; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.*; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.http.HttpEntity; -import org.springframework.http.ResponseEntity; -import org.springframework.util.ResourceUtils; -import org.springframework.web.client.RestTemplate; -import org.springframework.http.MediaType; -import org.springframework.http.HttpMethod; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.*; +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles("test") +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) @ExtendWith(MockitoExtension.class) public class AodnDiscoveryParameterVocabUtilsTest { - @Mock - private RestTemplate restTemplate; - - @Mock - private OgcApiRequestEntityCreator ogcApiRequestEntityCreator; - @InjectMocks - AodnDiscoveryParameterVocabUtils aodnDiscoveryParameterVocabUtils; + @Autowired + ArdcVocabsService ardcVocabsService; - @BeforeEach - void setup() throws IOException { - File f = ResourceUtils.getFile("classpath:canned/aodn_discovery_parameter_vocabs.json"); - String jsonString = new String(Files.readAllBytes(f.toPath())); - ObjectMapper objectMapper = new ObjectMapper(); - JsonNode responseJsonNode = objectMapper.readTree(jsonString); - HttpEntity mockHttpEntity = new HttpEntity<>(MediaType.APPLICATION_JSON_VALUE); - ResponseEntity mockResponseEntity = ResponseEntity.ok(responseJsonNode); - - when(ogcApiRequestEntityCreator.getRequestEntity(MediaType.APPLICATION_JSON, null)).thenReturn(mockHttpEntity); - when(restTemplate.exchange( - anyString(), - eq(HttpMethod.GET), - eq(mockHttpEntity), - eq(JsonNode.class) - )).thenReturn(mockResponseEntity); - } - - @Test - void testFetchAodnDiscoveryParameterVocabs() { - List results = aodnDiscoveryParameterVocabUtils.fetchAodnDiscoveryParameterVocabs(); - // Verification - assertEquals(33, results.size()); - } + @Autowired + AodnDiscoveryParameterVocabService aodnDiscoveryParameterVocabService; @Test void testGetAodnDiscoveryCategories() { @@ -79,7 +44,7 @@ void testGetAodnDiscoveryCategories() { ); // Perform the test - List categories = aodnDiscoveryParameterVocabUtils.getAodnDiscoveryCategories(themes); + List categories = aodnDiscoveryParameterVocabService.getAodnDiscoveryCategories(themes); // Assertions assertNotNull(categories); diff --git a/pom.xml b/pom.xml index 685d7a99..d237c68b 100644 --- a/pom.xml +++ b/pom.xml @@ -20,6 +20,7 @@ geonetwork stacmodel indexer + ardcvocabs From f08cdb40fc20813c349ef27b0b8240750584ad3a Mon Sep 17 00:00:00 2001 From: Viet Nguyen Date: Wed, 17 Apr 2024 23:18:20 +1000 Subject: [PATCH 08/19] minor updates --- .../src/main/java/au/org/aodn/ardcvocabs/ArdcVocabs.java | 1 - .../main/java/au/org/aodn/ardcvocabs/configuration/Config.java | 3 --- .../java/au/org/aodn/ardcvocabs/model/CategoryVocabModel.java | 1 - 3 files changed, 5 deletions(-) diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/ArdcVocabs.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/ArdcVocabs.java index b1fd729d..e2644c4d 100644 --- a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/ArdcVocabs.java +++ b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/ArdcVocabs.java @@ -7,7 +7,6 @@ import java.util.TimeZone; @SpringBootApplication - public class ArdcVocabs { @PostConstruct void init() { diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/Config.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/Config.java index 751b220e..f7ab1184 100644 --- a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/Config.java +++ b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/Config.java @@ -7,12 +7,9 @@ import org.springframework.cache.annotation.EnableCaching; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.web.client.RestTemplate; @Configuration -@EnableCaching -@EnableScheduling public class Config { @Autowired diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/model/CategoryVocabModel.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/model/CategoryVocabModel.java index f30c495f..94bd7807 100644 --- a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/model/CategoryVocabModel.java +++ b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/model/CategoryVocabModel.java @@ -5,7 +5,6 @@ import lombok.Setter; import java.util.List; -import java.util.Map; /** * This is the model class for http://vocabs.ardc.edu.au/repository/api/lda/aodn/aodn-parameter-category-vocabulary/ From a558077f369339362a7e80f1e9caaccb24825b00 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Date: Wed, 17 Apr 2024 23:32:04 +1000 Subject: [PATCH 09/19] Remove mockito --- .../esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java b/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java index c5ef0c4a..8776e4e8 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java @@ -5,8 +5,6 @@ import au.org.aodn.stac.model.ConceptModel; import au.org.aodn.stac.model.ThemesModel; import org.junit.jupiter.api.*; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ActiveProfiles; @@ -19,7 +17,6 @@ @ActiveProfiles("test") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @TestInstance(TestInstance.Lifecycle.PER_CLASS) -@ExtendWith(MockitoExtension.class) public class AodnDiscoveryParameterVocabUtilsTest { @Autowired From 5f444f3185ca7d31245d41e9c4655c16eff0b264 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Date: Thu, 18 Apr 2024 00:10:42 +1000 Subject: [PATCH 10/19] Re-populate cache after clearing the old one --- .../org/aodn/esindexer/utils/CacheArdcVocabsUtils.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java index 66780dd5..3d325c85 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java @@ -8,6 +8,8 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.cache.annotation.CacheEvict; import org.springframework.cache.annotation.Cacheable; +import org.springframework.context.annotation.Scope; +import org.springframework.context.annotation.ScopedProxyMode; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @@ -16,6 +18,8 @@ @Slf4j @Component +// create and inject a stub proxy to self due to the circular reference +@Scope(proxyMode = ScopedProxyMode.TARGET_CLASS) public class CacheArdcVocabsUtils { @Value(AppConstants.AODN_DISCOVERY_PARAMETER_VOCAB_API) protected String vocabApi; @@ -23,6 +27,10 @@ public class CacheArdcVocabsUtils { @Autowired ArdcVocabsService ardcVocabsService; + // self-injection to avoid self-invocation problems when calling the cachable method within the same bean + @Autowired + CacheArdcVocabsUtils self; + /* call it after the bean is created to populate the cache */ @@ -42,6 +50,8 @@ protected List fetchData() { public void refreshCache() { log.info("Refreshing AODN Discovery Parameter Vocabularies cache"); clearCache(); + // call the cachable method to re-populate the cache + self.getCachedData(); } @CacheEvict(value = AppConstants.AODN_DISCOVERY_CATEGORIES_CACHE, allEntries = true) From 184a771c7548e95d79eeba9a92b6395fe1729ce9 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Date: Fri, 19 Apr 2024 12:15:39 +1000 Subject: [PATCH 11/19] Switch to --- .../esindexer/service/IndexerServiceImpl.java | 7 +++- .../service/StacCollectionMapperService.java | 4 +- .../aodn/esindexer/utils/GeometryBase.java | 39 +++++++++++-------- .../aodn/esindexer/utils/GeometryUtils.java | 12 +++++- .../org/aodn/esindexer/utils/StacUtils.java | 4 +- .../portal_records_index_schema.json | 8 +--- .../test/resources/canned/sample4_stac.json | 2 +- .../org/aodn/stac/model/SummariesModel.java | 6 +-- 8 files changed, 50 insertions(+), 32 deletions(-) diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java b/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java index 005f42c6..f3786816 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java @@ -153,9 +153,10 @@ protected StacCollectionModel getMappedMetadataValues(String metadataValues) thr stacCollectionModel.setTitleSuggest(stacCollectionModel.getTitle()); - // set AODN Discovery Categories List aodnDiscoveryCategories = aodnDiscoveryParameterVocabService.getAodnDiscoveryCategories(stacCollectionModel.getThemes()); - stacCollectionModel.getSummaries().setAodnDiscoveryCategories(aodnDiscoveryCategories); + if (!aodnDiscoveryCategories.isEmpty()) { + stacCollectionModel.getSummaries().setDiscoveryCategories(aodnDiscoveryCategories); + } return stacCollectionModel; } @@ -311,6 +312,8 @@ public ResponseEntity indexAllMetadataRecordsFromGeoNetwork(boolean conf logger.info("Finished bulk indexing records to index: " + indexName); } + // TODO now processing for record_suggestions index + return ResponseEntity.status(HttpStatus.OK).body(result.toString()); } } diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java b/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java index 4ea3060d..a6fc2003 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java @@ -596,7 +596,9 @@ protected String mapContactsRole(CIResponsibilityType2 ciResponsibility) { protected String mapContactsOrganization(AbstractCIPartyPropertyType2 party) { String organisationString = ""; if (party.getAbstractCIParty() != null) { - organisationString = party.getAbstractCIParty().getValue().getName().getCharacterString().getValue().toString(); + if (party.getAbstractCIParty().getValue().getName().getCharacterString() != null) { + organisationString = party.getAbstractCIParty().getValue().getName().getCharacterString().getValue().toString(); + } } return organisationString; } diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryBase.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryBase.java index f1073b49..3a7094f5 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryBase.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryBase.java @@ -126,22 +126,14 @@ public static List findPolygonsFromEXGeographicBoundingBoxType(String r .toList(); for (EXGeographicBoundingBoxType bbt : input) { - Double east = bbt.getEastBoundLongitude().getDecimal().doubleValue(); - Double west = bbt.getWestBoundLongitude().getDecimal().doubleValue(); - Double north = bbt.getNorthBoundLatitude().getDecimal().doubleValue(); - Double south = bbt.getSouthBoundLatitude().getDecimal().doubleValue(); - - // Define the coordinates for the bounding box - Coordinate[] coordinates = new Coordinate[]{ - new Coordinate(west, south), - new Coordinate(east, south), - new Coordinate(east, north), - new Coordinate(west, north), - new Coordinate(west, south) // Closing the ring - }; - - Polygon polygon = geoJsonFactory.createPolygon(coordinates); - polygons.add(polygon); + if (bbt.getWestBoundLongitude().getDecimal() == null || bbt.getEastBoundLongitude().getDecimal() == null || bbt.getNorthBoundLatitude().getDecimal() == null || bbt.getSouthBoundLatitude().getDecimal() == null) { + logger.warn("Invalid BBOX found for findPolygonsFromEXGeographicBoundingBoxType using CRS {}", rawCRS); + } else { + logger.debug("BBOX found for findPolygonsFromEXGeographicBoundingBoxType using CRS {}", rawCRS); + Coordinate[] coordinates = getCoordinates(bbt); + Polygon polygon = geoJsonFactory.createPolygon(coordinates); + polygons.add(polygon); + } } } @@ -153,4 +145,19 @@ public static List findPolygonsFromEXGeographicBoundingBoxType(String r return null; } } + + protected static Coordinate[] getCoordinates(EXGeographicBoundingBoxType bbt) { + double east = bbt.getEastBoundLongitude().getDecimal().doubleValue(); + double west = bbt.getWestBoundLongitude().getDecimal().doubleValue(); + double north = bbt.getNorthBoundLatitude().getDecimal().doubleValue(); + double south = bbt.getSouthBoundLatitude().getDecimal().doubleValue(); + // Define the coordinates for the bounding box + return new Coordinate[]{ + new Coordinate(west, south), + new Coordinate(east, south), + new Coordinate(east, north), + new Coordinate(west, north), + new Coordinate(west, south) // Closing the ring + }; + } } diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java index a0912d5a..feb8a31f 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java @@ -53,13 +53,21 @@ public static Map createGeometryFromEXGeographicBoundingBoxType(List raw // The return polygon is in EPSG:4326, so we can call createGeoJson directly //TODO: avoid hardcode CRS, get it from document List polygons = GeometryBase.findPolygonsFromEXGeographicBoundingBoxType(GeometryBase.COORDINATE_SYSTEM_CRS84, rawInput); - return createGeoJson(polygons); + + if (polygons != null && !polygons.isEmpty()) { + return createGeoJson(polygons); + } + return null; } public static Map createGeometryFromFromEXBoundingPolygonType(List rawInput) { // The return polygon is in EPSG:4326, so we can call createGeoJson directly //TODO: avoid hardcode CRS, get it from document List polygons = GeometryBase.findPolygonsFromEXBoundingPolygonType(GeometryBase.COORDINATE_SYSTEM_CRS84, rawInput); - return createGeoJson(polygons); + + if (polygons != null && !polygons.isEmpty()) { + return createGeoJson(polygons); + } + return null; } } diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/StacUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/StacUtils.java index 32f81835..9db0b327 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/StacUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/StacUtils.java @@ -50,7 +50,9 @@ public static List> createStacBBox(List polygons) { for(Polygon polygon : polygons) { // Add polygon one by one to extend the overall bounding box area, this is requirement // of STAC to have an overall bounding box of all smaller area as the first bbox in the list. - envelope = GeometryBase.calculateBoundingBox(envelope, polygon); + if (polygon != null) { + envelope.expandToInclude(polygon.getEnvelopeInternal()); + } } if (!polygons.isEmpty()) { diff --git a/indexer/src/main/resources/config_files/portal_records_index_schema.json b/indexer/src/main/resources/config_files/portal_records_index_schema.json index 1f9a5dfe..aa248c51 100644 --- a/indexer/src/main/resources/config_files/portal_records_index_schema.json +++ b/indexer/src/main/resources/config_files/portal_records_index_schema.json @@ -30,11 +30,8 @@ "type": { "type": "text" }, "title": { "type": "text" }, "title_suggest": { - "type": "completion", - "analyzer": "my_stop_analyzer", - "preserve_separators": true, - "preserve_position_increments": true, - "max_input_length": 1000 + "type": "search_as_you_type", + "analyzer": "my_stop_analyzer" }, "keywords": { "type": "nested", @@ -76,7 +73,6 @@ "properties" : { "score": { "type": "long" }, "status": { "type": "text" }, - "aodn_discovery_categories" : { "type": "text" }, "scope" : { "type": "nested", "properties" : { diff --git a/indexer/src/test/resources/canned/sample4_stac.json b/indexer/src/test/resources/canned/sample4_stac.json index bf489d11..0dbc743f 100644 --- a/indexer/src/test/resources/canned/sample4_stac.json +++ b/indexer/src/test/resources/canned/sample4_stac.json @@ -22,7 +22,7 @@ "start" : "1870-07-16T14:10:44Z", "end" : "2013-06-16T14:00:00Z" } ], - "aodn_discovery_categories" : [ "Alkalinity", "Temperature", "Salinity" ] + "discovery_categories" : [ "Alkalinity", "Temperature", "Salinity" ] }, "contacts" : [ { "emails" : [ "Andrew.Lenton@csiro.au" ], diff --git a/stacmodel/src/main/java/au/org/aodn/stac/model/SummariesModel.java b/stacmodel/src/main/java/au/org/aodn/stac/model/SummariesModel.java index ad644666..99004bf1 100644 --- a/stacmodel/src/main/java/au/org/aodn/stac/model/SummariesModel.java +++ b/stacmodel/src/main/java/au/org/aodn/stac/model/SummariesModel.java @@ -42,8 +42,8 @@ public class SummariesModel { protected List> temporal; /** - * second-level AODN discovery parameter vocabularies + * Discovery categories */ - @JsonProperty("aodn_discovery_categories") - protected List aodnDiscoveryCategories; + @JsonProperty("discovery_categories") + protected List discoveryCategories; } From 48810c61b84a60942d5034738cda90e9d4e2d285 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Date: Fri, 19 Apr 2024 16:27:23 +1000 Subject: [PATCH 12/19] Store discovery categories in a separate index --- .../esindexer/configuration/AppConstants.java | 1 + .../AodnDiscoveryParameterVocabService.java | 21 ++-- .../service/ElasticSearchIndexService.java | 71 +++++++++++ .../esindexer/service/IndexerServiceImpl.java | 58 ++------- .../esindexer/utils/CacheArdcVocabsUtils.java | 111 ++++++++++++++++-- indexer/src/main/resources/application.yaml | 2 + ...iscovery_parameter_vocabularies_index.json | 73 ++++++++++++ .../au/org/aodn/esindexer/BaseTestClass.java | 7 +- .../service/GeoNetworkServiceTests.java | 6 +- .../service/IndexerServiceTests.java | 16 ++- .../service/RankingServiceTests.java | 6 +- .../AodnDiscoveryParameterVocabUtilsTest.java | 4 +- .../src/test/resources/application-test.yaml | 2 + 13 files changed, 294 insertions(+), 84 deletions(-) create mode 100644 indexer/src/main/java/au/org/aodn/esindexer/service/ElasticSearchIndexService.java create mode 100644 indexer/src/main/resources/config_files/aodn_discovery_parameter_vocabularies_index.json diff --git a/indexer/src/main/java/au/org/aodn/esindexer/configuration/AppConstants.java b/indexer/src/main/java/au/org/aodn/esindexer/configuration/AppConstants.java index 869a6834..7ac082b3 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/configuration/AppConstants.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/configuration/AppConstants.java @@ -18,4 +18,5 @@ public interface AppConstants { String AODN_DISCOVERY_PARAMETER_VOCAB_API = "https://vocabs.ardc.edu.au/repository/api/lda/aodn"; String AODN_DISCOVERY_CATEGORIES_CACHE = "parameter_categories"; + String AODN_DISCOVERY_PARAMETER_VOCABULARIES_MAPPING_JSON_FILE = "aodn_discovery_parameter_vocabularies_index.json"; } diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/AodnDiscoveryParameterVocabService.java b/indexer/src/main/java/au/org/aodn/esindexer/service/AodnDiscoveryParameterVocabService.java index e208e646..a91d5323 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/AodnDiscoveryParameterVocabService.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/AodnDiscoveryParameterVocabService.java @@ -1,14 +1,15 @@ package au.org.aodn.esindexer.service; -import au.org.aodn.ardcvocabs.model.CategoryVocabModel; import au.org.aodn.esindexer.abstracts.OgcApiRequestEntityCreator; import au.org.aodn.esindexer.utils.CacheArdcVocabsUtils; import au.org.aodn.stac.model.ConceptModel; import au.org.aodn.stac.model.ThemesModel; +import com.fasterxml.jackson.databind.JsonNode; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; +import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -40,19 +41,19 @@ comparing by hashcode (id and url) of the concept object return false; } - public List getAodnDiscoveryCategories(List themes) { + public List getAodnDiscoveryCategories(List themes) throws IOException { List results = new ArrayList<>(); // Iterate over the top-level vocabularies - for (CategoryVocabModel topLevelVocab : cacheArdcVocabsUtils.getCachedData()) { - if (topLevelVocab.getNarrower() != null && !topLevelVocab.getNarrower().isEmpty()) { - for (CategoryVocabModel secondLevelVocab : topLevelVocab.getNarrower()) { - String secondLevelVocabLabel = secondLevelVocab.getLabel(); - if (secondLevelVocab.getNarrower() != null && !secondLevelVocab.getNarrower().isEmpty()) { - for (CategoryVocabModel bottomLevelVocab : secondLevelVocab.getNarrower()) { + for (JsonNode topLevelVocab : cacheArdcVocabsUtils.getDiscoveryCategories()) { + if (topLevelVocab.has("narrower") && !topLevelVocab.get("narrower").isEmpty()) { + for (JsonNode secondLevelVocab : topLevelVocab.get("narrower")) { + String secondLevelVocabLabel = secondLevelVocab.get("label").asText(); + if (secondLevelVocab.has("narrower") && !secondLevelVocab.get("narrower").isEmpty()) { + for (JsonNode bottomLevelVocab : secondLevelVocab.get("narrower")) { // map the original values to a ConceptModel object for doing comparison ConceptModel bottomConcept = ConceptModel.builder() - .id(bottomLevelVocab.getLabel()) - .url(bottomLevelVocab.getAbout()) + .id(bottomLevelVocab.get("label").asText()) + .url(bottomLevelVocab.get("about").asText()) .build(); // Compare with themes' concepts if (themesMatchConcept(themes, bottomConcept)) { diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/ElasticSearchIndexService.java b/indexer/src/main/java/au/org/aodn/esindexer/service/ElasticSearchIndexService.java new file mode 100644 index 00000000..1c5a4082 --- /dev/null +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/ElasticSearchIndexService.java @@ -0,0 +1,71 @@ +package au.org.aodn.esindexer.service; + +import au.org.aodn.esindexer.exception.CreateIndexException; +import au.org.aodn.esindexer.exception.DeleteIndexException; +import au.org.aodn.esindexer.exception.IndexNotFoundException; +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch.indices.CreateIndexRequest; +import co.elastic.clients.elasticsearch.indices.CreateIndexResponse; +import co.elastic.clients.transport.endpoints.BooleanResponse; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ClassPathResource; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.io.InputStream; + + +@Slf4j +@Service +public class ElasticSearchIndexService { + + @Autowired + ElasticsearchClient portalElasticsearchClient; + + protected void deleteIndexStore(String indexName) { + try { + BooleanResponse response = portalElasticsearchClient.indices().exists(b -> b.index(indexName)); + if (response.value()) { + log.info("Deleting index: " + indexName); + portalElasticsearchClient.indices().delete(b -> b.index(indexName)); + log.info("Index: " + indexName + " deleted"); + } + } catch (ElasticsearchException | IOException e) { + throw new DeleteIndexException("Failed to delete index: " + indexName + " | " + e.getMessage()); + } + } + + public void createIndexFromMappingJSONFile(String indexMappingFile, String indexName) { + + // AppConstants.PORTAL_RECORDS_MAPPING_JSON_FILE + ClassPathResource resource = new ClassPathResource("config_files/" + indexMappingFile); + + // delete the existing index if found first + this.deleteIndexStore(indexName); + + try (InputStream input = resource.getInputStream()) { + log.info("Creating index: " + indexName); + CreateIndexRequest req = CreateIndexRequest.of(b -> b + .index(indexName) + .withJson(input) + ); + CreateIndexResponse response = portalElasticsearchClient.indices().create(req); + log.info(response.toString()); + } + catch (ElasticsearchException | IOException e) { + throw new CreateIndexException("Failed to elastic index from schema file: " + indexName + " | " + e.getMessage()); + } + } + + public long getDocumentsCount(String indexName) { + try { + return portalElasticsearchClient.count(s -> s + .index(indexName) + ).count(); + } catch (ElasticsearchException | IOException e) { + throw new IndexNotFoundException("Failed to get documents count from index: " + indexName + " | " + e.getMessage()); + } + } +} diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java b/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java index f3786816..1de9e5b9 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java @@ -12,10 +12,7 @@ import co.elastic.clients.elasticsearch.core.search.Hit; import co.elastic.clients.elasticsearch.core.search.TotalHits; import co.elastic.clients.elasticsearch.core.search.TotalHitsRelation; -import co.elastic.clients.elasticsearch.indices.CreateIndexRequest; -import co.elastic.clients.elasticsearch.indices.CreateIndexResponse; import co.elastic.clients.json.JsonData; -import co.elastic.clients.transport.endpoints.BooleanResponse; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import jakarta.xml.bind.JAXBException; @@ -25,7 +22,6 @@ import org.opengis.referencing.operation.TransformException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.io.ClassPathResource; import org.springframework.http.*; import org.springframework.stereotype.Service; import java.io.ByteArrayInputStream; @@ -43,6 +39,9 @@ public class IndexerServiceImpl implements IndexerService { @Autowired ElasticsearchClient portalElasticsearchClient; + @Autowired + ElasticSearchIndexService elasticSearchIndexService; + @Value("${elasticsearch.index.name}") private String indexName; @@ -63,16 +62,6 @@ public class IndexerServiceImpl implements IndexerService { private static final Logger logger = LogManager.getLogger(IndexerServiceImpl.class); - protected long getDocumentsCount() { - try { - return portalElasticsearchClient.count(s -> s - .index(indexName) - ).count(); - } catch (ElasticsearchException | IOException e) { - throw new IndexNotFoundException("Failed to get documents count from index: " + indexName + " | " + e.getMessage()); - } - } - public Hit getDocumentByUUID(String uuid) throws IOException { try { SearchResponse response = portalElasticsearchClient.search(s -> s @@ -119,19 +108,6 @@ protected boolean isMetadataPublished(String uuid) { } } - protected void deleteIndexStore() { - try { - BooleanResponse response = portalElasticsearchClient.indices().exists(b -> b.index(indexName)); - if (response.value()) { - logger.info("Deleting index: " + indexName); - portalElasticsearchClient.indices().delete(b -> b.index(indexName)); - logger.info("Index: " + indexName + " deleted"); - } - } catch (ElasticsearchException | IOException e) { - throw new DeleteIndexException("Failed to delete index: " + indexName + " | " + e.getMessage()); - } - } - protected StacCollectionModel getMappedMetadataValues(String metadataValues) throws IOException, FactoryException, TransformException, JAXBException { MDMetadataType metadataType = jaxbUtils.unmarshal(metadataValues); @@ -151,6 +127,7 @@ protected StacCollectionModel getMappedMetadataValues(String metadataValues) thr stacCollectionModel.getSummaries().setScore(score); + // TODO: in future, blah blah here stacCollectionModel.setTitleSuggest(stacCollectionModel.getTitle()); List aodnDiscoveryCategories = aodnDiscoveryParameterVocabService.getAodnDiscoveryCategories(stacCollectionModel.getThemes()); @@ -171,19 +148,19 @@ public ResponseEntity indexMetadata(String metadataValues) { // count portal index documents, or create index if not found from defined mapping JSON file try { - portalIndexDocumentsCount = this.getDocumentsCount(); + portalIndexDocumentsCount = elasticSearchIndexService.getDocumentsCount(indexName); // check if GeoNetwork instance has been reinstalled if (this.isGeoNetworkInstanceReinstalled(portalIndexDocumentsCount)) { logger.info("GeoNetwork instance has been reinstalled, recreating portal index: " + indexName); - this.createIndexFromMappingJSONFile(); + elasticSearchIndexService.createIndexFromMappingJSONFile(AppConstants.PORTAL_RECORDS_MAPPING_JSON_FILE, indexName); } else { // delete the existing document if found first this.deleteDocumentByUUID(uuid); } } catch (IndexNotFoundException e) { logger.info("Index: " + indexName + " not found, creating index"); - this.createIndexFromMappingJSONFile(); + elasticSearchIndexService.createIndexFromMappingJSONFile(AppConstants.PORTAL_RECORDS_MAPPING_JSON_FILE, indexName); } // index the metadata if it is published @@ -214,25 +191,6 @@ public ResponseEntity indexMetadata(String metadataValues) { } } - protected void createIndexFromMappingJSONFile() { - ClassPathResource resource = new ClassPathResource("config_files/" + AppConstants.PORTAL_RECORDS_MAPPING_JSON_FILE); - - // delete the existing index if found first - this.deleteIndexStore(); - - try (InputStream input = resource.getInputStream()) { - logger.info("Creating index: " + indexName); - CreateIndexRequest req = CreateIndexRequest.of(b -> b - .index(indexName) - .withJson(input) - ); - CreateIndexResponse response = portalElasticsearchClient.indices().create(req); - logger.info(response.toString()); - } - catch (ElasticsearchException | IOException e) { - throw new CreateIndexException("Failed to elastic index from schema file: " + indexName + " | " + e.getMessage()); - } - } public ResponseEntity deleteDocumentByUUID(String uuid) throws IOException { logger.info("Deleting document with UUID: " + uuid + " from index: " + indexName); @@ -264,7 +222,7 @@ public ResponseEntity indexAllMetadataRecordsFromGeoNetwork(boolean conf } // recreate index from mapping JSON file - this.createIndexFromMappingJSONFile(); + elasticSearchIndexService.createIndexFromMappingJSONFile(AppConstants.PORTAL_RECORDS_MAPPING_JSON_FILE, indexName); logger.info("Indexing all metadata records from GeoNetwork"); diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java index 3d325c85..8bac5ba6 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java @@ -3,6 +3,16 @@ import au.org.aodn.ardcvocabs.model.CategoryVocabModel; import au.org.aodn.ardcvocabs.service.ArdcVocabsService; import au.org.aodn.esindexer.configuration.AppConstants; +import au.org.aodn.esindexer.exception.DocumentNotFoundException; +import au.org.aodn.esindexer.service.ElasticSearchIndexService; +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch.core.*; +import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.annotation.PostConstruct; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -13,12 +23,14 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; +import java.io.IOException; +import java.util.ArrayList; import java.util.List; + @Slf4j @Component -// create and inject a stub proxy to self due to the circular reference +// create and inject a stub proxy to self due to the circular reference http://bit.ly/4aFvYtt @Scope(proxyMode = ScopedProxyMode.TARGET_CLASS) public class CacheArdcVocabsUtils { @Value(AppConstants.AODN_DISCOVERY_PARAMETER_VOCAB_API) @@ -31,27 +43,102 @@ public class CacheArdcVocabsUtils { @Autowired CacheArdcVocabsUtils self; + @Autowired + ElasticSearchIndexService elasticSearchIndexService; + + @Autowired + ElasticsearchClient portalElasticsearchClient; + + @Value("${elasticsearch.categories-index.name}") + String categoriesIndexName; + + @Autowired + ObjectMapper objectMapper; + + + protected void indexingDiscoveryCategoriesIndex(List categoryVocabModels) throws IOException { + BulkRequest.Builder bulkRequest = new BulkRequest.Builder(); + // recreate index from mapping JSON file + elasticSearchIndexService.createIndexFromMappingJSONFile(AppConstants.AODN_DISCOVERY_PARAMETER_VOCABULARIES_MAPPING_JSON_FILE, categoriesIndexName); + log.info("Indexing all categoryVocabModel to {}", categoriesIndexName); + for (CategoryVocabModel categoryVocabModel : categoryVocabModels) { + try { + // convert categoryVocabModel values to binary data + log.debug("Ingested json is {}", objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(categoryVocabModel)); + // send bulk request to Elasticsearch + bulkRequest.operations(op -> op + .index(idx -> idx + .index(categoriesIndexName) + .document(categoryVocabModel) + ) + ); + } catch (JsonProcessingException e) { + log.error("Failed to ingest categoryVocabModel to {}", categoriesIndexName); + throw new RuntimeException(e); + } + } + + BulkResponse result = portalElasticsearchClient.bulk(bulkRequest.build()); + + // Flush after insert, otherwise you need to wait for next auto-refresh. It is + // especially a problem with autotest, where assert happens very fast. + portalElasticsearchClient.indices().refresh(); + + // Log errors, if any + if (result.errors()) { + log.error("Bulk had errors"); + for (BulkResponseItem item: result.items()) { + if (item.error() != null) { + log.error("{} {}", item.error().reason(), item.error().causedBy()); + } + } + } else { + log.info("Finished bulk indexing categoryVocabModel items to index: " + categoriesIndexName); + } + log.info("Total documents in index: " + categoriesIndexName + " is " + elasticSearchIndexService.getDocumentsCount(categoriesIndexName)); + } + + /* - call it after the bean is created to populate the cache + fetch vocabularies from ARDC and index to discovery categories index once the bean is created */ @PostConstruct - @Cacheable(AppConstants.AODN_DISCOVERY_CATEGORIES_CACHE) - public List getCachedData() { - return fetchData(); + public void refreshDiscoveryCategoriesIndex() throws IOException { + log.info("Fetching AODN Discovery Parameter Vocabularies from ARDC"); + List categoryVocabModels = ardcVocabsService.getParameterCategory(vocabApi); + indexingDiscoveryCategoriesIndex(categoryVocabModels); } - protected List fetchData() { - log.info("Fetching AODN Discovery Parameter Vocabularies"); - return ardcVocabsService.getParameterCategory(vocabApi); + @Cacheable(AppConstants.AODN_DISCOVERY_CATEGORIES_CACHE) + public List getDiscoveryCategories() throws IOException { + List categories = new ArrayList<>(); + log.info("Fetching AODN Discovery Parameter Vocabularies from {}", categoriesIndexName); + try { + double totalHits = elasticSearchIndexService.getDocumentsCount(categoriesIndexName); + if (totalHits == 0) { + throw new DocumentNotFoundException("No documents found in " + categoriesIndexName); + } else { + SearchResponse response = portalElasticsearchClient.search(s -> s + .index(categoriesIndexName) + .size((int) totalHits), JsonNode.class + ); + response.hits().hits().forEach(hit -> { + categories.add(hit.source()); + }); + } + } catch (ElasticsearchException | IOException e) { + throw new IOException("Failed to get documents from " + categoriesIndexName + " | " + e.getMessage()); + } + return categories; } // refresh every day at midnight @Scheduled(cron = "0 0 0 * * *") - public void refreshCache() { + public void refreshCache() throws IOException { log.info("Refreshing AODN Discovery Parameter Vocabularies cache"); clearCache(); - // call the cachable method to re-populate the cache - self.getCachedData(); + this.refreshDiscoveryCategoriesIndex(); + self.getDiscoveryCategories(); } @CacheEvict(value = AppConstants.AODN_DISCOVERY_CATEGORIES_CACHE, allEntries = true) diff --git a/indexer/src/main/resources/application.yaml b/indexer/src/main/resources/application.yaml index 28568194..8b0d6453 100644 --- a/indexer/src/main/resources/application.yaml +++ b/indexer/src/main/resources/application.yaml @@ -30,6 +30,8 @@ management: elasticsearch: index: name: portal_records + categories-index: + name: discovery_category geonetwork: host: http://localhost:8080 diff --git a/indexer/src/main/resources/config_files/aodn_discovery_parameter_vocabularies_index.json b/indexer/src/main/resources/config_files/aodn_discovery_parameter_vocabularies_index.json new file mode 100644 index 00000000..a16e7731 --- /dev/null +++ b/indexer/src/main/resources/config_files/aodn_discovery_parameter_vocabularies_index.json @@ -0,0 +1,73 @@ +{ + "mappings": { + "dynamic": true, + "properties": { + "label": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "definition": { + "type": "text" + }, + "about": { + "type": "keyword" + }, + "broader": { + "type": "nested", + "properties": { + "label": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "about": { + "type": "keyword" + } + } + }, + "narrower": { + "type": "nested", + "properties": { + "label": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "about": { + "type": "keyword" + }, + "narrower": { + "type": "nested", + "properties": { + "label": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "about": { + "type": "keyword" + } + } + } + } + } + } + } +} diff --git a/indexer/src/test/java/au/org/aodn/esindexer/BaseTestClass.java b/indexer/src/test/java/au/org/aodn/esindexer/BaseTestClass.java index 756bd9af..84094928 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/BaseTestClass.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/BaseTestClass.java @@ -49,17 +49,14 @@ public class BaseTestClass { @Autowired protected ElasticsearchContainer container; - @Value("${elasticsearch.index.name}") - protected String INDEX_NAME; - @Autowired protected DockerComposeContainer dockerComposeContainer; - protected void clearElasticIndex() throws IOException { + protected void clearElasticIndex(String indexName) throws IOException { logger.debug("Clear elastic index"); try { client.deleteByQuery(f -> f - .index(INDEX_NAME) + .index(indexName) .query(QueryBuilders.matchAll().build()._toQuery()) ); // Must all, otherwise index is not rebuild immediately diff --git a/indexer/src/test/java/au/org/aodn/esindexer/service/GeoNetworkServiceTests.java b/indexer/src/test/java/au/org/aodn/esindexer/service/GeoNetworkServiceTests.java index e6f7070a..1ff29809 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/service/GeoNetworkServiceTests.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/service/GeoNetworkServiceTests.java @@ -7,6 +7,7 @@ import au.org.aodn.esindexer.exception.MetadataNotFoundException; import org.junit.jupiter.api.*; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ActiveProfiles; import org.springframework.util.ResourceUtils; @@ -28,6 +29,9 @@ public class GeoNetworkServiceTests extends BaseTestClass { @Autowired protected GeoNetworkServiceImpl geoNetworkService; + @Value("${elasticsearch.index.name}") + protected String INDEX_NAME; + @BeforeAll public void setup() { // Update the server for geonetwork RESTful URL @@ -39,7 +43,7 @@ public void setup() { @AfterEach public void clear() throws IOException { - clearElasticIndex(); + clearElasticIndex(INDEX_NAME); } /** * We need to make sure this works before you can do any meaningful transformation diff --git a/indexer/src/test/java/au/org/aodn/esindexer/service/IndexerServiceTests.java b/indexer/src/test/java/au/org/aodn/esindexer/service/IndexerServiceTests.java index 87fbc607..d03e456d 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/service/IndexerServiceTests.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/service/IndexerServiceTests.java @@ -7,8 +7,10 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.jupiter.api.*; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ActiveProfiles; +import org.testcontainers.shaded.org.checkerframework.checker.units.qual.A; import java.io.IOException; @@ -29,6 +31,12 @@ public class IndexerServiceTests extends BaseTestClass { @Autowired protected ObjectMapper objectMapper; + @Autowired + protected ElasticSearchIndexService elasticSearchIndexService; + + @Value("${elasticsearch.index.name}") + protected String INDEX_NAME; + @BeforeAll public void setup() { // Update the server for geonetwork RESTful URL @@ -40,7 +48,7 @@ public void setup() { @AfterEach public void clear() throws IOException { - clearElasticIndex(); + clearElasticIndex(INDEX_NAME); } @Test @@ -78,7 +86,7 @@ public void verifyGetDocumentCount() throws IOException { // ErrorCause: {"type":"illegal_argument_exception","reason":"Polygon self-intersection at lat=57.0 lon=-66.0"} // // So it will not insert correctly and result in 1 doc only - assertEquals("Doc count correct", 1L, indexerService.getDocumentsCount()); + assertEquals("Doc count correct", 1L, elasticSearchIndexService.getDocumentsCount(INDEX_NAME)); deleteRecord("9e5c3031-a026-48b3-a153-a70c2e2b78b9"); deleteRecord("830f9a83-ae6b-4260-a82a-24c4851f7119"); @@ -90,11 +98,11 @@ public void verifyDeleteDocumentByUUID() throws IOException { insertMetadataRecords("06b09398-d3d0-47dc-a54a-a745319fbece", "classpath:canned/sample3.xml"); indexerService.indexAllMetadataRecordsFromGeoNetwork(true); - assertEquals("Doc count correct", 2L, indexerService.getDocumentsCount()); + assertEquals("Doc count correct", 2L, elasticSearchIndexService.getDocumentsCount(INDEX_NAME)); // Only 2 doc in elastic, if we delete it then should be zero indexerService.deleteDocumentByUUID("830f9a83-ae6b-4260-a82a-24c4851f7119"); - assertEquals("Doc count correct", 1L, indexerService.getDocumentsCount()); + assertEquals("Doc count correct", 1L, elasticSearchIndexService.getDocumentsCount(INDEX_NAME)); deleteRecord("830f9a83-ae6b-4260-a82a-24c4851f7119"); deleteRecord("06b09398-d3d0-47dc-a54a-a745319fbece"); diff --git a/indexer/src/test/java/au/org/aodn/esindexer/service/RankingServiceTests.java b/indexer/src/test/java/au/org/aodn/esindexer/service/RankingServiceTests.java index 9c72b4b0..1090a9f4 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/service/RankingServiceTests.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/service/RankingServiceTests.java @@ -9,6 +9,7 @@ import org.junit.jupiter.api.*; import org.mockito.InjectMocks; import org.mockito.Spy; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ActiveProfiles; @@ -26,9 +27,12 @@ @TestInstance(TestInstance.Lifecycle.PER_CLASS) class RankingServiceTests extends BaseTestClass { + @Value("${elasticsearch.index.name}") + protected String INDEX_NAME; + @AfterAll public void clear() throws IOException { - super.clearElasticIndex(); + super.clearElasticIndex(INDEX_NAME); } @Spy diff --git a/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java b/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java index 8776e4e8..9154c968 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java @@ -8,6 +8,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ActiveProfiles; + +import java.io.IOException; import java.util.Arrays; import java.util.List; @@ -26,7 +28,7 @@ public class AodnDiscoveryParameterVocabUtilsTest { AodnDiscoveryParameterVocabService aodnDiscoveryParameterVocabService; @Test - void testGetAodnDiscoveryCategories() { + void testGetAodnDiscoveryCategories() throws IOException { // Prepare themes List themes = List.of( new ThemesModel(Arrays.asList( diff --git a/indexer/src/test/resources/application-test.yaml b/indexer/src/test/resources/application-test.yaml index 6766a765..d5a84dc0 100644 --- a/indexer/src/test/resources/application-test.yaml +++ b/indexer/src/test/resources/application-test.yaml @@ -8,6 +8,8 @@ elasticsearch: elasticVersion: "8.11.1" index: name: sample-portal-records + categories-index: + name: sample-discovery-category logging: level: From 78743cd8b5bbaa1c5893579468530f5c1127a056 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Date: Fri, 19 Apr 2024 20:29:25 +1000 Subject: [PATCH 13/19] Various changes to address the PR's reviews --- .../aodn/ardcvocabs/configuration/Config.java | 20 +++++-------- .../ardcvocabs/service/ArdcVocabsService.java | 8 ++--- .../AbstractRequestEntityCreator.java | 19 ------------ .../GeoNetworkRequestEntityCreator.java | 17 ----------- .../abstracts/OgcApiRequestEntityCreator.java | 18 ----------- .../configuration/ObjectMapperConfig.java | 2 +- .../esindexer/configuration/WebMvcConfig.java | 12 ++++---- .../AodnDiscoveryParameterVocabService.java | 24 +++++++-------- .../service/GeoNetworkServiceImpl.java | 30 +++++++++++-------- .../esindexer/service/IndexerServiceImpl.java | 10 +++---- .../esindexer/service/RankingService.java | 2 +- .../esindexer/utils/CacheArdcVocabsUtils.java | 5 ++-- 12 files changed, 58 insertions(+), 109 deletions(-) delete mode 100644 indexer/src/main/java/au/org/aodn/esindexer/abstracts/AbstractRequestEntityCreator.java delete mode 100644 indexer/src/main/java/au/org/aodn/esindexer/abstracts/GeoNetworkRequestEntityCreator.java delete mode 100644 indexer/src/main/java/au/org/aodn/esindexer/abstracts/OgcApiRequestEntityCreator.java diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/Config.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/Config.java index f7ab1184..99be1fef 100644 --- a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/Config.java +++ b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/Config.java @@ -2,26 +2,22 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.ObjectMapper; -import jakarta.annotation.PostConstruct; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cache.annotation.EnableCaching; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.web.client.RestTemplate; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @Configuration public class Config { - - @Autowired - ObjectMapper mapper; - - @PostConstruct - public void init() { - // configure ObjectMapper to exclude null fields while serializing - mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + @Bean("ardcObjectMapper") + public ObjectMapper ardcObjectMapper() { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + return objectMapper; } - @Bean + @Bean("ardcVocabRestTemplate") + @ConditionalOnMissingBean(name = "restTemplate") public RestTemplate ardcVocabRestTemplate() { return new RestTemplate(); } diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabsService.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabsService.java index 9b2517d5..e96d6b02 100644 --- a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabsService.java +++ b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabsService.java @@ -23,7 +23,7 @@ public class ArdcVocabsService { @Autowired @Qualifier("ardcVocabRestTemplate") - protected RestTemplate template; + protected RestTemplate ardcVocabRestTemplate; protected static String path = "/aodn-parameter-category-vocabulary/version-2-1/concept.json"; protected static String leafPath = "/aodn-discovery-parameter-vocabulary/version-1-6/concept.json"; @@ -53,7 +53,7 @@ protected Map> getLeafNodeOfParameterCategory(S while (url != null) { log.debug("Query api -> {}", url); - ObjectNode r = template.getForObject(url, ObjectNode.class); + ObjectNode r = ardcVocabRestTemplate.getForObject(url, ObjectNode.class); if (r != null && !r.isEmpty()) { JsonNode node = r.get("result"); @@ -63,7 +63,7 @@ protected Map> getLeafNodeOfParameterCategory(S String dl = String.format(vocabApiBase + details, about.apply(j)); try { log.debug("Query api -> {}", dl); - ObjectNode d = template.getForObject(dl, ObjectNode.class); + ObjectNode d = ardcVocabRestTemplate.getForObject(dl, ObjectNode.class); if(isNodeValid.apply(d, "result") && isNodeValid.apply(d.get("result"), "primaryTopic")) { JsonNode target = d.get("result").get("primaryTopic"); @@ -116,7 +116,7 @@ public List getParameterCategory(String vocabApiBase) { try { log.debug("Query api -> {}", url); - ObjectNode r = template.getForObject(url, ObjectNode.class); + ObjectNode r = ardcVocabRestTemplate.getForObject(url, ObjectNode.class); if (r != null && !r.isEmpty()) { JsonNode node = r.get("result"); diff --git a/indexer/src/main/java/au/org/aodn/esindexer/abstracts/AbstractRequestEntityCreator.java b/indexer/src/main/java/au/org/aodn/esindexer/abstracts/AbstractRequestEntityCreator.java deleted file mode 100644 index ed453147..00000000 --- a/indexer/src/main/java/au/org/aodn/esindexer/abstracts/AbstractRequestEntityCreator.java +++ /dev/null @@ -1,19 +0,0 @@ -package au.org.aodn.esindexer.abstracts; - -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; - -public abstract class AbstractRequestEntityCreator { - // abstract method to be implemented differently by subclasses - abstract HttpHeaders createHeaders(MediaType accept); - - public HttpEntity getRequestEntity(String body) { - return getRequestEntity(null, body); - } - - public HttpEntity getRequestEntity(MediaType accept, String body) { - HttpHeaders headers = createHeaders(accept); - return body == null ? new HttpEntity<>(headers) : new HttpEntity<>(body, headers); - } -} diff --git a/indexer/src/main/java/au/org/aodn/esindexer/abstracts/GeoNetworkRequestEntityCreator.java b/indexer/src/main/java/au/org/aodn/esindexer/abstracts/GeoNetworkRequestEntityCreator.java deleted file mode 100644 index 3acdef3b..00000000 --- a/indexer/src/main/java/au/org/aodn/esindexer/abstracts/GeoNetworkRequestEntityCreator.java +++ /dev/null @@ -1,17 +0,0 @@ -package au.org.aodn.esindexer.abstracts; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.stereotype.Component; - -import java.util.Collections; - -@Component -public class GeoNetworkRequestEntityCreator extends AbstractRequestEntityCreator { - // used by GeoNetwork related requests - @Override - HttpHeaders createHeaders(MediaType accept) { - HttpHeaders headers = new HttpHeaders(); - headers.setAccept(Collections.singletonList(accept == null ? MediaType.APPLICATION_XML : accept)); - return headers; - } -} diff --git a/indexer/src/main/java/au/org/aodn/esindexer/abstracts/OgcApiRequestEntityCreator.java b/indexer/src/main/java/au/org/aodn/esindexer/abstracts/OgcApiRequestEntityCreator.java deleted file mode 100644 index 69630a26..00000000 --- a/indexer/src/main/java/au/org/aodn/esindexer/abstracts/OgcApiRequestEntityCreator.java +++ /dev/null @@ -1,18 +0,0 @@ -package au.org.aodn.esindexer.abstracts; - -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.stereotype.Component; - -import java.util.Collections; - -@Component -public class OgcApiRequestEntityCreator extends AbstractRequestEntityCreator { - // used by OGC-API related requests - @Override - HttpHeaders createHeaders(MediaType accept) { - HttpHeaders headers = new HttpHeaders(); - headers.setAccept(Collections.singletonList(accept == null ? MediaType.APPLICATION_JSON : accept)); - return headers; - } -} diff --git a/indexer/src/main/java/au/org/aodn/esindexer/configuration/ObjectMapperConfig.java b/indexer/src/main/java/au/org/aodn/esindexer/configuration/ObjectMapperConfig.java index 5505c64a..6d02af5b 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/configuration/ObjectMapperConfig.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/configuration/ObjectMapperConfig.java @@ -9,7 +9,7 @@ @Configuration public class ObjectMapperConfig { - @Bean + @Bean("indexerObjectMapper") public static ObjectMapper objectMapper() { ObjectMapper objectMapper = new ObjectMapper(); diff --git a/indexer/src/main/java/au/org/aodn/esindexer/configuration/WebMvcConfig.java b/indexer/src/main/java/au/org/aodn/esindexer/configuration/WebMvcConfig.java index d8eed21a..ef750705 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/configuration/WebMvcConfig.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/configuration/WebMvcConfig.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import jakarta.annotation.PostConstruct; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.annotation.EnableCaching; import org.springframework.cache.concurrent.ConcurrentMapCacheManager; @@ -16,7 +17,6 @@ import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.web.client.RestTemplate; -import javax.annotation.PostConstruct; import java.util.Arrays; @Configuration @@ -26,7 +26,7 @@ public class WebMvcConfig { @Autowired - protected ObjectMapper objectMapper; + protected ObjectMapper indexObjectMapper; @Autowired protected ArdcVocabsService ardcVocabsService; @@ -41,12 +41,12 @@ public void init() { JavaTimeModule module = new JavaTimeModule(); // Avoid output date-time string become number - objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); - objectMapper.registerModule(module); + indexObjectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + indexObjectMapper.registerModule(module); } - @Bean - public RestTemplate restTemplate() { + @Bean("indexerRestTemplate") + public RestTemplate indexerRestTemplate() { final RestTemplate restTemplate = new RestTemplate(); restTemplate.getMessageConverters().add(jacksonSupportedTypes()); return restTemplate; diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/AodnDiscoveryParameterVocabService.java b/indexer/src/main/java/au/org/aodn/esindexer/service/AodnDiscoveryParameterVocabService.java index a91d5323..1008224a 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/AodnDiscoveryParameterVocabService.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/AodnDiscoveryParameterVocabService.java @@ -1,13 +1,11 @@ package au.org.aodn.esindexer.service; -import au.org.aodn.esindexer.abstracts.OgcApiRequestEntityCreator; import au.org.aodn.esindexer.utils.CacheArdcVocabsUtils; import au.org.aodn.stac.model.ConceptModel; import au.org.aodn.stac.model.ThemesModel; import com.fasterxml.jackson.databind.JsonNode; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.springframework.web.client.RestTemplate; import java.io.IOException; import java.util.ArrayList; @@ -16,24 +14,22 @@ @Service public class AodnDiscoveryParameterVocabService { - @Autowired - RestTemplate restTemplate; - - @Autowired - OgcApiRequestEntityCreator ogcApiRequestEntityCreator; - @Autowired CacheArdcVocabsUtils cacheArdcVocabsUtils; - protected boolean themesMatchConcept(List themes, ConceptModel concept) { + protected boolean themesMatchConcept(List themes, ConceptModel thatConcept) { for (ThemesModel theme : themes) { - for (ConceptModel themeConcept : theme.getConcepts()) { + for (ConceptModel thisConcept : theme.getConcepts()) { /* - comparing by hashcode (id and url) of the concept object + comparing by combined values (id and url) of the concept object this will prevent cases where bottom-level vocabs are the same in text, but their parent vocabs are different e.g "car -> parts" vs "bike -> parts" ("parts" is the same but different parent) */ - if (themeConcept.equals(concept)) { + if (thisConcept.equals(thatConcept)) { + /* thisConcept is the extracted from the themes of the record...theme.getConcepts() + thatConcept is the object created by iterating over the discovery_categories cache...ConceptModel thatConcept = ConceptModel.builder() + using overriding equals method to compare the two objects, this is not checking instanceof ConceptModel class + */ return true; } } @@ -41,6 +37,10 @@ comparing by hashcode (id and url) of the concept object return false; } + /* + this method for analysing the AODN discovery parameter vocabularies of a record aka bottom-level vocabs (found in the themes section) + and returning the second-level vocabularies that match (1 level up from the bottom-level vocabularies) + */ public List getAodnDiscoveryCategories(List themes) throws IOException { List results = new ArrayList<>(); // Iterate over the top-level vocabularies diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/GeoNetworkServiceImpl.java b/indexer/src/main/java/au/org/aodn/esindexer/service/GeoNetworkServiceImpl.java index a90822dc..547b86fc 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/GeoNetworkServiceImpl.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/GeoNetworkServiceImpl.java @@ -1,6 +1,5 @@ package au.org.aodn.esindexer.service; -import au.org.aodn.esindexer.abstracts.GeoNetworkRequestEntityCreator; import au.org.aodn.esindexer.exception.MetadataNotFoundException; import au.org.aodn.esindexer.utils.StringUtil; import au.org.aodn.esindexer.configuration.AppConstants; @@ -25,13 +24,10 @@ import java.util.*; @Service -public class GeoNetworkServiceImpl implements GeoNetworkService { +public class GeoNetworkServiceImpl implements GeoNetworkService { @Autowired - RestTemplate restTemplate; - - @Autowired - GeoNetworkRequestEntityCreator geoNetworkRequestEntityCreator; + RestTemplate indexerRestTemplate; @Autowired @Qualifier("gn4ElasticsearchClient") @@ -45,6 +41,16 @@ public class GeoNetworkServiceImpl implements GeoNetworkService { protected String server; + protected HttpEntity getRequestEntity(String body) { + return getRequestEntity(null, body); + } + + protected HttpEntity getRequestEntity(MediaType accept, String body) { + HttpHeaders headers = new HttpHeaders(); + headers.setAccept(Collections.singletonList(accept == null ? MediaType.APPLICATION_XML : accept)); + return body == null ? new HttpEntity<>(headers) : new HttpEntity<>(body, headers); + } + protected final static String UUID = "uuid"; protected final static String GEONETWORK_GROUP = "groupOwner"; @@ -88,9 +94,9 @@ public String findGroupById(String uuid) throws IOException { Map params = new HashMap<>(); params.put("id", group); - HttpEntity requestEntity = geoNetworkRequestEntityCreator.getRequestEntity(MediaType.APPLICATION_JSON, null); + HttpEntity requestEntity = getRequestEntity(MediaType.APPLICATION_JSON, null); - ResponseEntity responseEntity = restTemplate.exchange( + ResponseEntity responseEntity = indexerRestTemplate.exchange( getGeoNetworkGroupsEndpoint(), HttpMethod.GET, requestEntity, @@ -111,13 +117,13 @@ public String findGroupById(String uuid) throws IOException { protected String findFormatterId(String uuid) { try { - HttpEntity requestEntity = geoNetworkRequestEntityCreator.getRequestEntity(MediaType.APPLICATION_JSON, null); + HttpEntity requestEntity = getRequestEntity(MediaType.APPLICATION_JSON, null); Map params = new HashMap<>(); params.put("indexName", getIndexName()); params.put(UUID, uuid); - ResponseEntity responseEntity = restTemplate.exchange( + ResponseEntity responseEntity = indexerRestTemplate.exchange( getGeoNetworkRecordsEndpoint(), HttpMethod.GET, requestEntity, @@ -142,14 +148,14 @@ protected String findFormatterId(String uuid) { public String searchRecordBy(String uuid) { try { - HttpEntity requestEntity = geoNetworkRequestEntityCreator.getRequestEntity(null); + HttpEntity requestEntity = getRequestEntity(null); Map params = new HashMap<>(); params.put("indexName", getIndexName()); params.put(UUID, uuid); params.put("formatterId", this.findFormatterId(uuid)); - ResponseEntity responseEntity = restTemplate.exchange( + ResponseEntity responseEntity = indexerRestTemplate.exchange( getGeoNetworkRecordsEndpoint() + "/formatters/{formatterId}", HttpMethod.GET, requestEntity, diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java b/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java index 1de9e5b9..245e4100 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java @@ -46,10 +46,10 @@ public class IndexerServiceImpl implements IndexerService { private String indexName; @Autowired - ObjectMapper objectMapper; + ObjectMapper indexerObjectMapper; @Autowired - protected StacCollectionMapperService mapper; + protected StacCollectionMapperService stacCollectionMapperService; @Autowired JaxbUtils jaxbUtils; @@ -111,7 +111,7 @@ protected boolean isMetadataPublished(String uuid) { protected StacCollectionModel getMappedMetadataValues(String metadataValues) throws IOException, FactoryException, TransformException, JAXBException { MDMetadataType metadataType = jaxbUtils.unmarshal(metadataValues); - StacCollectionModel stacCollectionModel = mapper.mapToSTACCollection(metadataType); + StacCollectionModel stacCollectionModel = stacCollectionMapperService.mapToSTACCollection(metadataType); // evaluate completeness Integer completeness = rankingService.evaluateCompleteness(stacCollectionModel); @@ -165,7 +165,7 @@ public ResponseEntity indexMetadata(String metadataValues) { // index the metadata if it is published if (this.isMetadataPublished(uuid)) { - try(InputStream is = new ByteArrayInputStream(objectMapper.writeValueAsBytes(mappedMetadataValues))) { + try(InputStream is = new ByteArrayInputStream(indexerObjectMapper.writeValueAsBytes(mappedMetadataValues))) { logger.info("Ingesting a new metadata with UUID: " + uuid + " to index: " + indexName); logger.info("{}", mappedMetadataValues); req = IndexRequest.of(b -> b @@ -233,7 +233,7 @@ public ResponseEntity indexAllMetadataRecordsFromGeoNetwork(boolean conf StacCollectionModel mappedMetadataValues = this.getMappedMetadataValues(metadataRecord); // convert mapped values to binary data - logger.debug("Ingested json is {}", objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(mappedMetadataValues)); + logger.debug("Ingested json is {}", indexerObjectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(mappedMetadataValues)); // send bulk request to Elasticsearch bulkRequest.operations(op -> op diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/RankingService.java b/indexer/src/main/java/au/org/aodn/esindexer/service/RankingService.java index e212bea8..d09f7f44 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/RankingService.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/RankingService.java @@ -3,5 +3,5 @@ import au.org.aodn.stac.model.StacCollectionModel; public interface RankingService { - public Integer evaluateCompleteness(StacCollectionModel stacCollectionModel); + Integer evaluateCompleteness(StacCollectionModel stacCollectionModel); } diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java index 8bac5ba6..fb1bb531 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java @@ -53,7 +53,7 @@ public class CacheArdcVocabsUtils { String categoriesIndexName; @Autowired - ObjectMapper objectMapper; + ObjectMapper indexerObjectMapper; protected void indexingDiscoveryCategoriesIndex(List categoryVocabModels) throws IOException { @@ -64,7 +64,7 @@ protected void indexingDiscoveryCategoriesIndex(List categor for (CategoryVocabModel categoryVocabModel : categoryVocabModels) { try { // convert categoryVocabModel values to binary data - log.debug("Ingested json is {}", objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(categoryVocabModel)); + log.debug("Ingested json is {}", indexerObjectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(categoryVocabModel)); // send bulk request to Elasticsearch bulkRequest.operations(op -> op .index(idx -> idx @@ -109,6 +109,7 @@ public void refreshDiscoveryCategoriesIndex() throws IOException { indexingDiscoveryCategoriesIndex(categoryVocabModels); } + // TODO research strategy to avoid multiple refresh runs on multiple indexer instance @Cacheable(AppConstants.AODN_DISCOVERY_CATEGORIES_CACHE) public List getDiscoveryCategories() throws IOException { List categories = new ArrayList<>(); From d2a5922971df6f7a7a3656ba5708e76ed744be37 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Date: Fri, 19 Apr 2024 21:14:15 +1000 Subject: [PATCH 14/19] Fix issues with bean init and remove component scan to use ardcvocabs module --- .../au/org/aodn/ardcvocabs/ArdcVocabs.java | 19 ------------------- .../aodn/ardcvocabs/configuration/Config.java | 2 +- .../ardcvocabs/service/ArdcVocabsService.java | 2 -- .../configuration/ArdcVocabsConfig.java | 13 +++++++++++++ .../esindexer/configuration/WebMvcConfig.java | 12 +++--------- .../esindexer/utils/CacheArdcVocabsUtils.java | 2 +- .../service/IndexerServiceTests.java | 4 ++-- .../AodnDiscoveryParameterVocabUtilsTest.java | 5 ----- .../src/test/resources/application-test.yaml | 2 -- 9 files changed, 20 insertions(+), 41 deletions(-) delete mode 100644 ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/ArdcVocabs.java create mode 100644 indexer/src/main/java/au/org/aodn/esindexer/configuration/ArdcVocabsConfig.java diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/ArdcVocabs.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/ArdcVocabs.java deleted file mode 100644 index e2644c4d..00000000 --- a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/ArdcVocabs.java +++ /dev/null @@ -1,19 +0,0 @@ -package au.org.aodn.ardcvocabs; - -import jakarta.annotation.PostConstruct; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; - -import java.util.TimeZone; - -@SpringBootApplication -public class ArdcVocabs { - @PostConstruct - void init() { - TimeZone.setDefault(TimeZone.getTimeZone("UTC")); - } - - public static void main(String[] args) { - SpringApplication.run(ArdcVocabs.class, args); - } -} diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/Config.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/Config.java index 99be1fef..f4684f85 100644 --- a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/Config.java +++ b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/Config.java @@ -17,7 +17,7 @@ public ObjectMapper ardcObjectMapper() { } @Bean("ardcVocabRestTemplate") - @ConditionalOnMissingBean(name = "restTemplate") + @ConditionalOnMissingBean(RestTemplate.class) public RestTemplate ardcVocabRestTemplate() { return new RestTemplate(); } diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabsService.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabsService.java index e96d6b02..bfa8793e 100644 --- a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabsService.java +++ b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabsService.java @@ -5,7 +5,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; @@ -22,7 +21,6 @@ @Service public class ArdcVocabsService { @Autowired - @Qualifier("ardcVocabRestTemplate") protected RestTemplate ardcVocabRestTemplate; protected static String path = "/aodn-parameter-category-vocabulary/version-2-1/concept.json"; diff --git a/indexer/src/main/java/au/org/aodn/esindexer/configuration/ArdcVocabsConfig.java b/indexer/src/main/java/au/org/aodn/esindexer/configuration/ArdcVocabsConfig.java new file mode 100644 index 00000000..b896d5b2 --- /dev/null +++ b/indexer/src/main/java/au/org/aodn/esindexer/configuration/ArdcVocabsConfig.java @@ -0,0 +1,13 @@ +package au.org.aodn.esindexer.configuration; + +import au.org.aodn.ardcvocabs.service.ArdcVocabsService; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class ArdcVocabsConfig { + @Bean + public ArdcVocabsService ardcVocabsService() { + return new ArdcVocabsService(); + } +} diff --git a/indexer/src/main/java/au/org/aodn/esindexer/configuration/WebMvcConfig.java b/indexer/src/main/java/au/org/aodn/esindexer/configuration/WebMvcConfig.java index ef750705..95fa4730 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/configuration/WebMvcConfig.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/configuration/WebMvcConfig.java @@ -1,6 +1,5 @@ package au.org.aodn.esindexer.configuration; -import au.org.aodn.ardcvocabs.service.ArdcVocabsService; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; @@ -9,7 +8,6 @@ import org.springframework.cache.annotation.EnableCaching; import org.springframework.cache.concurrent.ConcurrentMapCacheManager; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.http.MediaType; import org.springframework.http.converter.HttpMessageConverter; @@ -20,16 +18,12 @@ import java.util.Arrays; @Configuration -@ComponentScan("au.org.aodn.ardcvocabs") @EnableCaching @EnableScheduling public class WebMvcConfig { @Autowired - protected ObjectMapper indexObjectMapper; - - @Autowired - protected ArdcVocabsService ardcVocabsService; + protected ObjectMapper indexerObjectMapper; @Bean public ConcurrentMapCacheManager cacheManager() { @@ -41,8 +35,8 @@ public void init() { JavaTimeModule module = new JavaTimeModule(); // Avoid output date-time string become number - indexObjectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); - indexObjectMapper.registerModule(module); + indexerObjectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + indexerObjectMapper.registerModule(module); } @Bean("indexerRestTemplate") diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java index fb1bb531..35200b8f 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java @@ -109,8 +109,8 @@ public void refreshDiscoveryCategoriesIndex() throws IOException { indexingDiscoveryCategoriesIndex(categoryVocabModels); } - // TODO research strategy to avoid multiple refresh runs on multiple indexer instance @Cacheable(AppConstants.AODN_DISCOVERY_CATEGORIES_CACHE) + // TODO research strategy to avoid multiple refresh runs at the same schedule by multiple indexer instances public List getDiscoveryCategories() throws IOException { List categories = new ArrayList<>(); log.info("Fetching AODN Discovery Parameter Vocabularies from {}", categoriesIndexName); diff --git a/indexer/src/test/java/au/org/aodn/esindexer/service/IndexerServiceTests.java b/indexer/src/test/java/au/org/aodn/esindexer/service/IndexerServiceTests.java index d03e456d..33992e00 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/service/IndexerServiceTests.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/service/IndexerServiceTests.java @@ -29,7 +29,7 @@ public class IndexerServiceTests extends BaseTestClass { protected IndexerServiceImpl indexerService; @Autowired - protected ObjectMapper objectMapper; + protected ObjectMapper indexerObjectMapper; @Autowired protected ElasticSearchIndexService elasticSearchIndexService; @@ -119,7 +119,7 @@ public void verifyGetDocumentByUUID() throws IOException { String test = objectNodeHit.source().toPrettyString(); - assertEquals("Stac equals", objectMapper.readTree(expected), objectMapper.readTree(test)); + assertEquals("Stac equals", indexerObjectMapper.readTree(expected), indexerObjectMapper.readTree(test)); deleteRecord("7709f541-fc0c-4318-b5b9-9053aa474e0e"); } diff --git a/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java b/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java index 9154c968..2446e672 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java @@ -1,6 +1,5 @@ package au.org.aodn.esindexer.utils; -import au.org.aodn.ardcvocabs.service.ArdcVocabsService; import au.org.aodn.esindexer.service.AodnDiscoveryParameterVocabService; import au.org.aodn.stac.model.ConceptModel; import au.org.aodn.stac.model.ThemesModel; @@ -20,10 +19,6 @@ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class AodnDiscoveryParameterVocabUtilsTest { - - @Autowired - ArdcVocabsService ardcVocabsService; - @Autowired AodnDiscoveryParameterVocabService aodnDiscoveryParameterVocabService; diff --git a/indexer/src/test/resources/application-test.yaml b/indexer/src/test/resources/application-test.yaml index d5a84dc0..6766a765 100644 --- a/indexer/src/test/resources/application-test.yaml +++ b/indexer/src/test/resources/application-test.yaml @@ -8,8 +8,6 @@ elasticsearch: elasticVersion: "8.11.1" index: name: sample-portal-records - categories-index: - name: sample-discovery-category logging: level: From 44a7366f836ed9491126f5dbe888c99ce067fbc1 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Date: Sat, 20 Apr 2024 01:21:25 +1000 Subject: [PATCH 15/19] Switch to standard analyser --- .../portal_records_index_schema.json | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/indexer/src/main/resources/config_files/portal_records_index_schema.json b/indexer/src/main/resources/config_files/portal_records_index_schema.json index aa248c51..af937ff7 100644 --- a/indexer/src/main/resources/config_files/portal_records_index_schema.json +++ b/indexer/src/main/resources/config_files/portal_records_index_schema.json @@ -1,24 +1,4 @@ { - "settings":{ - "analysis":{ - "analyzer":{ - "my_stop_analyzer":{ - "type":"custom", - "tokenizer":"standard", - "filter":[ - "lowercase", - "english_stop" - ] - } - }, - "filter":{ - "english_stop":{ - "type":"stop", - "stopwords":"_english_" - } - } - } - }, "mappings": { "dynamic": true, "properties": { @@ -31,7 +11,7 @@ "title": { "type": "text" }, "title_suggest": { "type": "search_as_you_type", - "analyzer": "my_stop_analyzer" + "analyzer": "standard" }, "keywords": { "type": "nested", From a781067ba79cc440d9962b074778ca10c7228c3f Mon Sep 17 00:00:00 2001 From: Viet Nguyen Date: Sat, 20 Apr 2024 17:07:29 +1000 Subject: [PATCH 16/19] use keyword type for discovery_categories field to enhance filtering accuracy --- .../service/AodnDiscoveryParameterVocabService.java | 2 +- .../resources/config_files/portal_records_index_schema.json | 1 + .../utils/AodnDiscoveryParameterVocabUtilsTest.java | 6 +++--- indexer/src/test/resources/canned/sample4_stac.json | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/AodnDiscoveryParameterVocabService.java b/indexer/src/main/java/au/org/aodn/esindexer/service/AodnDiscoveryParameterVocabService.java index 1008224a..933a5579 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/AodnDiscoveryParameterVocabService.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/AodnDiscoveryParameterVocabService.java @@ -57,7 +57,7 @@ public List getAodnDiscoveryCategories(List themes) throws .build(); // Compare with themes' concepts if (themesMatchConcept(themes, bottomConcept)) { - results.add(secondLevelVocabLabel); + results.add(secondLevelVocabLabel.toLowerCase()); break; // To avoid duplicates because under the same second-level vocab there can be multiple bottom-level vocabs that pass the condition } } diff --git a/indexer/src/main/resources/config_files/portal_records_index_schema.json b/indexer/src/main/resources/config_files/portal_records_index_schema.json index af937ff7..358eb5d3 100644 --- a/indexer/src/main/resources/config_files/portal_records_index_schema.json +++ b/indexer/src/main/resources/config_files/portal_records_index_schema.json @@ -13,6 +13,7 @@ "type": "search_as_you_type", "analyzer": "standard" }, + "discovery_categories" : { "type": "keyword" }, "keywords": { "type": "nested", "properties": { diff --git a/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java b/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java index 2446e672..a275e208 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java @@ -42,9 +42,9 @@ void testGetAodnDiscoveryCategories() throws IOException { // Assertions assertNotNull(categories); - assertTrue(categories.contains("Alkalinity")); - assertTrue(categories.contains("Temperature")); - assertTrue(categories.contains("Salinity")); + assertTrue(categories.contains("alkalinity")); + assertTrue(categories.contains("temperature")); + assertTrue(categories.contains("salinity")); assertEquals(3, categories.size()); } } diff --git a/indexer/src/test/resources/canned/sample4_stac.json b/indexer/src/test/resources/canned/sample4_stac.json index 0dbc743f..34f910ee 100644 --- a/indexer/src/test/resources/canned/sample4_stac.json +++ b/indexer/src/test/resources/canned/sample4_stac.json @@ -22,7 +22,7 @@ "start" : "1870-07-16T14:10:44Z", "end" : "2013-06-16T14:00:00Z" } ], - "discovery_categories" : [ "Alkalinity", "Temperature", "Salinity" ] + "discovery_categories" : [ "alkalinity", "temperature", "salinity" ] }, "contacts" : [ { "emails" : [ "Andrew.Lenton@csiro.au" ], From 5f2ef84eff521ac90e9772e455f062b2b3ee6e2f Mon Sep 17 00:00:00 2001 From: Viet Nguyen Date: Sat, 20 Apr 2024 17:24:19 +1000 Subject: [PATCH 17/19] use custom analyser for the search_as_you_type field --- .../portal_records_index_schema.json | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/indexer/src/main/resources/config_files/portal_records_index_schema.json b/indexer/src/main/resources/config_files/portal_records_index_schema.json index 358eb5d3..69fa4d38 100644 --- a/indexer/src/main/resources/config_files/portal_records_index_schema.json +++ b/indexer/src/main/resources/config_files/portal_records_index_schema.json @@ -1,4 +1,24 @@ { + "settings":{ + "analysis":{ + "analyzer":{ + "custom_analyser":{ + "type":"custom", + "tokenizer":"standard", + "filter":[ + "lowercase", + "english_stop" + ] + } + }, + "filter":{ + "english_stop":{ + "type":"stop", + "stopwords":"_english_" + } + } + } + }, "mappings": { "dynamic": true, "properties": { @@ -11,7 +31,7 @@ "title": { "type": "text" }, "title_suggest": { "type": "search_as_you_type", - "analyzer": "standard" + "analyzer": "custom_analyser" }, "discovery_categories" : { "type": "keyword" }, "keywords": { From 7c4636e14da1a897ac7212a19fe0812c63bbd1c7 Mon Sep 17 00:00:00 2001 From: utas-raymondng Date: Mon, 22 Apr 2024 12:01:28 +1000 Subject: [PATCH 18/19] Change to make it autoconfig --- ardcvocabs/pom.xml | 5 ++++ .../configuration/ArdcAutoConfiguration.java | 29 +++++++++++++++++++ .../aodn/ardcvocabs/configuration/Config.java | 24 --------------- .../ardcvocabs/service/ArdcVocabsService.java | 2 -- ...ot.autoconfigure.AutoConfiguration.imports | 1 + .../configuration/ArdcVocabsConfig.java | 13 --------- .../esindexer/configuration/WebMvcConfig.java | 12 ++++---- .../esindexer/utils/CacheArdcVocabsUtils.java | 1 + .../AodnDiscoveryParameterVocabUtilsTest.java | 7 ++++- 9 files changed, 47 insertions(+), 47 deletions(-) create mode 100644 ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/ArdcAutoConfiguration.java delete mode 100644 ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/Config.java create mode 100644 ardcvocabs/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports delete mode 100644 indexer/src/main/java/au/org/aodn/esindexer/configuration/ArdcVocabsConfig.java diff --git a/ardcvocabs/pom.xml b/ardcvocabs/pom.xml index cc31feb6..9f1c3646 100644 --- a/ardcvocabs/pom.xml +++ b/ardcvocabs/pom.xml @@ -42,6 +42,11 @@ org.springframework.boot spring-boot-autoconfigure + + org.springframework.boot + spring-boot-autoconfigure-processor + true + org.springframework.boot spring-boot-starter-test diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/ArdcAutoConfiguration.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/ArdcAutoConfiguration.java new file mode 100644 index 00000000..ccbf47a9 --- /dev/null +++ b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/ArdcAutoConfiguration.java @@ -0,0 +1,29 @@ +package au.org.aodn.ardcvocabs.configuration; + +import au.org.aodn.ardcvocabs.service.ArdcVocabsService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.web.client.RestTemplate; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; + +@Slf4j +@AutoConfiguration // More expressive vs @Configuration +@ConditionalOnMissingBean(ArdcVocabsService.class) +public class ArdcAutoConfiguration { + + @Bean + public ArdcVocabsService createArdcVocabsService() { + log.info("Create ArdcVocabsService"); + return new ArdcVocabsService(); + } + /** + * In case the one who use this lib have not created it. + * @return RestTemplate + */ + @Bean + @ConditionalOnMissingBean(RestTemplate.class) + public RestTemplate ardcVocabRestTemplate() { + return new RestTemplate(); + } +} diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/Config.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/Config.java deleted file mode 100644 index f4684f85..00000000 --- a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/Config.java +++ /dev/null @@ -1,24 +0,0 @@ -package au.org.aodn.ardcvocabs.configuration; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.web.client.RestTemplate; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; - -@Configuration -public class Config { - @Bean("ardcObjectMapper") - public ObjectMapper ardcObjectMapper() { - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); - return objectMapper; - } - - @Bean("ardcVocabRestTemplate") - @ConditionalOnMissingBean(RestTemplate.class) - public RestTemplate ardcVocabRestTemplate() { - return new RestTemplate(); - } -} diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabsService.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabsService.java index bfa8793e..b7c8034e 100644 --- a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabsService.java +++ b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabsService.java @@ -5,7 +5,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; @@ -18,7 +17,6 @@ @Slf4j -@Service public class ArdcVocabsService { @Autowired protected RestTemplate ardcVocabRestTemplate; diff --git a/ardcvocabs/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/ardcvocabs/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 00000000..b7e579a9 --- /dev/null +++ b/ardcvocabs/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1 @@ +au.org.aodn.ardcvocabs.configuration.ArdcAutoConfiguration \ No newline at end of file diff --git a/indexer/src/main/java/au/org/aodn/esindexer/configuration/ArdcVocabsConfig.java b/indexer/src/main/java/au/org/aodn/esindexer/configuration/ArdcVocabsConfig.java deleted file mode 100644 index b896d5b2..00000000 --- a/indexer/src/main/java/au/org/aodn/esindexer/configuration/ArdcVocabsConfig.java +++ /dev/null @@ -1,13 +0,0 @@ -package au.org.aodn.esindexer.configuration; - -import au.org.aodn.ardcvocabs.service.ArdcVocabsService; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class ArdcVocabsConfig { - @Bean - public ArdcVocabsService ardcVocabsService() { - return new ArdcVocabsService(); - } -} diff --git a/indexer/src/main/java/au/org/aodn/esindexer/configuration/WebMvcConfig.java b/indexer/src/main/java/au/org/aodn/esindexer/configuration/WebMvcConfig.java index 95fa4730..bb0d9d9c 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/configuration/WebMvcConfig.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/configuration/WebMvcConfig.java @@ -41,14 +41,12 @@ public void init() { @Bean("indexerRestTemplate") public RestTemplate indexerRestTemplate() { - final RestTemplate restTemplate = new RestTemplate(); - restTemplate.getMessageConverters().add(jacksonSupportedTypes()); - return restTemplate; - } - - private HttpMessageConverter jacksonSupportedTypes() { MappingJackson2HttpMessageConverter converter = new MappingJackson2HttpMessageConverter(); converter.setSupportedMediaTypes(Arrays.asList(MediaType.parseMediaType("text/plain;charset=utf-8"), MediaType.APPLICATION_OCTET_STREAM)); - return converter; + + final RestTemplate restTemplate = new RestTemplate(); + restTemplate.getMessageConverters().add(converter); + + return restTemplate; } } diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java index 35200b8f..5c5aa111 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/CacheArdcVocabsUtils.java @@ -111,6 +111,7 @@ public void refreshDiscoveryCategoriesIndex() throws IOException { @Cacheable(AppConstants.AODN_DISCOVERY_CATEGORIES_CACHE) // TODO research strategy to avoid multiple refresh runs at the same schedule by multiple indexer instances + // A way to do it is read the value from Elastic search, if it has updated within say 24 hrs then use it public List getDiscoveryCategories() throws IOException { List categories = new ArrayList<>(); log.info("Fetching AODN Discovery Parameter Vocabularies from {}", categoriesIndexName); diff --git a/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java b/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java index a275e208..e0f6b67a 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/utils/AodnDiscoveryParameterVocabUtilsTest.java @@ -1,10 +1,13 @@ package au.org.aodn.esindexer.utils; +import au.org.aodn.ardcvocabs.configuration.ArdcAutoConfiguration; import au.org.aodn.esindexer.service.AodnDiscoveryParameterVocabService; import au.org.aodn.stac.model.ConceptModel; import au.org.aodn.stac.model.ThemesModel; import org.junit.jupiter.api.*; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.ImportAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ActiveProfiles; @@ -14,7 +17,9 @@ import static org.junit.jupiter.api.Assertions.*; -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT +) @ActiveProfiles("test") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @TestInstance(TestInstance.Lifecycle.PER_CLASS) From f25549e60d192e7a0517a9c0f0085f7b383241b1 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Date: Mon, 22 Apr 2024 12:32:32 +1000 Subject: [PATCH 19/19] Fix end-of-file pre-commit error --- ...springframework.boot.autoconfigure.AutoConfiguration.imports | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ardcvocabs/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/ardcvocabs/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports index b7e579a9..bcb97a40 100644 --- a/ardcvocabs/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ b/ardcvocabs/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -1 +1 @@ -au.org.aodn.ardcvocabs.configuration.ArdcAutoConfiguration \ No newline at end of file +au.org.aodn.ardcvocabs.configuration.ArdcAutoConfiguration