Skip to content

Commit

Permalink
Merge pull request #71 from aodn/features/5522-thumbnail
Browse files Browse the repository at this point in the history 
Add thumbnail
  • Loading branch information
@vietnguyengit
vietnguyengit authored May 23, 2024
2 parents 986aacc + d31460f commit 3e8b23c
Show file tree
Hide file tree
Showing 11 changed files with 5,284 additions and 103 deletions.
5 changes: 4 additions & 1 deletion indexer/src/main/java/au/org/aodn/esindexer/service/GeoNetworkService.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package au.org.aodn.esindexer.service;

import au.org.aodn.stac.model.LinkModel;
import com.fasterxml.jackson.databind.JsonNode;
import org.springframework.http.ResponseEntity;

import java.io.IOException;
Expand All @@ -16,7 +18,8 @@ public interface GeoNetworkService {
String searchRecordBy(String uuid);
String findGroupById(String uuid) throws IOException;

Optional<Map<String, Object>> getRecordExtraInfo(String uuid);
Optional<LinkModel> getThumbnail(String uuid);
Optional<LinkModel> getLogo(String uuid);
/**
* Return Iterable of records, noted that the item inside can be null, so please check null on each item
* @return
Expand Down
221 changes: 212 additions & 9 deletions indexer/src/main/java/au/org/aodn/esindexer/service/GeoNetworkServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,19 @@
import au.org.aodn.esindexer.exception.MetadataNotFoundException;
import au.org.aodn.esindexer.utils.StringUtil;
import au.org.aodn.esindexer.configuration.AppConstants;
import au.org.aodn.esindexer.utils.UrlUtils;
import au.org.aodn.stac.model.LinkModel;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.elasticsearch._types.query_dsl.MatchAllQuery;

import co.elastic.clients.elasticsearch.core.SearchRequest;
import co.elastic.clients.elasticsearch.core.SearchResponse;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.*;
import org.springframework.web.client.HttpClientErrorException;
Expand All @@ -22,6 +26,13 @@

public class GeoNetworkServiceImpl implements GeoNetworkService {

public static final String SUGGEST_LOGOS = "suggest_logos";
public static final String THUMB_NAILS = "thumbnails";
public static final String URL = "url";

@Autowired
protected UrlUtils urlUtils;

protected static final Logger logger = LogManager.getLogger(GeoNetworkServiceImpl.class);

protected RestTemplate indexerRestTemplate;
Expand Down Expand Up @@ -107,18 +118,81 @@ public String findGroupById(String uuid) throws IOException {
}

}

/**
* Please check comment section of getRecordRelated to understand how the structure looks like for
* thumbnail section
* @param uuid - UUID of record
* @return - The LinkModel ref the thumbnail.
*/
@Override
public Optional<Map<String, Object>> getRecordExtraInfo(String uuid) {
HttpEntity<String> requestEntity = getRequestEntity(MediaType.APPLICATION_JSON, null);
public Optional<LinkModel> getThumbnail(String uuid) {
Optional<Map<String, ?>> optRelated = getRecordRelated(uuid);
if(optRelated.isPresent()) {
Map<String, ?> node = optRelated.get();
if(node.containsKey(THUMB_NAILS) && node.get(THUMB_NAILS) instanceof List<?> thumbnails) {
// Always use the first item, not sure if it is good?
if(!thumbnails.isEmpty() && thumbnails.get(0) instanceof Map<?, ?> thumbnail) {
if(thumbnail.containsKey(URL) && thumbnail.get(URL) instanceof Map<?,?> link) {
return link.entrySet()
.stream()
.findFirst()
.map(i -> {
LinkModel linkModel = LinkModel.builder().build();
linkModel.setType("image");
linkModel.setRel("thumbnail");
linkModel.setHref(i.getValue().toString());
return linkModel;
});
}
}
}
}
return Optional.empty();
}
/**
* Return a link to the logo if exist.
*
* @param uuid - UUID of record
* @return - Link if logo exist
*/
@Override
public Optional<LinkModel> getLogo(String uuid) {
Optional<Map<String, Object>> optAdditionalInfo = getRecordExtraInfo(uuid);
if(optAdditionalInfo.isPresent()) {
// We iterate logos link and add it to STAC
Map<String, Object> additionalInfo = optAdditionalInfo.get();
if(additionalInfo.containsKey(SUGGEST_LOGOS)) {
if(additionalInfo.get(SUGGEST_LOGOS) instanceof List) {
return ((List<?>) additionalInfo.get(SUGGEST_LOGOS))
.stream()
.map(p -> (p instanceof String) ? (String) p : null)
.filter(Objects::nonNull)
.filter(i -> urlUtils.checkUrlExists(i))
.findFirst() // We only pick the first reachable one
.map(i -> {
LinkModel linkModel = LinkModel.builder().build();
linkModel.setHref(i);
// Geonetwork always return png logo
linkModel.setType("image/png");
linkModel.setRel("icon");
linkModel.setTitle("Suggest icon for dataset");
return linkModel;
});
}
}
}
return Optional.empty();
}

protected Optional<Map<String, Object>> getRecordExtraInfo(String uuid) {
Map<String, Object> params = new HashMap<>();
params.put(UUID, uuid);

try {
ResponseEntity<Map<String, Object>> responseEntity = indexerRestTemplate.exchange(
getAodnExtRecordsEndpoint(),
HttpMethod.GET,
requestEntity,
defaultRequestEntity,
new ParameterizedTypeReference<>() {},
params);

Expand All @@ -127,25 +201,150 @@ public Optional<Map<String, Object>> getRecordExtraInfo(String uuid) {
}
}
catch(HttpClientErrorException clientErrorException) {
logger.warn("Fail to call API on additional info, please check api exist?", clientErrorException);
logger.warn("Fail to call API on additional info, please check api exist?");
return Optional.empty();
}
return Optional.empty();
}
/**
* Call the geonetwork API to get the related info given a uuid, in the related record, you will find the link
* to the thumbnail
* @param uuid - UUID of record
* @return - A Json structure like this
* {
* "children": null,
* "parent": null,
* "siblings": null,
* "associated": null,
* "services": null,
* "datasets": null,
* "fcats": null,
* "hasfeaturecats": null,
* "sources": null,
* "hassources": null,
* "related": null,
* "onlines": [
* {
* "id": "https://www.marine.csiro.au/data/trawler/survey_details.cfm?survey=IN2024_V01",
* "url": {
* "eng": "https://www.marine.csiro.au/data/trawler/survey_details.cfm?survey=IN2024_V01"
* },
* "type": "onlinesrc",
* "title": {
* "eng": "MNF Data Trawler"
* },
* "protocol": "WWW:DOWNLOAD-1.0-http--csiro-oa-app",
* "description": {
* "eng": "Link to processed data and survey information (plans, summaries, etc.) via MNF Data Trawler"
* },
* "function": "",
* "mimeType": "",
* "applicationProfile": ""
* },
* {
* "id": "https://mnf.csiro.au/",
* "url": {
* "eng": "https://mnf.csiro.au/"
* },
* "type": "onlinesrc",
* "title": {
* "eng": "Marine National Facility"
* },
* "protocol": "WWW:LINK-1.0-http--link",
* "description": {
* "eng": "Link to the Marine National Facility Webpage"
* },
* "function": "",
* "mimeType": "",
* "applicationProfile": ""
* },
* {
* "id": "https://doi.org/10.25919/rdrt-bd71",
* "url": {
* "eng": "https://doi.org/10.25919/rdrt-bd71"
* },
* "type": "onlinesrc",
* "title": {
* "eng": "Data Access Portal (DOI)"
* },
* "protocol": "WWW:DOWNLOAD-1.0-http--csiro-dap",
* "description": {
* "eng": "Link to this record at the CSIRO Data Access Portal"
* },
* "function": "",
* "mimeType": "",
* "applicationProfile": ""
* },
* {
* "id": "http://www.marine.csiro.au/data/underway/?survey=IN2024_V01",
* "url": {
* "eng": "http://www.marine.csiro.au/data/underway/?survey=IN2024_V01"
* },
* "type": "onlinesrc",
* "title": {
* "eng": "Underway Visualisation Tool"
* },
* "protocol": "WWW:DOWNLOAD-1.0-http--csiro-oa-app",
* "description": {
* "eng": "Link to visualisation tool for Near Real-Time Underway Data (NRUD)"
* },
* "function": "",
* "mimeType": "",
* "applicationProfile": ""
* }
* ],
* "thumbnails": [
* {
* "id": "https://www.marine.csiro.au/data/trawler/survey_mapfile.cfm?survey=IN2024_V01&data_type=uwy",
* "url": {
* "eng": "https://www.marine.csiro.au/data/trawler/survey_mapfile.cfm?survey=IN2024_V01&data_type=uwy"
* },
* "type": "thumbnail",
* "title": {
* "eng": "Voyage track"
* }
* }
* ]
* }
*/
protected Optional<Map<String, ?>> getRecordRelated(String uuid) {
try {
Map<String, Object> params = new HashMap<>();
params.put(UUID, uuid);

ResponseEntity<Map<String, ?>> responseEntity = indexerRestTemplate.exchange(
getGeoNetworkRelatedEndpoint(),
HttpMethod.GET,
defaultRequestEntity,
new ParameterizedTypeReference<>() {},
params
);

if (responseEntity.getStatusCode().is2xxSuccessful()) {
return Optional.ofNullable(responseEntity.getBody());
}
else {
return Optional.empty();
}
}
catch (HttpClientErrorException.NotFound e) {
throw new MetadataNotFoundException("Unable to find metadata record with UUID: " + uuid + " in GeoNetwork");
}
}

protected String findFormatterId(String uuid) {
try {
HttpEntity<String> requestEntity = getRequestEntity(MediaType.APPLICATION_JSON, null);

Map<String, Object> params = new HashMap<>();
params.put("indexName", getIndexName());
params.put(UUID, uuid);

ResponseEntity<JsonNode> responseEntity = indexerRestTemplate.exchange(
getGeoNetworkRecordsEndpoint(),
HttpMethod.GET,
requestEntity,
JsonNode.class, params);
defaultRequestEntity,
JsonNode.class,
params
);

if (responseEntity.getStatusCode().is2xxSuccessful()) {
if (Objects.requireNonNull(responseEntity.getBody()).get("@xsi:schemaLocation").asText().contains("www.isotc211.org/2005/gmd")) {
Expand Down Expand Up @@ -278,6 +477,10 @@ public String next() {
@Override
public void setServer(String s) { server = s; }

protected String getGeoNetworkRelatedEndpoint() {
return getServer() + "/geonetwork/srv/api/records/{uuid}/related";
}

protected String getGeoNetworkRecordsEndpoint() {
return getServer() + "/geonetwork/srv/api/{indexName}/{uuid}";
}
Expand Down
25 changes: 21 additions & 4 deletions indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,14 +238,15 @@ public ResponseEntity<String> indexAllMetadataRecordsFromGeoNetwork(boolean conf
if(metadataRecord != null) {
try {
// get mapped metadata values from GeoNetwork to STAC collection schema
StacCollectionModel mappedMetadataValues = this.getMappedMetadataValues(metadataRecord);
final StacCollectionModel mappedMetadataValues = this.getMappedMetadataValues(metadataRecord);

// convert mapped values to binary data
logger.debug("Ingested json is {}", indexerObjectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(mappedMetadataValues));

// send bulk request to Elasticsearch
bulkRequest.operations(op -> op
.index(idx -> idx
.id(mappedMetadataValues.getUuid())
.index(indexName)
.document(mappedMetadataValues)
)
Expand All @@ -267,11 +268,27 @@ public ResponseEntity<String> indexAllMetadataRecordsFromGeoNetwork(boolean conf
portalElasticsearchClient.indices().refresh();

// Log errors, if any
if (result.errors()) {
logger.error("Bulk had errors");
if (!result.items().isEmpty()) {
logger.error("Bulk load have errors? {}", result.errors());
for (BulkResponseItem item: result.items()) {
if (item.error() != null) {
logger.error("{} {}", item.error().reason(), item.error().causedBy());
try {
logger.error("UUID {} {} {} {}",
item.id(),
item.error().reason(),
item.error().causedBy(),
indexerObjectMapper
.writerWithDefaultPrettyPrinter()
.writeValueAsString(
this.getMappedMetadataValues(
geoNetworkResourceService.searchRecordBy(item.id())
)
)
);
}
catch (FactoryException | TransformException | JAXBException e) {
logger.warn("Parse error on display stac record");
}
}
}
} else {
Expand Down
Loading

0 comments on commit 3e8b23c

Please sign in to comment.