From b27917da087f71231bfe6f718c8cfe4e03aecf11 Mon Sep 17 00:00:00 2001 From: utas-raymondng Date: Wed, 22 May 2024 11:45:13 +1000 Subject: [PATCH] select the first reachable icon --- .../configuration/GeoNetworkSearchConfig.java | 7 +++++ .../service/StacCollectionMapperService.java | 12 ++++++--- .../au/org/aodn/esindexer/utils/UrlUtils.java | 23 ++++++++++++++++ .../src/main/resources/application-dev.yaml | 2 +- .../GeoNetworkSearchTestConfig.java | 27 ++++++++++++++++--- .../service/IndexerServiceTests.java | 1 + .../test/resources/canned/sample5_stac.json | 13 +++------ 7 files changed, 67 insertions(+), 18 deletions(-) create mode 100644 indexer/src/main/java/au/org/aodn/esindexer/utils/UrlUtils.java diff --git a/indexer/src/main/java/au/org/aodn/esindexer/configuration/GeoNetworkSearchConfig.java b/indexer/src/main/java/au/org/aodn/esindexer/configuration/GeoNetworkSearchConfig.java index 231fd0a8..fc6f1166 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/configuration/GeoNetworkSearchConfig.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/configuration/GeoNetworkSearchConfig.java @@ -1,6 +1,7 @@ package au.org.aodn.esindexer.configuration; import au.org.aodn.esindexer.service.GeoNetworkServiceImpl; +import au.org.aodn.esindexer.utils.UrlUtils; import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.json.jackson.JacksonJsonpMapper; import co.elastic.clients.transport.rest_client.RestClientTransport; @@ -52,4 +53,10 @@ public GeoNetworkServiceImpl createGeoNetworkServiceImpl( return new GeoNetworkServiceImpl(server, indexName, gn4ElasticsearchClient, indexerRestTemplate); } + + @Bean + @ConditionalOnMissingBean(UrlUtils.class) + public UrlUtils createUrlUtils() { + return new UrlUtils(); + } } diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java b/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java index 73564d18..ec5b403d 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java @@ -3,6 +3,7 @@ import au.org.aodn.esindexer.exception.MappingValueException; import au.org.aodn.esindexer.utils.GeometryUtils; import au.org.aodn.esindexer.configuration.AppConstants; +import au.org.aodn.esindexer.utils.UrlUtils; import au.org.aodn.stac.model.*; import au.org.aodn.esindexer.utils.BBoxUtils; @@ -25,7 +26,6 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.*; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.stream.Collectors; @@ -69,6 +69,9 @@ public abstract class StacCollectionMapperService { @Autowired private GeoNetworkService geoNetworkService; + @Autowired + protected UrlUtils urlUtils; + @Named("mapUUID") String mapUUID(MDMetadataType source) { return source.getMetadataIdentifier().getMDIdentifier().getCode().getCharacterString().getValue().toString(); @@ -434,18 +437,19 @@ List mapLinks(MDMetadataType source) { Map additionalInfo = optAdditionalInfo.get(); if(additionalInfo.containsKey(SUGGEST_LOGOS)) { if(additionalInfo.get(SUGGEST_LOGOS) instanceof List) { - final AtomicInteger index = new AtomicInteger(1); ((List) additionalInfo.get(SUGGEST_LOGOS)) .stream() .map(p -> (p instanceof String) ? (String) p : null) .filter(Objects::nonNull) - .forEach(i -> { + .filter(i -> urlUtils.checkUrlExists(i)) + .findFirst() // We only pick the first reachable one + .ifPresent(i -> { LinkModel linkModel = LinkModel.builder().build(); linkModel.setHref(i); // Geonetwork always return png logo linkModel.setType("image/png"); linkModel.setRel("icon"); - linkModel.setTitle("Suggest icon for dataset " + index.getAndIncrement()); + linkModel.setTitle("Suggest icon for dataset"); results.add(linkModel); }); } diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/UrlUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/UrlUtils.java new file mode 100644 index 00000000..48e81c36 --- /dev/null +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/UrlUtils.java @@ -0,0 +1,23 @@ +package au.org.aodn.esindexer.utils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.RestTemplate; + +public class UrlUtils { + + @Autowired + protected RestTemplate restTemplate; + + public boolean checkUrlExists(String url) { + try { + ResponseEntity response = restTemplate.exchange(url, HttpMethod.HEAD, null, Void.class); + return response.getStatusCode() == HttpStatus.OK; + } + catch (Exception e) { + return false; + } + } +} diff --git a/indexer/src/main/resources/application-dev.yaml b/indexer/src/main/resources/application-dev.yaml index f3d7573b..c3990229 100644 --- a/indexer/src/main/resources/application-dev.yaml +++ b/indexer/src/main/resources/application-dev.yaml @@ -8,7 +8,7 @@ elasticsearch: apiKey: sample-api-key geonetwork: - host: https://geonetwork-edge.mvp.aodn.org.au + host: https://geonetwork-edge.edge.aodn.org.au search: api: index: "records" diff --git a/indexer/src/test/java/au/org/aodn/esindexer/configuration/GeoNetworkSearchTestConfig.java b/indexer/src/test/java/au/org/aodn/esindexer/configuration/GeoNetworkSearchTestConfig.java index 6d53b71e..efc5b745 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/configuration/GeoNetworkSearchTestConfig.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/configuration/GeoNetworkSearchTestConfig.java @@ -2,10 +2,11 @@ import au.org.aodn.esindexer.BaseTestClass; import au.org.aodn.esindexer.service.GeoNetworkServiceImpl; +import au.org.aodn.esindexer.utils.UrlUtils; import co.elastic.clients.elasticsearch.ElasticsearchClient; import lombok.extern.slf4j.Slf4j; import org.springframework.core.ParameterizedTypeReference; -import org.springframework.http.HttpEntity; +import org.springframework.http.*; import org.apache.http.HttpHeaders; import org.apache.http.HttpHost; import org.apache.http.message.BasicHeader; @@ -16,9 +17,6 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.http.HttpMethod; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; import org.springframework.web.client.RestTemplate; import org.testcontainers.containers.DockerComposeContainer; import org.testcontainers.containers.wait.strategy.Wait; @@ -28,6 +26,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.time.Duration; +import java.util.HashMap; import java.util.Map; import static org.mockito.ArgumentMatchers.eq; @@ -126,4 +125,24 @@ public GeoNetworkServiceImpl createGeoNetworkServiceImpl( return impl; } + /** + * Hardcode the result here for testing, please add more if you need to return different result + * @return + */ + @Bean + public UrlUtils createUrlUtils() { + UrlUtils urlUtils = Mockito.mock(UrlUtils.class); + + final Map status = new HashMap<>(); + status.put("https://catalogue-imos.aodn.org.au/geonetwork/images/logos/dbee258b-8730-4072-96d4-2818a69a4afd.png", Boolean.TRUE); + + doAnswer(answer -> { + String url = answer.getArgument(0); + return status.getOrDefault(url, Boolean.FALSE); + }) + .when(urlUtils) + .checkUrlExists(anyString()); + + return urlUtils; + } } diff --git a/indexer/src/test/java/au/org/aodn/esindexer/service/IndexerServiceTests.java b/indexer/src/test/java/au/org/aodn/esindexer/service/IndexerServiceTests.java index 95c384fd..0e561220 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/service/IndexerServiceTests.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/service/IndexerServiceTests.java @@ -137,6 +137,7 @@ public void verifyLogoLinkAddedOnIndex() throws IOException { Hit objectNodeHit = indexerService.getDocumentByUUID("2852a776-cbfc-4bc8-a126-f3c036814892"); String test = objectNodeHit.source().toPrettyString(); + logger.info(test); assertEquals("Stac equals", indexerObjectMapper.readTree(expected), indexerObjectMapper.readTree(test)); deleteRecord("2852a776-cbfc-4bc8-a126-f3c036814892"); diff --git a/indexer/src/test/resources/canned/sample5_stac.json b/indexer/src/test/resources/canned/sample5_stac.json index f34733f3..637ab28a 100644 --- a/indexer/src/test/resources/canned/sample5_stac.json +++ b/indexer/src/test/resources/canned/sample5_stac.json @@ -6,7 +6,7 @@ "temporal" : [ [ "2010-01-21T01:00:00Z", "2017-03-26T13:00:00Z" ], [ "2010-01-21T01:00:00Z", "2017-03-26T13:00:00Z" ] ] }, "summaries" : { - "score" : 90, + "score" : 85, "status" : "completed", "scope" : { "code" : "series", @@ -58,16 +58,11 @@ "rel" : "self", "type" : "text/html", "title" : "Ocean Radar page on IMOS website" - }, { - "href" : "http://localhost:8080/geonetwork/images/logos/dbee258b-8730-4072-96d4-2818a69a4afd.png", - "rel" : "icon", - "type" : "image/png", - "title" : "Suggest icon for dataset 1" }, { "href" : "https://catalogue-imos.aodn.org.au/geonetwork/images/logos/dbee258b-8730-4072-96d4-2818a69a4afd.png", "rel" : "icon", "type" : "image/png", - "title" : "Suggest icon for dataset 2" + "title" : "Suggest icon for dataset" } ], "license" : "Creative Commons Attribution 4.0 International License", "providers" : [ { @@ -122,6 +117,6 @@ "title" : "IMOS - ACORN - Bonney Coast HF ocean radar site (South Australia, Australia)" }, "type" : "Collection", - "stac_extensions" : [ "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", "https://stac-extensions.github.io/projection/v1.1.0/schema.json", "https://stac-extensions.github.io/language/v1.0.0/schema.json", "https://stac-extensions.github.io/themes/v1.0.0/schema.json" ], - "stac_version" : "1.0.0" + "stac_version" : "1.0.0", + "stac_extensions" : [ "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", "https://stac-extensions.github.io/projection/v1.1.0/schema.json", "https://stac-extensions.github.io/language/v1.0.0/schema.json", "https://stac-extensions.github.io/themes/v1.0.0/schema.json" ] }