select the first reachable icon

aodn · May 22, 2024 · b27917d · b27917d
1 parent 8fef893
commit b27917d
Show file tree

Hide file tree

Showing 7 changed files with 67 additions and 18 deletions.
diff --git a/indexer/src/main/java/au/org/aodn/esindexer/configuration/GeoNetworkSearchConfig.java b/indexer/src/main/java/au/org/aodn/esindexer/configuration/GeoNetworkSearchConfig.java
@@ -1,6 +1,7 @@
 package au.org.aodn.esindexer.configuration;
 
 import au.org.aodn.esindexer.service.GeoNetworkServiceImpl;
+import au.org.aodn.esindexer.utils.UrlUtils;
 import co.elastic.clients.elasticsearch.ElasticsearchClient;
 import co.elastic.clients.json.jackson.JacksonJsonpMapper;
 import co.elastic.clients.transport.rest_client.RestClientTransport;
@@ -52,4 +53,10 @@ public GeoNetworkServiceImpl createGeoNetworkServiceImpl(
 
         return new GeoNetworkServiceImpl(server, indexName, gn4ElasticsearchClient, indexerRestTemplate);
     }
+
+    @Bean
+    @ConditionalOnMissingBean(UrlUtils.class)
+    public UrlUtils createUrlUtils() {
+        return new UrlUtils();
+    }
 }
diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java b/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java
@@ -3,6 +3,7 @@
 import au.org.aodn.esindexer.exception.MappingValueException;
 import au.org.aodn.esindexer.utils.GeometryUtils;
 import au.org.aodn.esindexer.configuration.AppConstants;
+import au.org.aodn.esindexer.utils.UrlUtils;
 import au.org.aodn.stac.model.*;
 import au.org.aodn.esindexer.utils.BBoxUtils;
 
@@ -25,7 +26,6 @@
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.*;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -69,6 +69,9 @@ public abstract class StacCollectionMapperService {
     @Autowired
     private GeoNetworkService geoNetworkService;
 
+    @Autowired
+    protected UrlUtils urlUtils;
+
     @Named("mapUUID")
     String mapUUID(MDMetadataType source) {
         return source.getMetadataIdentifier().getMDIdentifier().getCode().getCharacterString().getValue().toString();
@@ -434,18 +437,19 @@ List<LinkModel> mapLinks(MDMetadataType source) {
             Map<String, Object> additionalInfo = optAdditionalInfo.get();
             if(additionalInfo.containsKey(SUGGEST_LOGOS)) {
                 if(additionalInfo.get(SUGGEST_LOGOS) instanceof List) {
-                    final AtomicInteger index = new AtomicInteger(1);
                     ((List<?>) additionalInfo.get(SUGGEST_LOGOS))
                             .stream()
                             .map(p -> (p instanceof String) ? (String) p : null)
                             .filter(Objects::nonNull)
-                            .forEach(i -> {
+                            .filter(i -> urlUtils.checkUrlExists(i))
+                            .findFirst()        // We only pick the first reachable one
+                            .ifPresent(i -> {
                                 LinkModel linkModel = LinkModel.builder().build();
                                 linkModel.setHref(i);
                                 // Geonetwork always return png logo
                                 linkModel.setType("image/png");
                                 linkModel.setRel("icon");
-                                linkModel.setTitle("Suggest icon for dataset " + index.getAndIncrement());
+                                linkModel.setTitle("Suggest icon for dataset");
                                 results.add(linkModel);
                             });
                 }

diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/UrlUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/UrlUtils.java
@@ -0,0 +1,23 @@
+package au.org.aodn.esindexer.utils;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.client.RestTemplate;
+
+public class UrlUtils {
+
+    @Autowired
+    protected RestTemplate restTemplate;
+
+    public boolean checkUrlExists(String url) {
+        try {
+            ResponseEntity<Void> response = restTemplate.exchange(url, HttpMethod.HEAD, null, Void.class);
+            return response.getStatusCode() == HttpStatus.OK;
+        }
+        catch (Exception e) {
+            return false;
+        }
+    }
+}
diff --git a/indexer/src/main/resources/application-dev.yaml b/indexer/src/main/resources/application-dev.yaml
@@ -8,7 +8,7 @@ elasticsearch:
   apiKey: sample-api-key
 
 geonetwork:
-  host: https://geonetwork-edge.mvp.aodn.org.au
+  host: https://geonetwork-edge.edge.aodn.org.au
   search:
     api:
       index: "records"

diff --git a/indexer/src/test/java/au/org/aodn/esindexer/configuration/GeoNetworkSearchTestConfig.java b/indexer/src/test/java/au/org/aodn/esindexer/configuration/GeoNetworkSearchTestConfig.java
@@ -2,10 +2,11 @@
 
 import au.org.aodn.esindexer.BaseTestClass;
 import au.org.aodn.esindexer.service.GeoNetworkServiceImpl;
+import au.org.aodn.esindexer.utils.UrlUtils;
 import co.elastic.clients.elasticsearch.ElasticsearchClient;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.core.ParameterizedTypeReference;
-import org.springframework.http.HttpEntity;
+import org.springframework.http.*;
 import org.apache.http.HttpHeaders;
 import org.apache.http.HttpHost;
 import org.apache.http.message.BasicHeader;
@@ -16,9 +17,6 @@
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
-import org.springframework.http.HttpMethod;
-import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
 import org.springframework.web.client.RestTemplate;
 import org.testcontainers.containers.DockerComposeContainer;
 import org.testcontainers.containers.wait.strategy.Wait;
@@ -28,6 +26,7 @@
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.time.Duration;
+import java.util.HashMap;
 import java.util.Map;
 
 import static org.mockito.ArgumentMatchers.eq;
@@ -126,4 +125,24 @@ public GeoNetworkServiceImpl createGeoNetworkServiceImpl(
 
         return impl;
     }
+    /**
+     * Hardcode the result here for testing, please add more if you need to return different result
+     * @return
+     */
+    @Bean
+    public UrlUtils createUrlUtils() {
+        UrlUtils urlUtils = Mockito.mock(UrlUtils.class);
+
+        final Map<String, Boolean> status = new HashMap<>();
+        status.put("https://catalogue-imos.aodn.org.au/geonetwork/images/logos/dbee258b-8730-4072-96d4-2818a69a4afd.png", Boolean.TRUE);
+
+        doAnswer(answer -> {
+            String url = answer.getArgument(0);
+            return status.getOrDefault(url, Boolean.FALSE);
+        })
+                .when(urlUtils)
+                .checkUrlExists(anyString());
+
+        return urlUtils;
+    }
 }
diff --git a/indexer/src/test/java/au/org/aodn/esindexer/service/IndexerServiceTests.java b/indexer/src/test/java/au/org/aodn/esindexer/service/IndexerServiceTests.java
@@ -137,6 +137,7 @@ public void verifyLogoLinkAddedOnIndex() throws IOException {
         Hit<ObjectNode> objectNodeHit = indexerService.getDocumentByUUID("2852a776-cbfc-4bc8-a126-f3c036814892");
 
         String test = objectNodeHit.source().toPrettyString();
+        logger.info(test);
         assertEquals("Stac equals", indexerObjectMapper.readTree(expected), indexerObjectMapper.readTree(test));
 
         deleteRecord("2852a776-cbfc-4bc8-a126-f3c036814892");

diff --git a/indexer/src/test/resources/canned/sample5_stac.json b/indexer/src/test/resources/canned/sample5_stac.json
@@ -6,7 +6,7 @@
     "temporal" : [ [ "2010-01-21T01:00:00Z", "2017-03-26T13:00:00Z" ], [ "2010-01-21T01:00:00Z", "2017-03-26T13:00:00Z" ] ]
   },
   "summaries" : {
-    "score" : 90,
+    "score" : 85,
     "status" : "completed",
     "scope" : {
       "code" : "series",
@@ -58,16 +58,11 @@
     "rel" : "self",
     "type" : "text/html",
     "title" : "Ocean Radar page on IMOS website"
-  }, {
-    "href" : "http://localhost:8080/geonetwork/images/logos/dbee258b-8730-4072-96d4-2818a69a4afd.png",
-    "rel" : "icon",
-    "type" : "image/png",
-    "title" : "Suggest icon for dataset 1"
   }, {
     "href" : "https://catalogue-imos.aodn.org.au/geonetwork/images/logos/dbee258b-8730-4072-96d4-2818a69a4afd.png",
     "rel" : "icon",
     "type" : "image/png",
-    "title" : "Suggest icon for dataset 2"
+    "title" : "Suggest icon for dataset"
   } ],
   "license" : "Creative Commons Attribution 4.0 International License",
   "providers" : [ {
@@ -122,6 +117,6 @@
     "title" : "IMOS - ACORN - Bonney Coast HF ocean radar site (South Australia, Australia)"
   },
   "type" : "Collection",
-  "stac_extensions" : [ "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", "https://stac-extensions.github.io/projection/v1.1.0/schema.json", "https://stac-extensions.github.io/language/v1.0.0/schema.json", "https://stac-extensions.github.io/themes/v1.0.0/schema.json" ],
-  "stac_version" : "1.0.0"
+  "stac_version" : "1.0.0",
+  "stac_extensions" : [ "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", "https://stac-extensions.github.io/contacts/v0.1.1/schema.json", "https://stac-extensions.github.io/projection/v1.1.0/schema.json", "https://stac-extensions.github.io/language/v1.0.0/schema.json", "https://stac-extensions.github.io/themes/v1.0.0/schema.json" ]
 }