diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/VocabApiPaths.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/VocabApiPaths.java deleted file mode 100644 index b078cc92..00000000 --- a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/configuration/VocabApiPaths.java +++ /dev/null @@ -1,37 +0,0 @@ -package au.org.aodn.ardcvocabs.configuration; - -import lombok.Getter; - -@Getter -public enum VocabApiPaths { - PARAMETER_VOCAB( - "/aodn-parameter-category-vocabulary/version-2-1/concept.json", - "/aodn-parameter-category-vocabulary/version-2-1/resource.json?uri=%s", - "/aodn-discovery-parameter-vocabulary/version-1-6/concept.json", - "/aodn-discovery-parameter-vocabulary/version-1-6/resource.json?uri=%s" - ), - PLATFORM_VOCAB( - "/aodn-platform-category-vocabulary/version-1-2/concept.json", - "/aodn-platform-category-vocabulary/version-1-2/resource.json?uri=%s", - "/aodn-platform-vocabulary/version-6-1/concept.json", - "/aodn-platform-vocabulary/version-6-1/resource.json?uri=%s" - ), - ORGANISATION_VOCAB( - "/aodn-organisation-category-vocabulary/version-2-5/concept", - "/aodn-organisation-category-vocabulary/version-2-5/resource.json?uri=%s", - "/aodn-organisation-vocabulary/version-2-5/concept", - "/aodn-organisation-vocabulary/version-2-5/resource.json?uri=%s" - ); - - private final String vocabCategoryApiPath; - private final String vocabCategoryDetailsApiPath; - private final String vocabApiPath; - private final String vocabDetailsApiPath; - - VocabApiPaths(String vocabCategoryApiPath, String vocabCategoryDetailsApiPath, String vocabApiPath, String vocabDetailsApiPath) { - this.vocabCategoryApiPath = vocabCategoryApiPath; - this.vocabCategoryDetailsApiPath = vocabCategoryDetailsApiPath; - this.vocabApiPath = vocabApiPath; - this.vocabDetailsApiPath = vocabDetailsApiPath; - } -} diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/model/ArdcCurrentPaths.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/model/ArdcCurrentPaths.java new file mode 100644 index 00000000..1644c25b --- /dev/null +++ b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/model/ArdcCurrentPaths.java @@ -0,0 +1,29 @@ +package au.org.aodn.ardcvocabs.model; + +import lombok.Getter; + +@Getter +public enum ArdcCurrentPaths { + PARAMETER_VOCAB( + "/aodn-parameter-category-vocabulary/current/concept.json", + "/aodn-discovery-parameter-vocabulary/current/concept.json" + ), + PLATFORM_VOCAB( + "/aodn-platform-category-vocabulary/current/concept.json", + "/aodn-platform-vocabulary/current/concept.json" + ), + ORGANISATION_VOCAB( + "/aodn-organisation-category-vocabulary/current/concept.json", + "/aodn-organisation-vocabulary/current/concept.json" + ); + + + private final String categoryCurrent; + private final String vocabCurrent; + + ArdcCurrentPaths(String categoryCurrent, String vocabCurrent) { + String baseUrl = "https://vocabs.ardc.edu.au/repository/api/lda/aodn"; + this.categoryCurrent = baseUrl + categoryCurrent; + this.vocabCurrent = baseUrl + vocabCurrent; + } +} diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/model/PathName.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/model/PathName.java new file mode 100644 index 00000000..9f3a673d --- /dev/null +++ b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/model/PathName.java @@ -0,0 +1,8 @@ +package au.org.aodn.ardcvocabs.model; + +public enum PathName { + categoryApi, + categoryDetailsApi, + vocabApi, + vocabDetailsApi +} diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/model/VocabApiPaths.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/model/VocabApiPaths.java index ab5f7b7b..5ffa4793 100644 --- a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/model/VocabApiPaths.java +++ b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/model/VocabApiPaths.java @@ -1,37 +1,36 @@ package au.org.aodn.ardcvocabs.model; - import lombok.Getter; @Getter public enum VocabApiPaths { PARAMETER_VOCAB( - "/aodn-parameter-category-vocabulary/version-2-1/concept.json", - "/aodn-parameter-category-vocabulary/version-2-1/resource.json?uri=%s", - "/aodn-discovery-parameter-vocabulary/version-1-6/concept.json", - "/aodn-discovery-parameter-vocabulary/version-1-6/resource.json?uri=%s" + "/aodn-parameter-category-vocabulary/%s/concept.json", + "/aodn-parameter-category-vocabulary/%s/resource.json?uri=%s", + "/aodn-discovery-parameter-vocabulary/%s/concept.json", + "/aodn-discovery-parameter-vocabulary/%s/resource.json?uri=%s" ), PLATFORM_VOCAB( - "/aodn-platform-category-vocabulary/version-1-2/concept.json", - "/aodn-platform-category-vocabulary/version-1-2/resource.json?uri=%s", - "/aodn-platform-vocabulary/version-6-1/concept.json", - "/aodn-platform-vocabulary/version-6-1/resource.json?uri=%s" + "/aodn-platform-category-vocabulary/%s/concept.json", + "/aodn-platform-category-vocabulary/%s/resource.json?uri=%s", + "/aodn-platform-vocabulary/%s/concept.json", + "/aodn-platform-vocabulary/%s/resource.json?uri=%s" ), ORGANISATION_VOCAB( - "/aodn-organisation-category-vocabulary/version-2-5/concept.json", - "/aodn-organisation-category-vocabulary/version-2-5/resource.json?uri=%s", - "/aodn-organisation-vocabulary/version-2-5/concept.json", - "/aodn-organisation-vocabulary/version-2-5/resource.json?uri=%s" + "/aodn-organisation-category-vocabulary/%s/concept.json", + "/aodn-organisation-category-vocabulary/%s/resource.json?uri=%s", + "/aodn-organisation-vocabulary/%s/concept.json", + "/aodn-organisation-vocabulary/%s/resource.json?uri=%s" ); - private final String vocabCategoryApiPath; - private final String vocabCategoryDetailsApiPath; - private final String vocabApiPath; - private final String vocabDetailsApiPath; + private final String categoryApiTemplate; + private final String categoryDetailsTemplate; + private final String vocabApiTemplate; + private final String vocabDetailsTemplate; - VocabApiPaths(String vocabCategoryApiPath, String vocabCategoryDetailsApiPath, String vocabApiPath, String vocabDetailsApiPath) { - this.vocabCategoryApiPath = vocabCategoryApiPath; - this.vocabCategoryDetailsApiPath = vocabCategoryDetailsApiPath; - this.vocabApiPath = vocabApiPath; - this.vocabDetailsApiPath = vocabDetailsApiPath; + VocabApiPaths(String categoryApiTemplate, String categoryDetailsTemplate, String vocabApiTemplate, String vocabDetailsTemplate) { + this.categoryApiTemplate = categoryApiTemplate; + this.categoryDetailsTemplate = categoryDetailsTemplate; + this.vocabApiTemplate = vocabApiTemplate; + this.vocabDetailsTemplate = vocabDetailsTemplate; } } diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabService.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabService.java index aea64f32..ddea556f 100644 --- a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabService.java +++ b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabService.java @@ -1,10 +1,13 @@ package au.org.aodn.ardcvocabs.service; +import au.org.aodn.ardcvocabs.model.PathName; import au.org.aodn.ardcvocabs.model.VocabApiPaths; import au.org.aodn.ardcvocabs.model.VocabModel; import java.util.List; +import java.util.Map; public interface ArdcVocabService { - List getVocabTreeFromArdcByType(VocabApiPaths vocabApiPaths); + Map> getResolvedPathCollection(); + List getVocabTreeFromArdcByType(Map resolvedPaths); } diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabServiceImpl.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabServiceImpl.java index a736a6e0..ee562f8a 100644 --- a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabServiceImpl.java +++ b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabServiceImpl.java @@ -1,5 +1,7 @@ package au.org.aodn.ardcvocabs.service; +import au.org.aodn.ardcvocabs.model.ArdcCurrentPaths; +import au.org.aodn.ardcvocabs.model.PathName; import au.org.aodn.ardcvocabs.model.VocabApiPaths; import au.org.aodn.ardcvocabs.model.VocabModel; import com.fasterxml.jackson.databind.JsonNode; @@ -22,7 +24,7 @@ public class ArdcVocabServiceImpl implements ArdcVocabService { - protected Logger log = LoggerFactory.getLogger(ArdcVocabServiceImpl.class); + protected static Logger log = LoggerFactory.getLogger(ArdcVocabServiceImpl.class); @Value("${ardcvocabs.baseUrl:https://vocabs.ardc.edu.au/repository/api/lda/aodn}") protected String vocabApiBase; @@ -30,6 +32,86 @@ public class ArdcVocabServiceImpl implements ArdcVocabService { protected RestTemplate restTemplate; protected RetryTemplate retryTemplate; + protected static final String VERSION_REGEX = "/(version-\\d+-\\d+)(?:/|$)"; + + public Map> getResolvedPathCollection() { + Map> resolvedPathCollection = new HashMap<>(); + for (ArdcCurrentPaths currentPath : ArdcCurrentPaths.values()) { + try { + ObjectNode categoryCurrentContent = fetchCurrentContents(currentPath.getCategoryCurrent()); + ObjectNode vocabCurrentContent = fetchCurrentContents(currentPath.getVocabCurrent()); + + if (categoryCurrentContent != null && vocabCurrentContent != null) { + // Extract versions + String categoryVersion = extractVersionFromCurrentContent(categoryCurrentContent); + String vocabVersion = extractVersionFromCurrentContent(vocabCurrentContent); + + if (categoryVersion != null && vocabVersion != null) { + log.info("Fetched ARDC category version for {}: {}", currentPath.name(), categoryVersion); + log.info("Fetched ARDC vocab version for {}: {}", currentPath.name(), vocabVersion); + + // Build and store resolved paths + Map resolvedPaths = buildResolvedPaths(currentPath, categoryVersion, vocabVersion); + resolvedPathCollection.put(currentPath.name(), resolvedPaths); + } else { + log.error("Failed to extract versions for {}", currentPath.name()); + } + } else { + log.error("Failed to fetch HTML content for {}", currentPath.name()); + } + } catch (Exception e) { + log.error("Error initialising versions for {}: {}", currentPath.name(), e.getMessage(), e); + } + } + return resolvedPathCollection; + } + + private ObjectNode fetchCurrentContents(String url) { + try { + return retryTemplate.execute(context -> restTemplate.getForObject(url, ObjectNode.class)); + } catch (RestClientException e) { + log.error("Failed to fetch HTML content from URL {}: {}", url, e.getMessage()); + } catch (Exception e) { + log.error("Unexpected error while fetching HTML content from URL {}: {}", url, e.getMessage(), e); + } + return null; + } + + protected Map buildResolvedPaths(ArdcCurrentPaths currentPaths, String categoryVersion, String vocabVersion) { + Map resolvedPaths = new HashMap<>(); + for (VocabApiPaths vocabApiPath : VocabApiPaths.values()) { + if (currentPaths.name().equals(vocabApiPath.name())) { + resolvedPaths.put(PathName.categoryApi, String.format(vocabApiPath.getCategoryApiTemplate(), categoryVersion)); + resolvedPaths.put(PathName.categoryDetailsApi, String.format(vocabApiPath.getCategoryDetailsTemplate(), categoryVersion, "%s")); + resolvedPaths.put(PathName.vocabApi, String.format(vocabApiPath.getVocabApiTemplate(), vocabVersion)); + resolvedPaths.put(PathName.vocabDetailsApi, String.format(vocabApiPath.getVocabDetailsTemplate(), vocabVersion, "%s")); + } + } + return resolvedPaths; + } + + protected String extractVersionFromCurrentContent(ObjectNode currentContent) { + if (currentContent != null && !currentContent.isEmpty()) { + JsonNode node = currentContent.get("result"); + if (!about.apply(node).isEmpty()) { + Pattern pattern = Pattern.compile(VERSION_REGEX); + Matcher matcher = pattern.matcher(about.apply(node)); + + if (matcher.find()) { + String version = matcher.group(1); + log.info("Valid Version Found: {}", version); + return version; + } else { + log.warn("Version does not match the required format: {}", about.apply(node)); + } + } + + } else { + log.warn("Current content is empty or null."); + } + return null; + } + protected Function extractSingleText(String key) { return (node) -> { JsonNode labelNode = node.get(key); @@ -90,10 +172,10 @@ public ArdcVocabServiceImpl(RestTemplate restTemplate, RetryTemplate retryTempla this.retryTemplate = retryTemplate; } - protected VocabModel buildVocabByResourceUri(String vocabUri, String vocabApiBase, VocabApiPaths vocabApiPaths) { + protected VocabModel buildVocabByResourceUri(String vocabUri, String vocabApiBase, Map resolvedPaths) { String resourceDetailsApi = vocabUri.contains("_classes") - ? vocabApiPaths.getVocabCategoryDetailsApiPath() - : vocabApiPaths.getVocabDetailsApiPath(); + ? resolvedPaths.get(PathName.categoryDetailsApi) + : resolvedPaths.get(PathName.vocabDetailsApi); String detailsUrl = String.format(vocabApiBase + resourceDetailsApi, vocabUri); @@ -123,7 +205,7 @@ protected VocabModel buildVocabByResourceUri(String vocabUri, String vocabApiBas for (JsonNode j : target.get("narrower")) { if (!about.apply(j).isEmpty()) { // recursive call - VocabModel narrowerNode = buildVocabByResourceUri(about.apply(j), vocabApiBase, vocabApiPaths); + VocabModel narrowerNode = buildVocabByResourceUri(about.apply(j), vocabApiBase, resolvedPaths); if (narrowerNode != null) { narrowerNodes.add(narrowerNode); } @@ -143,7 +225,7 @@ protected VocabModel buildVocabByResourceUri(String vocabUri, String vocabApiBas return null; } - protected VocabModel buildVocabModel(T currentNode, String vocabApiBase, VocabApiPaths vocabApiPaths) { + protected VocabModel buildVocabModel(T currentNode, String vocabApiBase, Map resolvedPaths) { String resourceUri = null; if (currentNode instanceof ObjectNode objectNode) { @@ -161,12 +243,12 @@ protected VocabModel buildVocabModel(T currentNode, String vocabApiBase, Voc throw new IllegalArgumentException("Unsupported node type: " + currentNode.getClass().getName()); } - return buildVocabByResourceUri(resourceUri, vocabApiBase, vocabApiPaths); + return buildVocabByResourceUri(resourceUri, vocabApiBase, resolvedPaths); } - protected Map> getVocabLeafNodes(String vocabApiBase, VocabApiPaths vocabApiPaths) { + protected Map> getVocabLeafNodes(String vocabApiBase, Map resolvedPaths) { Map> results = new HashMap<>(); - String url = String.format(vocabApiBase + vocabApiPaths.getVocabApiPath()); + String url = String.format(vocabApiBase + resolvedPaths.get(PathName.vocabApi)); while (url != null && !url.isEmpty()) { try { @@ -180,7 +262,7 @@ protected Map> getVocabLeafNodes(String vocabApiBase, V if (isNodeValid.apply(node, "items")) { for (JsonNode j : node.get("items")) { // Now we need to construct link to detail resources - String dl = String.format(vocabApiBase + vocabApiPaths.getVocabDetailsApiPath(), about.apply(j)); + String dl = String.format(vocabApiBase + resolvedPaths.get(PathName.vocabDetailsApi), about.apply(j)); try { log.debug("getVocabLeafNodes -> {}", dl); ObjectNode d = retryTemplate.execute(context -> restTemplate.getForObject(dl, ObjectNode.class)); @@ -205,7 +287,7 @@ protected Map> getVocabLeafNodes(String vocabApiBase, V List vocabNarrower = new ArrayList<>(); if(target.has("narrower") && !target.get("narrower").isEmpty()) { for(JsonNode currentNode : target.get("narrower")) { - VocabModel narrowerNode = buildVocabModel(currentNode, vocabApiBase, vocabApiPaths); + VocabModel narrowerNode = buildVocabModel(currentNode, vocabApiBase, resolvedPaths); if (narrowerNode != null) { vocabNarrower.add(narrowerNode); } @@ -230,7 +312,7 @@ protected Map> getVocabLeafNodes(String vocabApiBase, V List completedInternalNodes = new ArrayList<>(); vocab.getNarrower().forEach(currentInternalNode -> { // rebuild currentInternalNode (no linked leaf nodes) to completedInternalNode (with linked leaf nodes) - VocabModel completedInternalNode = buildVocabModel(currentInternalNode, vocabApiBase, vocabApiPaths); + VocabModel completedInternalNode = buildVocabModel(currentInternalNode, vocabApiBase, resolvedPaths); if (completedInternalNode != null) { // each internal node now will have linked narrower nodes (if available) completedInternalNodes.add(completedInternalNode); @@ -269,9 +351,9 @@ protected Map> getVocabLeafNodes(String vocabApiBase, V } @Override - public List getVocabTreeFromArdcByType(VocabApiPaths vocabApiPaths) { - Map> vocabLeafNodes = getVocabLeafNodes(vocabApiBase, vocabApiPaths); - String url = String.format(vocabApiBase + vocabApiPaths.getVocabCategoryApiPath()); + public List getVocabTreeFromArdcByType(Map resolvedPaths) { + Map> vocabLeafNodes = getVocabLeafNodes(vocabApiBase, resolvedPaths); + String url = String.format(vocabApiBase + resolvedPaths.get(PathName.categoryApi)); List vocabCategoryNodes = new ArrayList<>(); while (url != null && !url.isEmpty()) { try { @@ -299,7 +381,7 @@ public List getVocabTreeFromArdcByType(VocabApiPaths vocabApiPaths) Map> internalVocabCategoryNodes = new HashMap<>(); if (j.has("narrower") && !j.get("narrower").isEmpty()) { j.get("narrower").forEach(currentNode -> { - VocabModel internalNode = buildVocabModel(currentNode, vocabApiBase, vocabApiPaths); + VocabModel internalNode = buildVocabModel(currentNode, vocabApiBase, resolvedPaths); if (internalNode != null) { List leafNodes = vocabLeafNodes.getOrDefault(internalNode.getAbout(), Collections.emptyList()); if (!leafNodes.isEmpty()) { diff --git a/ardcvocabs/src/test/java/au/org/aodn/ardcvocabs/service/ArdcVocabServiceImplTest.java b/ardcvocabs/src/test/java/au/org/aodn/ardcvocabs/service/ArdcVocabServiceImplTest.java index 0feed35f..8286e21e 100644 --- a/ardcvocabs/src/test/java/au/org/aodn/ardcvocabs/service/ArdcVocabServiceImplTest.java +++ b/ardcvocabs/src/test/java/au/org/aodn/ardcvocabs/service/ArdcVocabServiceImplTest.java @@ -1,6 +1,7 @@ package au.org.aodn.ardcvocabs.service; import au.org.aodn.ardcvocabs.BaseTestClass; +import au.org.aodn.ardcvocabs.model.PathName; import au.org.aodn.ardcvocabs.model.VocabApiPaths; import au.org.aodn.ardcvocabs.model.VocabModel; import com.fasterxml.jackson.databind.ObjectMapper; @@ -14,24 +15,21 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.stubbing.Answer; import org.skyscreamer.jsonassert.JSONAssert; import org.skyscreamer.jsonassert.JSONCompareMode; -import org.springframework.retry.RetryCallback; -import org.springframework.retry.RetryContext; import org.springframework.retry.support.RetryTemplate; import org.springframework.web.client.RestTemplate; import java.io.FileNotFoundException; import java.io.IOException; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import static org.springframework.test.util.AssertionErrors.assertTrue; @ExtendWith(MockitoExtension.class) @@ -40,6 +38,8 @@ public class ArdcVocabServiceImplTest extends BaseTestClass { protected ArdcVocabServiceImpl ardcVocabService; protected ObjectMapper mapper = new ObjectMapper(); + protected Map> resolvedPathCollection = new HashMap<>(); + @Mock RestTemplate mockRestTemplate; @@ -199,6 +199,27 @@ public void init() { //this.ardcVocabService = new ArdcVocabServiceImpl(new RestTemplate()); this.ardcVocabService = new ArdcVocabServiceImpl(mockRestTemplate, new RetryTemplate()); this.ardcVocabService.vocabApiBase = "https://vocabs.ardc.edu.au/repository/api/lda/aodn"; + + resolvedPathCollection.put(VocabApiPaths.PARAMETER_VOCAB.name(), Map.of( + PathName.vocabApi, "/aodn-discovery-parameter-vocabulary/version-1-6/concept.json", + PathName.categoryApi, "/aodn-parameter-category-vocabulary/version-2-1/concept.json", + PathName.categoryDetailsApi, "/aodn-parameter-category-vocabulary/version-2-1/resource.json?uri=%s", + PathName.vocabDetailsApi, "/aodn-discovery-parameter-vocabulary/version-1-6/resource.json?uri=%s" + )); + + resolvedPathCollection.put(VocabApiPaths.PLATFORM_VOCAB.name(), Map.of( + PathName.vocabApi, "/aodn-platform-vocabulary/version-6-1/concept.json", + PathName.categoryApi, "/aodn-platform-category-vocabulary/version-1-2/concept.json", + PathName.categoryDetailsApi, "/aodn-platform-category-vocabulary/version-1-2/resource.json?uri=%s", + PathName.vocabDetailsApi, "/aodn-platform-vocabulary/version-6-1/resource.json?uri=%s" + )); + + resolvedPathCollection.put(VocabApiPaths.ORGANISATION_VOCAB.name(), Map.of( + PathName.vocabApi, "/aodn-organisation-vocabulary/version-2-5/concept.json", + PathName.categoryApi, "/aodn-organisation-category-vocabulary/version-2-5/concept.json", + PathName.categoryDetailsApi, "/aodn-organisation-category-vocabulary/version-2-5/resource.json?uri=%s", + PathName.vocabDetailsApi, "/aodn-organisation-vocabulary/version-2-5/resource.json?uri=%s" + )); } @AfterEach void clear() { @@ -210,7 +231,7 @@ public void verifyParameterVocab() throws IOException, JSONException { mockRestTemplate = setupParameterVocabMockRestTemplate(mockRestTemplate); - List parameterVocabModelList = ardcVocabService.getVocabTreeFromArdcByType(VocabApiPaths.PARAMETER_VOCAB); + List parameterVocabModelList = ardcVocabService.getVocabTreeFromArdcByType(resolvedPathCollection.get(VocabApiPaths.PARAMETER_VOCAB.name())); assertEquals(4, parameterVocabModelList.size(), "Total equals"); Optional c = parameterVocabModelList @@ -304,7 +325,7 @@ public void verifyParameterVocab() throws IOException, JSONException { public void verifyPlatform() throws IOException, JSONException { mockRestTemplate = setupPlatformMockRestTemplate(mockRestTemplate); - List platformVocabsFromArdc = ardcVocabService.getVocabTreeFromArdcByType(VocabApiPaths.PLATFORM_VOCAB); + List platformVocabsFromArdc = ardcVocabService.getVocabTreeFromArdcByType(resolvedPathCollection.get(VocabApiPaths.PLATFORM_VOCAB.name())); // verify the contents randomly assertNotNull(platformVocabsFromArdc); @@ -344,7 +365,7 @@ public void verifyPlatform() throws IOException, JSONException { public void verifyOrganization() throws IOException, JSONException { mockRestTemplate = setupOrganizationMockRestTemplate(mockRestTemplate); - List organisationVocabsFromArdc = ardcVocabService.getVocabTreeFromArdcByType(VocabApiPaths.ORGANISATION_VOCAB); + List organisationVocabsFromArdc = ardcVocabService.getVocabTreeFromArdcByType(resolvedPathCollection.get(VocabApiPaths.ORGANISATION_VOCAB.name())); // verify the contents randomly assertNotNull(organisationVocabsFromArdc); diff --git a/indexer/src/main/java/au/org/aodn/esindexer/controller/IndexerExtController.java b/indexer/src/main/java/au/org/aodn/esindexer/controller/IndexerExtController.java index 7c235e61..54e5ff03 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/controller/IndexerExtController.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/controller/IndexerExtController.java @@ -65,7 +65,7 @@ public ResponseEntity> getOrganisationVocabs() throws IOException @GetMapping(path="/ardc/parameter/vocabs") @Operation(security = { @SecurityRequirement(name = "X-API-Key") }, description = "Get parameter vocabs from ARDC directly") public ResponseEntity> getParameterVocabsFromArdc() { - List vocabs = ardcVocabService.getVocabTreeFromArdcByType(VocabApiPaths.PARAMETER_VOCAB); + List vocabs = ardcVocabService.getVocabTreeFromArdcByType(ardcVocabService.getResolvedPathCollection().get(VocabApiPaths.PARAMETER_VOCAB.name())); return ResponseEntity.ok(indexerObjectMapper.valueToTree(vocabs)); } @@ -73,7 +73,7 @@ public ResponseEntity> getParameterVocabsFromArdc() { @GetMapping(path="/ardc/platform/vocabs") @Operation(security = { @SecurityRequirement(name = "X-API-Key") }, description = "Get platform vocabs from ARDC directly") public ResponseEntity> getPlatformVocabsFromArdc() { - List vocabs = ardcVocabService.getVocabTreeFromArdcByType(VocabApiPaths.PLATFORM_VOCAB); + List vocabs = ardcVocabService.getVocabTreeFromArdcByType(ardcVocabService.getResolvedPathCollection().get(VocabApiPaths.PLATFORM_VOCAB.name())); return ResponseEntity.ok(indexerObjectMapper.valueToTree(vocabs)); } @@ -81,7 +81,7 @@ public ResponseEntity> getPlatformVocabsFromArdc() { @GetMapping(path="/ardc/organisation/vocabs") @Operation(security = { @SecurityRequirement(name = "X-API-Key") }, description = "Get organisation vocabs from ARDC directly") public ResponseEntity> getOrganisationVocabsFromArdc() { - List vocabs = ardcVocabService.getVocabTreeFromArdcByType(VocabApiPaths.ORGANISATION_VOCAB); + List vocabs = ardcVocabService.getVocabTreeFromArdcByType(ardcVocabService.getResolvedPathCollection().get(VocabApiPaths.ORGANISATION_VOCAB.name())); return ResponseEntity.ok(indexerObjectMapper.valueToTree(vocabs)); } @@ -94,7 +94,7 @@ public ResponseEntity populateDataToVocabsIndex() throws IOException, Ex vocabService.clearPlatformVocabCache(); vocabService.clearOrganisationVocabCache(); // populate new data - vocabService.populateVocabsData(); + vocabService.populateVocabsData(ardcVocabService.getResolvedPathCollection()); return ResponseEntity.status(HttpStatus.NO_CONTENT).body("Populated data to the vocabs index"); } } diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/VocabService.java b/indexer/src/main/java/au/org/aodn/esindexer/service/VocabService.java index bb308d27..c78e7026 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/VocabService.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/VocabService.java @@ -1,5 +1,6 @@ package au.org.aodn.esindexer.service; +import au.org.aodn.ardcvocabs.model.PathName; import au.org.aodn.ardcvocabs.model.VocabModel; import au.org.aodn.stac.model.ContactsModel; import au.org.aodn.stac.model.ThemesModel; @@ -7,13 +8,14 @@ import java.io.IOException; import java.util.List; +import java.util.Map; public interface VocabService { List extractVocabLabelsFromThemes(List themes, String vocabType) throws IOException; List extractOrganisationVocabLabelsFromThemes(List themes) throws IOException; List getMappedOrganisationVocabsFromContacts(List contacts) throws IOException; - void populateVocabsData() throws IOException; - void populateVocabsDataAsync(); + void populateVocabsData(Map> resolvedPathCollection) throws IOException; + void populateVocabsDataAsync(Map> resolvedPathCollection); void clearParameterVocabCache(); void clearPlatformVocabCache(); void clearOrganisationVocabCache(); diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/VocabServiceImpl.java b/indexer/src/main/java/au/org/aodn/esindexer/service/VocabServiceImpl.java index f2fea138..5c80fdab 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/VocabServiceImpl.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/VocabServiceImpl.java @@ -1,5 +1,6 @@ package au.org.aodn.esindexer.service; +import au.org.aodn.ardcvocabs.model.PathName; import au.org.aodn.ardcvocabs.model.VocabApiPaths; import au.org.aodn.ardcvocabs.model.VocabDto; import au.org.aodn.ardcvocabs.model.VocabModel; @@ -370,24 +371,24 @@ protected void bulkIndexVocabs(List vocabs) throws IOException { } } - public void populateVocabsData() throws IOException { + public void populateVocabsData(Map> resolvedPathCollection) throws IOException { log.info("Starting fetching vocabs data process synchronously..."); - List parameterVocabs = ardcVocabService.getVocabTreeFromArdcByType(VocabApiPaths.PARAMETER_VOCAB); - List platformVocabs = ardcVocabService.getVocabTreeFromArdcByType(VocabApiPaths.PLATFORM_VOCAB); - List organisationVocabs = ardcVocabService.getVocabTreeFromArdcByType(VocabApiPaths.ORGANISATION_VOCAB); + List parameterVocabs = ardcVocabService.getVocabTreeFromArdcByType(resolvedPathCollection.get(VocabApiPaths.PARAMETER_VOCAB.name())); + List platformVocabs = ardcVocabService.getVocabTreeFromArdcByType(resolvedPathCollection.get(VocabApiPaths.PLATFORM_VOCAB.name())); + List organisationVocabs = ardcVocabService.getVocabTreeFromArdcByType(resolvedPathCollection.get(VocabApiPaths.ORGANISATION_VOCAB.name())); indexAllVocabs(parameterVocabs, platformVocabs, organisationVocabs); } - public void populateVocabsDataAsync() { + public void populateVocabsDataAsync(Map> resolvedPathCollection) { log.info("Starting async vocabs data fetching process..."); ExecutorService executorService = Executors.newFixedThreadPool(3); List>> vocabTasks = List.of( - createVocabFetchTask(VocabApiPaths.PARAMETER_VOCAB, "parameter"), - createVocabFetchTask(VocabApiPaths.PLATFORM_VOCAB, "platform"), - createVocabFetchTask(VocabApiPaths.ORGANISATION_VOCAB, "organisation") + createVocabFetchTask(resolvedPathCollection.get(VocabApiPaths.PARAMETER_VOCAB.name()), "parameter"), + createVocabFetchTask(resolvedPathCollection.get(VocabApiPaths.PLATFORM_VOCAB.name()), "platform"), + createVocabFetchTask(resolvedPathCollection.get(VocabApiPaths.ORGANISATION_VOCAB.name()), "organisation") ); CompletableFuture.runAsync(() -> { @@ -422,10 +423,10 @@ public void populateVocabsDataAsync() { log.info("Vocabs data fetching process started in the background."); } - private Callable> createVocabFetchTask(VocabApiPaths vocabType, String vocabName) { + private Callable> createVocabFetchTask(Map resolvedPaths, String vocabName) { return () -> { log.info("Fetching {} vocabs from ARDC", vocabName); - return ardcVocabService.getVocabTreeFromArdcByType(vocabType); + return ardcVocabService.getVocabTreeFromArdcByType(resolvedPaths); }; } } diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/VocabsIndexUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/VocabsIndexUtils.java index 155d921e..3dcb15e6 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/VocabsIndexUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/VocabsIndexUtils.java @@ -1,5 +1,7 @@ package au.org.aodn.esindexer.utils; +import au.org.aodn.ardcvocabs.model.PathName; +import au.org.aodn.ardcvocabs.service.ArdcVocabService; import au.org.aodn.esindexer.service.VocabService; import jakarta.annotation.PostConstruct; import lombok.extern.slf4j.Slf4j; @@ -8,6 +10,7 @@ import org.springframework.scheduling.annotation.Scheduled; import java.io.IOException; +import java.util.Map; @Slf4j @@ -24,30 +27,67 @@ public void setVocabService(VocabService vocabService) { this.vocabService = vocabService; } + protected ArdcVocabService ardcVocabService; + @Autowired + public void setArdcVocabService(ArdcVocabService ardcVocabService) { + this.ardcVocabService = ardcVocabService; + } + + /* + The storedResolvedPathCollection is shared between the @PostConstruct method and the @Scheduled method. + If the scheduledRefreshVocabsData method runs while init is still processing, there could be concurrency issues. + To mitigate this, synchronize access to this shared resource with volatile keyword to ensure proper visibility. + The volatile modifier guarantees that any thread that reads a field will see the most recently written value + */ + private volatile Map> storedResolvedPathCollection; + @PostConstruct public void init() throws IOException { // Check if the initialiseVocabsIndex flag is enabled if (initialiseVocabsIndex) { log.info("Initialising {} asynchronously", vocabsIndexName); - vocabService.populateVocabsDataAsync(); + storedResolvedPathCollection = ardcVocabService.getResolvedPathCollection(); + vocabService.populateVocabsDataAsync(storedResolvedPathCollection); } } @Scheduled(cron = "0 0 0 * * *") - public void scheduledRefreshVocabsData() throws IOException { - log.info("Refreshing ARDC vocabularies data"); + public void scheduledRefreshVocabsData() { + try { + log.info("Refreshing ARDC vocabularies data"); + Map> latestResolvedPathCollection = ardcVocabService.getResolvedPathCollection(); - // call synchronous populating method, otherwise existing vocab caches will be emptied while new data hasn't been fully processed yet. - vocabService.populateVocabsData(); + if (!latestResolvedPathCollection.equals(storedResolvedPathCollection)) { + log.info("Detected changes in the resolved path collection, updating vocabularies..."); + vocabService.populateVocabsData(latestResolvedPathCollection); + refreshCaches(); - // clear existing caches - vocabService.clearParameterVocabCache(); - vocabService.clearPlatformVocabCache(); - vocabService.clearOrganisationVocabCache(); + // update the head if there are new versions + synchronized (this) { + storedResolvedPathCollection = latestResolvedPathCollection; + log.info("Updated storedResolvedPathCollection with the latest data."); + } + } else { + log.info("No changes detected in the resolved path collection. Skip updating caches"); + } + } catch (IOException e) { + log.error("Error refreshing vocabularies data: ", e); + } + } - // update the caches - vocabService.getParameterVocabs(); - vocabService.getPlatformVocabs(); - vocabService.getOrganisationVocabs(); + private void refreshCaches() { + try { + log.info("Clearing existing caches..."); + vocabService.clearParameterVocabCache(); + vocabService.clearPlatformVocabCache(); + vocabService.clearOrganisationVocabCache(); + + log.info("Updating vocabularies caches..."); + vocabService.getParameterVocabs(); + vocabService.getPlatformVocabs(); + vocabService.getOrganisationVocabs(); + } catch (IOException e) { + log.error("Error refreshing caches: ", e); + } } } diff --git a/indexer/src/test/java/au/org/aodn/esindexer/BaseTestClass.java b/indexer/src/test/java/au/org/aodn/esindexer/BaseTestClass.java index c1589b93..ce23ed7b 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/BaseTestClass.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/BaseTestClass.java @@ -1,9 +1,10 @@ package au.org.aodn.esindexer; +import au.org.aodn.ardcvocabs.model.PathName; +import au.org.aodn.ardcvocabs.model.VocabApiPaths; import au.org.aodn.esindexer.configuration.GeoNetworkSearchTestConfig; import au.org.aodn.esindexer.service.VocabServiceImpl; import au.org.aodn.esindexer.utils.CommonUtils; -import au.org.aodn.esindexer.utils.VocabsIndexUtils; import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.ElasticsearchException; import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders; @@ -73,9 +74,33 @@ protected void clearElasticIndex(String indexName) throws IOException { } } + protected Map> resolvedPathCollection = new HashMap<>(); + @PostConstruct public void init() throws IOException { - vocabService.populateVocabsData(); + + resolvedPathCollection.put(VocabApiPaths.PARAMETER_VOCAB.name(), Map.of( + PathName.vocabApi, "/aodn-discovery-parameter-vocabulary/version-1-6/concept.json", + PathName.categoryApi, "/aodn-parameter-category-vocabulary/version-2-1/concept.json", + PathName.categoryDetailsApi, "/aodn-parameter-category-vocabulary/version-2-1/resource.json?uri=%s", + PathName.vocabDetailsApi, "/aodn-discovery-parameter-vocabulary/version-1-6/resource.json?uri=%s" + )); + + resolvedPathCollection.put(VocabApiPaths.PLATFORM_VOCAB.name(), Map.of( + PathName.vocabApi, "/aodn-platform-vocabulary/version-6-1/concept.json", + PathName.categoryApi, "/aodn-platform-category-vocabulary/version-1-2/concept.json", + PathName.categoryDetailsApi, "/aodn-platform-category-vocabulary/version-1-2/resource.json?uri=%s", + PathName.vocabDetailsApi, "/aodn-platform-vocabulary/version-6-1/resource.json?uri=%s" + )); + + resolvedPathCollection.put(VocabApiPaths.ORGANISATION_VOCAB.name(), Map.of( + PathName.vocabApi, "/aodn-organisation-vocabulary/version-2-5/concept.json", + PathName.categoryApi, "/aodn-organisation-category-vocabulary/version-2-5/concept.json", + PathName.categoryDetailsApi, "/aodn-organisation-category-vocabulary/version-2-5/resource.json?uri=%s", + PathName.vocabDetailsApi, "/aodn-organisation-vocabulary/version-2-5/resource.json?uri=%s" + )); + + vocabService.populateVocabsData(resolvedPathCollection); } protected HttpEntity getRequestEntity(String body) { diff --git a/indexer/src/test/java/au/org/aodn/esindexer/service/VocabServiceIT.java b/indexer/src/test/java/au/org/aodn/esindexer/service/VocabServiceIT.java index 86d73999..4ad8454c 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/service/VocabServiceIT.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/service/VocabServiceIT.java @@ -101,7 +101,7 @@ void testExtractPlatformVocabLabelsFromThemes() throws IOException { @Test void testProcessParameterVocabs() throws IOException, JSONException { // read from ARDC - List parameterVocabsFromArdc = ardcVocabService.getVocabTreeFromArdcByType(VocabApiPaths.PARAMETER_VOCAB); + List parameterVocabsFromArdc = ardcVocabService.getVocabTreeFromArdcByType(resolvedPathCollection.get(VocabApiPaths.PARAMETER_VOCAB.name())); // read from Elastic search List parameterVocabsFromEs = vocabService.getParameterVocabs(); @@ -118,7 +118,7 @@ void testProcessParameterVocabs() throws IOException, JSONException { @Test void testProcessPlatformVocabs() throws IOException, JSONException { // read from ARDC - List platformVocabsFromArdc = ardcVocabService.getVocabTreeFromArdcByType(VocabApiPaths.PLATFORM_VOCAB); + List platformVocabsFromArdc = ardcVocabService.getVocabTreeFromArdcByType(resolvedPathCollection.get(VocabApiPaths.PLATFORM_VOCAB.name())); // read from Elastic search List platformVocabsFromEs = vocabService.getPlatformVocabs(); @@ -134,7 +134,7 @@ void testProcessPlatformVocabs() throws IOException, JSONException { @Test void testProcessOrganisationVocabs() throws IOException, JSONException { // read from ARDC - List organisationVocabsFromArdc = ardcVocabService.getVocabTreeFromArdcByType(VocabApiPaths.ORGANISATION_VOCAB); + List organisationVocabsFromArdc = ardcVocabService.getVocabTreeFromArdcByType(resolvedPathCollection.get(VocabApiPaths.ORGANISATION_VOCAB.name())); // read from Elastic search List organisationVocabsFromEs = vocabService.getOrganisationVocabs();