Merge pull request #124 from aodn/bugs/5768-change-reindex-threads

Fix delete issues
aodn · Aug 30, 2024 · c9c28e0 · c9c28e0
2 parents 42d78bb + cd49940
commit c9c28e0
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 23 deletions.
diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/GeoNetworkServiceImpl.java b/indexer/src/main/java/au/org/aodn/esindexer/service/GeoNetworkServiceImpl.java
@@ -370,6 +370,7 @@ public Long getAllMetadataCounts() throws IOException {
         TotalHits total = response.hits().total();
 
         if(total != null) {
+            logger.debug("Document count is {}", total.value());
             return total.value();
         }
         else {

diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java b/indexer/src/main/java/au/org/aodn/esindexer/service/IndexerServiceImpl.java
@@ -104,8 +104,8 @@ public Hit<ObjectNode> getDocumentByUUID(String uuid) throws IOException {
 
     @Override
     public boolean isGeoNetworkInstanceReinstalled(long portalIndexDocumentsCount) {
-        /**
-         * compare if GeoNetwork has 1 only metadata (the recently added one which triggered the indexer)
+        /*
+         * compare if GeoNetwork has only one metadata (the recently added one which triggered the indexer)
          * and the portal index has more than 0 documents (the most recent metadata yet indexed to portal index at this point)
          */
         return geoNetworkResourceService.isMetadataRecordsCountLessThan(2) && portalIndexDocumentsCount > 0;

diff --git a/indexer/src/test/java/au/org/aodn/esindexer/BaseTestClass.java b/indexer/src/test/java/au/org/aodn/esindexer/BaseTestClass.java
@@ -50,9 +50,6 @@ public class BaseTestClass {
     @Qualifier("portalElasticsearchClient")
     protected ElasticsearchClient client;
 
-    @Autowired
-    protected ElasticsearchContainer container;
-
     @Autowired
     protected DockerComposeContainer dockerComposeContainer;
 
@@ -109,11 +106,11 @@ protected String getLoginUrl() {
 
     }
 
-    protected String getIndexUrl() {
-        return String.format("http://%s:%s/geonetwork/srv/api/site/index?reset=false&asynchronous=false",
+    protected String getIndexUrl(boolean reset) {
+        return String.format("http://%s:%s/geonetwork/srv/api/site/index?reset=%s&asynchronous=false",
                 dockerComposeContainer.getServiceHost(GeoNetworkSearchTestConfig.GN_NAME, GeoNetworkSearchTestConfig.GN_PORT),
-                dockerComposeContainer.getServicePort(GeoNetworkSearchTestConfig.GN_NAME, GeoNetworkSearchTestConfig.GN_PORT));
-
+                dockerComposeContainer.getServicePort(GeoNetworkSearchTestConfig.GN_NAME, GeoNetworkSearchTestConfig.GN_PORT),
+                reset ? "true" : "false");
     }
 
     protected String isIndexUrl() {
@@ -202,17 +199,18 @@ public void deleteRecord(String... uuids) {
             // of the concurrency issue in elastic search, and can be resolved by retrying )
             persevere(() -> delete(uuid, requestEntity));
         }
+    }
 
-        // retry the request if the server is not ready yet (sometimes will return 403 and can be resolved by retrying )
-        persevere(() -> triggerIndexer(requestEntity));
+    protected boolean triggerIndexer(HttpEntity<String> requestEntity) {
+        return triggerIndexer(requestEntity, false);
     }
 
-    private boolean triggerIndexer(HttpEntity<String> requestEntity) {
+    protected boolean triggerIndexer(HttpEntity<String> requestEntity, boolean reset) {
 
         // Index the item so that query yield the right result before delete
         ResponseEntity<Void> trigger = testRestTemplate
                 .exchange(
-                        getIndexUrl(),
+                        getIndexUrl(reset),
                         HttpMethod.PUT,
                         requestEntity,
                         Void.class
@@ -241,7 +239,8 @@ private boolean delete(String uuid, HttpEntity<String> requestEntity) {
                             requestEntity,
                             String.class
                     );
-            if (response.getStatusCode().is2xxSuccessful()) {
+            if (response.getStatusCode().is2xxSuccessful() ||
+                    response.getStatusCode() == HttpStatus.NOT_FOUND) {
                 logger.info("Deleted GN doc {}", uuid);
                 return true;
             }
@@ -288,15 +287,8 @@ public String insertMetadataRecords(String uuid, String path) throws RestClientE
         assertEquals("Published OK", HttpStatus.NO_CONTENT, responseEntity.getStatusCode());
 
         // Index the item so that query yield the right result
-        responseEntity = testRestTemplate
-                .exchange(
-                        getIndexUrl(),
-                        HttpMethod.PUT,
-                        requestEntity,
-                        String.class
-                );
+        persevere(() -> triggerIndexer(getRequestEntity(null)));
 
-        assertEquals("Trigger index OK", HttpStatus.OK, responseEntity.getStatusCode());
         return content;
     }
 }
diff --git a/indexer/src/test/java/au/org/aodn/esindexer/service/IndexerServiceTests.java b/indexer/src/test/java/au/org/aodn/esindexer/service/IndexerServiceTests.java
@@ -2,12 +2,14 @@
 
 import au.org.aodn.esindexer.BaseTestClass;
 import au.org.aodn.esindexer.configuration.GeoNetworkSearchTestConfig;
+import co.elastic.clients.elasticsearch.ElasticsearchClient;
 import co.elastic.clients.elasticsearch.core.search.Hit;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.junit.jupiter.api.*;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.ActiveProfiles;
@@ -16,6 +18,10 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static au.org.aodn.esindexer.utils.CommonUtils.persevere;
 
 @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @ActiveProfiles("test")
@@ -26,6 +32,9 @@ public class IndexerServiceTests extends BaseTestClass {
     @Autowired
     protected GeoNetworkServiceImpl geoNetworkService;
 
+    @Qualifier("gn4ElasticsearchClient")
+    ElasticsearchClient gn4ElasticsearchClient;
+
     @Autowired
     protected IndexerService indexerService;
 
@@ -56,9 +65,10 @@ public void clear() throws IOException {
      * @throws IOException Not expected to throws
      */
     @Test
-    public void verifyGeoNetworkInstanceReinstalled() throws IOException {
+    public void verifyGeoNetworkInstanceReinstalled() throws Exception {
         String uuid = "9e5c3031-a026-48b3-a153-a70c2e2b78b9";
         try {
+            persevere(() -> triggerIndexer(getRequestEntity(null), true));
             insertMetadataRecords(uuid, "classpath:canned/sample1.xml");
             Assertions.assertTrue(indexerService.isGeoNetworkInstanceReinstalled(1), "New installed");
         }