From 07506b3877e3a91b13cee7c5acd0e378d65c8c81 Mon Sep 17 00:00:00 2001 From: Boubaker Khanfir Date: Wed, 19 Jun 2024 10:40:07 +0100 Subject: [PATCH] feat: Allow to purge automatically Analytics Indices - MEED-6999 - Meeds-io/meeds#2097 (#195) This change will allow to purge Analytics indices by specifying the maximum number of indices to keep. A parameter has been added `analytics.es.index.maxCount` to allow specifying it with default value to `500`. This will include a fix as well on index creation periodicity (`exo.es.analytics.index.per.days` param with default to 7 days) which was creating an index daily independently from parameter value. (cherry picked from commit 72a2192187442408a548aa93f1ca061ccd050385) --- .../analytics/es/AnalyticsESClient.java | 37 ++++++++++++++++++- .../resources/conf/portal/configuration.xml | 4 ++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/analytics-services/src/main/java/org/exoplatform/analytics/es/AnalyticsESClient.java b/analytics-services/src/main/java/org/exoplatform/analytics/es/AnalyticsESClient.java index 53f2e58ab..2f383d1f9 100644 --- a/analytics-services/src/main/java/org/exoplatform/analytics/es/AnalyticsESClient.java +++ b/analytics-services/src/main/java/org/exoplatform/analytics/es/AnalyticsESClient.java @@ -22,12 +22,14 @@ import java.time.format.DateTimeFormatter; import java.time.format.ResolverStyle; import java.util.*; +import java.util.concurrent.CompletableFuture; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.lang.StringUtils; import org.apache.http.conn.HttpClientConnectionManager; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.json.JSONException; +import org.json.JSONObject; import org.exoplatform.analytics.model.StatisticDataQueueEntry; import org.exoplatform.commons.search.es.client.*; @@ -60,6 +62,8 @@ public class AnalyticsESClient extends ElasticClient { private static final String ES_ANALYTICS_INDEX_PER_DAYS = "exo.es.analytics.index.per.days"; + private static final String ES_ANALYTICS_MAX_INDEX_COUNT = "analytics.es.index.maxCount"; + private static final long DAY_IN_MS = 86400000L; private static final String DAY_DATE_FORMAT = "yyyy-MM-dd"; @@ -77,6 +81,8 @@ public class AnalyticsESClient extends ElasticClient { private int indexPerDays; + private int maxIndexCount; + private String esIndexTemplateQuery; private String username; @@ -111,7 +117,10 @@ public AnalyticsESClient(ConfigurationManager configurationManager, } } if (initParams.containsKey(ES_ANALYTICS_INDEX_PER_DAYS)) { - this.indexPerDays = Integer.parseInt(initParams.getValueParam(ES_ANALYTICS_INDEX_PER_DAYS).getValue()); + this.indexPerDays = Math.max(Integer.parseInt(initParams.getValueParam(ES_ANALYTICS_INDEX_PER_DAYS).getValue()), 1); + } + if (initParams.containsKey(ES_ANALYTICS_MAX_INDEX_COUNT)) { + this.maxIndexCount = Integer.parseInt(initParams.getValueParam(ES_ANALYTICS_MAX_INDEX_COUNT).getValue()); } } if (StringUtils.isBlank(this.urlClient)) { @@ -131,6 +140,7 @@ public AnalyticsESClient(ConfigurationManager configurationManager, public void init() { checkIndexTemplateExistence(); LOG.info("Analytics client initialized and is ready to proceed analytics data"); + CompletableFuture.runAsync(this::sendRolloverRequest); } public boolean sendCreateIndexRequest(String index) { @@ -145,6 +155,7 @@ public boolean sendCreateIndexRequest(String index) { if (sendIsIndexExistsRequest(index)) { LOG.info("New analytics index {} created.", index); + CompletableFuture.runAsync(this::sendRolloverRequest); return true; } else { throw new IllegalStateException("Error creating index " + index + " on elasticsearch, response code = " @@ -319,7 +330,7 @@ public String getIndexSuffix(long timestamp) { if (indexSuffix != null) { return indexSuffix; } - indexSuffix = DAY_DATE_FORMATTER.format(Instant.ofEpochMilli(timestamp).atZone(ZoneOffset.UTC)); + indexSuffix = DAY_DATE_FORMATTER.format(Instant.ofEpochMilli(indexSuffixLong * DAY_IN_MS * indexPerDays).atZone(ZoneOffset.UTC)); indexSuffixPerDayIndice.put(indexSuffixLong, indexSuffix); return indexSuffix; } @@ -405,6 +416,28 @@ private void checkIndexTemplateExistence() { } } + private void sendRolloverRequest() { + LOG.info("Analytics Indices rollover process start"); + ElasticResponse response = sendHttpGetRequest(analyticsIndexingConnector.getIndexPrefix() + + "_*?allow_no_indices=true&ignore_unavailable=true"); + String indexListJsonString = response.getMessage(); + JSONObject jsonObject = new JSONObject(indexListJsonString); + List outdatedIndices = jsonObject.keySet() + .stream() + .sorted((s1, s2) -> s2.compareTo(s1)) + .skip(maxIndexCount) + .filter(Objects::nonNull) + .toList(); + while (!outdatedIndices.isEmpty()) { + List outdatedIndicesSubList = outdatedIndices.stream().limit(10).toList(); + String outdatedIndiceNames = StringUtils.join(outdatedIndicesSubList, ","); + LOG.info("Deleting {} outdated analytics Indices: [{}]", outdatedIndicesSubList.size(), outdatedIndiceNames); + sendHttpDeleteRequest(outdatedIndiceNames); + outdatedIndices = outdatedIndices.stream().skip(10).toList(); + } + LOG.info("Analytics Indices rollover process finished successfully."); + } + private final String getIndex(long timestamp) { if (indexPerDays > 0) { String indexSuffix = getIndexSuffix(timestamp); diff --git a/analytics-services/src/main/resources/conf/portal/configuration.xml b/analytics-services/src/main/resources/conf/portal/configuration.xml index c2efd1e57..f3c7a6cdb 100644 --- a/analytics-services/src/main/resources/conf/portal/configuration.xml +++ b/analytics-services/src/main/resources/conf/portal/configuration.xml @@ -41,6 +41,10 @@ exo.es.analytics.index.per.days ${exo.es.analytics.index.per.days:7} + + analytics.es.index.maxCount + ${analytics.es.index.maxCount:500} + index.template.file.path ${exo.es.analytics.index.template.path:jar:/analytics-es-template.json}