Skip to content

Commit

Permalink
feat: Allow to purge automatically Analytics Indices - MEED-6999 - Me…
Browse files Browse the repository at this point in the history 
…eds-io/meeds#2097 (#195)

This change will allow to purge Analytics indices by specifying the
maximum number of indices to keep. A parameter has been added
`analytics.es.index.maxCount` to allow specifying it with default value
to `500`.
This will include a fix as well on index creation periodicity
(`exo.es.analytics.index.per.days` param with default to 7 days) which
was creating an index daily independently from parameter value.
  • Loading branch information
@boubaker
boubaker authored Jun 19, 2024
1 parent e606fe8 commit 72a2192
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 2 deletions.
37 changes: 35 additions & 2 deletions analytics-services/src/main/java/org/exoplatform/analytics/es/AnalyticsESClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,14 @@
import java.time.format.DateTimeFormatter;
import java.time.format.ResolverStyle;
import java.util.*;
import java.util.concurrent.CompletableFuture;

import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.lang.StringUtils;
import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.json.JSONException;
import org.json.JSONObject;

import org.exoplatform.analytics.model.StatisticDataQueueEntry;
import org.exoplatform.commons.search.es.client.*;
Expand Down Expand Up @@ -60,6 +62,8 @@ public class AnalyticsESClient extends ElasticClient {

private static final String ES_ANALYTICS_INDEX_PER_DAYS = "exo.es.analytics.index.per.days";

private static final String ES_ANALYTICS_MAX_INDEX_COUNT = "analytics.es.index.maxCount";

private static final long DAY_IN_MS = 86400000L;

private static final String DAY_DATE_FORMAT = "yyyy-MM-dd";
Expand All @@ -77,6 +81,8 @@ public class AnalyticsESClient extends ElasticClient {

private int indexPerDays;

private int maxIndexCount;

private String esIndexTemplateQuery;

private String username;
Expand Down Expand Up @@ -111,7 +117,10 @@ public AnalyticsESClient(ConfigurationManager configurationManager,
}
}
if (initParams.containsKey(ES_ANALYTICS_INDEX_PER_DAYS)) {
this.indexPerDays = Integer.parseInt(initParams.getValueParam(ES_ANALYTICS_INDEX_PER_DAYS).getValue());
this.indexPerDays = Math.max(Integer.parseInt(initParams.getValueParam(ES_ANALYTICS_INDEX_PER_DAYS).getValue()), 1);
}
if (initParams.containsKey(ES_ANALYTICS_MAX_INDEX_COUNT)) {
this.maxIndexCount = Integer.parseInt(initParams.getValueParam(ES_ANALYTICS_MAX_INDEX_COUNT).getValue());
}
}
if (StringUtils.isBlank(this.urlClient)) {
Expand All @@ -131,6 +140,7 @@ public AnalyticsESClient(ConfigurationManager configurationManager,
public void init() {
checkIndexTemplateExistence();
LOG.info("Analytics client initialized and is ready to proceed analytics data");
CompletableFuture.runAsync(this::sendRolloverRequest);
}

public boolean sendCreateIndexRequest(String index) {
Expand All @@ -145,6 +155,7 @@ public boolean sendCreateIndexRequest(String index) {

if (sendIsIndexExistsRequest(index)) {
LOG.info("New analytics index {} created.", index);
CompletableFuture.runAsync(this::sendRolloverRequest);
return true;
} else {
throw new IllegalStateException("Error creating index " + index + " on elasticsearch, response code = "
Expand Down Expand Up @@ -319,7 +330,7 @@ public String getIndexSuffix(long timestamp) {
if (indexSuffix != null) {
return indexSuffix;
}
indexSuffix = DAY_DATE_FORMATTER.format(Instant.ofEpochMilli(timestamp).atZone(ZoneOffset.UTC));
indexSuffix = DAY_DATE_FORMATTER.format(Instant.ofEpochMilli(indexSuffixLong * DAY_IN_MS * indexPerDays).atZone(ZoneOffset.UTC));
indexSuffixPerDayIndice.put(indexSuffixLong, indexSuffix);
return indexSuffix;
}
Expand Down Expand Up @@ -405,6 +416,28 @@ private void checkIndexTemplateExistence() {
}
}

private void sendRolloverRequest() {
LOG.info("Analytics Indices rollover process start");
ElasticResponse response = sendHttpGetRequest(analyticsIndexingConnector.getIndexPrefix() +
"_*?allow_no_indices=true&ignore_unavailable=true");
String indexListJsonString = response.getMessage();
JSONObject jsonObject = new JSONObject(indexListJsonString);
List<String> outdatedIndices = jsonObject.keySet()
.stream()
.sorted((s1, s2) -> s2.compareTo(s1))
.skip(maxIndexCount)
.filter(Objects::nonNull)
.toList();
while (!outdatedIndices.isEmpty()) {
List<String> outdatedIndicesSubList = outdatedIndices.stream().limit(10).toList();
String outdatedIndiceNames = StringUtils.join(outdatedIndicesSubList, ",");
LOG.info("Deleting {} outdated analytics Indices: [{}]", outdatedIndicesSubList.size(), outdatedIndiceNames);
sendHttpDeleteRequest(outdatedIndiceNames);
outdatedIndices = outdatedIndices.stream().skip(10).toList();
}
LOG.info("Analytics Indices rollover process finished successfully.");
}

private final String getIndex(long timestamp) {
if (indexPerDays > 0) {
String indexSuffix = getIndexSuffix(timestamp);
Expand Down
4 changes: 4 additions & 0 deletions analytics-services/src/main/resources/conf/portal/configuration.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,10 @@
<name>exo.es.analytics.index.per.days</name>
<value>${exo.es.analytics.index.per.days:7}</value>
</value-param>
<value-param>
<name>analytics.es.index.maxCount</name>
<value>${analytics.es.index.maxCount:500}</value>
</value-param>
<value-param>
<name>index.template.file.path</name>
<value>${exo.es.analytics.index.template.path:jar:/analytics-es-template.json}</value>
Expand Down

0 comments on commit 72a2192

Please sign in to comment.