Skip to content

Commit

Permalink
feat: Cleanup Addon and Allow to purge automatically Analytics Indice…
Browse files Browse the repository at this point in the history 
…s - MEED-6999 - Meeds-io/meeds#2097 (#189)

This change will allow to purge Analytics indices by specifying the
maximum number of indices to keep. A parameter has been added
`analytics.es.index.maxCount` to allow specifying it with default value
to `500`. This will include a fix as well on index creation periodicity
(`analytics.es.index.writePeriod` param with default to 7 days) which
was creating an index daily independently from parameter value.
In addition, this change will rename artifact (`io.meed` instead of
`org.exoplatform`) and migrate it to use Spring instead of Kernel for
IoC & DI.
  • Loading branch information
@boubaker @exo-swf
boubaker authored and exo-swf committed Jun 13, 2024
1 parent 4a5b3f3 commit a2bfbb0
Show file tree
Hide file tree
Showing 159 changed files with 2,999 additions and 3,464 deletions.
413 changes: 10 additions & 403 deletions README.md
View file

Large diffs are not rendered by default.

11 changes: 7 additions & 4 deletions analytics-api/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -1,29 +1,32 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
This file is part of the Meeds project (https://meeds.io/).
Copyright (C) 2022 Meeds Association
[email protected]
Copyright (C) 2020 - 2024 Meeds Association [email protected]
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 3 of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with this program; if not, write to the Free Software Foundation,
Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.exoplatform.addons.analytics</groupId>
<groupId>io.meeds.analytics</groupId>
<artifactId>analytics-parent</artifactId>
<version>7.0.x-exo-SNAPSHOT</version>
</parent>
<artifactId>analytics-api</artifactId>
<name>eXo Analytics - API</name>
<name>Meeds:: Analytics - API</name>
<properties>
<exo.test.coverage.ratio>0</exo.test.coverage.ratio>
</properties>
Expand Down
25 changes: 15 additions & 10 deletions ...alytics/api/service/AnalyticsService.java → ...alytics/api/service/AnalyticsService.java
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,34 @@
/**
* This file is part of the Meeds project (https://meeds.io/).
* Copyright (C) 2022 Meeds Association
* [email protected]
*
* Copyright (C) 2020 - 2024 Meeds Association [email protected]
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.exoplatform.analytics.api.service;
package io.meeds.analytics.api.service;

import java.util.List;
import java.util.Set;

import org.exoplatform.analytics.model.*;
import org.exoplatform.analytics.model.chart.*;
import org.exoplatform.analytics.model.filter.*;
import org.springframework.stereotype.Service;

import io.meeds.analytics.model.*;
import io.meeds.analytics.model.chart.*;
import io.meeds.analytics.model.filter.*;

@Service
public interface AnalyticsService {

/**
Expand Down Expand Up @@ -130,10 +136,9 @@ PercentageChartValue computePercentageChartData(AnalyticsFilter filter,
StatisticWatcher getUIWatcher(String name);

/**
* Add watcher plugin
*
* @param uiWatcherPlugin Kernel component plugin to add
* @param uiWatcher {@link StatisticWatcher} containing DOM selector of
* elements to watch
*/
void addUIWatcherPlugin(StatisticUIWatcherPlugin uiWatcherPlugin);
void addUIWatcher(StatisticWatcher uiWatcher);

}
116 changes: 41 additions & 75 deletions ...ervice/StatisticDataProcessorService.java → ...ervice/StatisticDataProcessorService.java
View file
Original file line number Diff line number Diff line change
@@ -1,89 +1,58 @@
/**
* This file is part of the Meeds project (https://meeds.io/).
* Copyright (C) 2022 Meeds Association
* [email protected]
*
* Copyright (C) 2020 - 2024 Meeds Association [email protected]
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.exoplatform.analytics.api.service;
package io.meeds.analytics.api.service;

import static org.exoplatform.analytics.utils.AnalyticsUtils.MAX_BULK_DOCUMENTS;
import static io.meeds.analytics.utils.AnalyticsUtils.MAX_BULK_DOCUMENTS;

import java.util.*;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.stream.Collectors;

import org.apache.commons.lang3.StringUtils;
import org.picocontainer.Startable;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import org.exoplatform.analytics.api.processor.StatisticDataProcessorPlugin;
import org.exoplatform.analytics.model.StatisticDataQueueEntry;
import org.exoplatform.services.log.ExoLogger;
import org.exoplatform.services.log.Log;

public class StatisticDataProcessorService implements Startable {
import io.meeds.analytics.model.StatisticDataQueueEntry;
import io.meeds.analytics.plugin.StatisticDataProcessorPlugin;

private static final Log LOG =
ExoLogger.getLogger(StatisticDataProcessorService.class);
@Service
public class StatisticDataProcessorService {

private static final short MAX_PROCESS_ATTEMPTS_COUNT = 5;
private static final Log LOG =
ExoLogger.getLogger(StatisticDataProcessorService.class);

private ArrayList<StatisticDataProcessorPlugin> dataProcessorPlugins = new ArrayList<>();
private static final short MAX_PROCESS_ATTEMPTS_COUNT = 5;

@Override
public void start() {
for (StatisticDataProcessorPlugin statisticDataProcessorPlugin : dataProcessorPlugins) {
try {
statisticDataProcessorPlugin.init();
} catch (Exception e) {
LOG.error("Error initializing processor with id {}", statisticDataProcessorPlugin.getId(), e);
}
}
}

@Override
public void stop() {
// Nothing to stop
}
@Autowired(required = false)
private Collection<StatisticDataProcessorPlugin> dataProcessorPlugins;

public void addProcessor(StatisticDataProcessorPlugin dataProcessorPlugin) {
this.dataProcessorPlugins.add(dataProcessorPlugin);
}

public boolean pauseProcessor(String id) {
StatisticDataProcessorPlugin processor = getProcessor(id);
if (processor == null || processor.isPaused()) {
return false;
} else {
processor.setPaused(true);
return true;
}
}

public boolean unpauseProcessor(String id) {
StatisticDataProcessorPlugin processor = getProcessor(id);
if (processor == null || !processor.isPaused()) {
return false;
} else {
processor.setPaused(false);
return true;
}
}

public boolean isProcessorInitialized(String id) {
StatisticDataProcessorPlugin processor = getProcessor(id);
return processor != null && processor.isInitialized();
}

public List<StatisticDataProcessorPlugin> getProcessors() {
public Collection<StatisticDataProcessorPlugin> getProcessors() {
return this.dataProcessorPlugins;
}

Expand All @@ -94,16 +63,17 @@ public StatisticDataProcessorPlugin getProcessor(String id) {
.orElse(null);
}

public void process(List<? extends StatisticDataQueueEntry> queueEntries) {
public void process(List<? extends StatisticDataQueueEntry> queueEntries) { // NOSONAR
if (queueEntries == null || queueEntries.isEmpty()) {
return;
}

List<? extends StatisticDataQueueEntry> queueEntriesToProcess = queueEntries.stream()
.filter(queueEntry -> !queueEntry.isProcessed()
&& (!queueEntry.isError()
|| queueEntry.getAttemptCount() < MAX_PROCESS_ATTEMPTS_COUNT))
.collect(Collectors.toList());
&& (!queueEntry.isError()
|| queueEntry.getAttemptCount()
< MAX_PROCESS_ATTEMPTS_COUNT))
.toList();
if (queueEntriesToProcess.isEmpty()) {
return;
} else if (queueEntriesToProcess.size() > MAX_BULK_DOCUMENTS) {
Expand All @@ -115,24 +85,20 @@ public void process(List<? extends StatisticDataQueueEntry> queueEntries) {
List<? extends StatisticDataQueueEntry> subList = queueEntriesToProcess.stream()
.skip(index)
.limit(MAX_BULK_DOCUMENTS)
.collect(Collectors.toList());
.toList();
process(subList);
index += subList.size();
} while (index < queueEntriesToProcess.size());
return;
}

for (StatisticDataProcessorPlugin statisticDataProcessorPlugin : dataProcessorPlugins) {// NOSONAR need more than one continue statement
if (statisticDataProcessorPlugin.isPaused()) {
LOG.debug("Statistic Data Processor with id '{}' is paused, ignore processing queue items",
statisticDataProcessorPlugin.getId());
continue;
}
if (!statisticDataProcessorPlugin.isInitialized()) {
LOG.info("Statistic Data Processor with id '{}' is not initialized yet, ignore processing queue items",
statisticDataProcessorPlugin.getId());
continue;
}
for (StatisticDataProcessorPlugin statisticDataProcessorPlugin : dataProcessorPlugins) {// NOSONAR
// need
// more
// than
// one
// continue
// statement
String processorId = statisticDataProcessorPlugin.getId();
List<StatisticDataQueueEntry> processorQueueEntries = queueEntriesToProcess.stream()
.filter(queueEntry -> !isProcessorRun(queueEntry,
Expand Down Expand Up @@ -171,7 +137,7 @@ public void process(List<? extends StatisticDataQueueEntry> queueEntries) {

private void markProcessorAsSuccess(StatisticDataQueueEntry dataQueueEntry,
String processorId,
List<StatisticDataProcessorPlugin> processors) {
Collection<StatisticDataProcessorPlugin> processors) {
if (dataQueueEntry.getProcessingStatus() == null) {
dataQueueEntry.setProcessingStatus(new HashMap<>());
}
Expand All @@ -194,7 +160,7 @@ private void markProcessorAsError(StatisticDataQueueEntry dataQueueEntry, String
}
}

private boolean isProcessedForAll(StatisticDataQueueEntry dataQueueEntry, List<StatisticDataProcessorPlugin> processors) {
private boolean isProcessedForAll(StatisticDataQueueEntry dataQueueEntry, Collection<StatisticDataProcessorPlugin> processors) {
if (processors == null || processors.isEmpty()) {
return true;
}
Expand All @@ -208,9 +174,9 @@ private boolean isProcessedForAll(StatisticDataQueueEntry dataQueueEntry, List<S

private boolean isProcessorRun(StatisticDataQueueEntry dataQueueEntry, String processorId) {
Boolean processorStatus =
dataQueueEntry.getProcessingStatus() == null ? null
: dataQueueEntry.getProcessingStatus().get(processorId);
return processorStatus != null && processorStatus;
dataQueueEntry.getProcessingStatus() == null ? null :
dataQueueEntry.getProcessingStatus().get(processorId);
return processorStatus != null && processorStatus.booleanValue();
}

}
14 changes: 10 additions & 4 deletions ...pi/service/StatisticDataQueueService.java → ...pi/service/StatisticDataQueueService.java
View file
Original file line number Diff line number Diff line change
@@ -1,26 +1,32 @@
/**
* This file is part of the Meeds project (https://meeds.io/).
* Copyright (C) 2022 Meeds Association
* [email protected]
*
* Copyright (C) 2020 - 2024 Meeds Association [email protected]
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.exoplatform.analytics.api.service;
package io.meeds.analytics.api.service;

import org.exoplatform.analytics.model.StatisticData;
import org.springframework.stereotype.Service;

import io.meeds.analytics.model.StatisticData;

/**
* A service to manage statistic data ingestion processing
*/
@Service
public interface StatisticDataQueueService {

/**
Expand Down
69 changes: 69 additions & 0 deletions ...src/main/java/io/meeds/analytics/api/service/injection/StatisticWatcherImportService.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
/**
* This file is part of the Meeds project (https://meeds.io/).
*
* Copyright (C) 2020 - 2024 Meeds Association [email protected]
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package io.meeds.analytics.api.service.injection;

import java.io.InputStream;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;

import org.apache.commons.io.IOUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import io.meeds.analytics.api.service.AnalyticsService;
import io.meeds.analytics.model.StatisticWatcher;
import io.meeds.analytics.model.StatisticWatchersDescriptor;
import io.meeds.social.util.JsonUtils;

import jakarta.annotation.PostConstruct;
import lombok.SneakyThrows;

@Component
public class StatisticWatcherImportService {

@Autowired
private AnalyticsService analyticsService;

@PostConstruct
public void init() {
CompletableFuture.runAsync(this::importUIWatchers);
}

@SneakyThrows
public void importUIWatchers() {
Collections.list(getClass().getClassLoader().getResources("analytics-ui-watchers.json"))
.stream()
.map(this::parseDescriptors)
.flatMap(List::stream)
.forEach(analyticsService::addUIWatcher);
}

@SneakyThrows
protected List<StatisticWatcher> parseDescriptors(URL url) {
try (InputStream inputStream = url.openStream()) {
String content = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
StatisticWatchersDescriptor list = JsonUtils.fromJsonString(content, StatisticWatchersDescriptor.class);
return list.getDescriptors();
}
}

}
Loading

0 comments on commit a2bfbb0

Please sign in to comment.