feat: scheduled task for importing video learning events (#1921)

pom.xml

@@ -9,7 +9,7 @@
     <properties>
         <java.version>17</java.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <model.version>2.0.73</model.version>
+        <model.version>2.0.74</model.version>
         <hibernate.version>5.6.15.Final</hibernate.version>
         <jetty.version>10.0.22</jetty.version>
         <spring.version>5.3.18</spring.version>

src/main/java/ai/elimu/dao/VideoLearningEventDao.java

@@ -0,0 +1,10 @@
+package ai.elimu.dao;
+
+import ai.elimu.model.analytics.VideoLearningEvent;
+import java.util.Calendar;
+import org.springframework.dao.DataAccessException;
+
+public interface VideoLearningEventDao extends GenericDao<VideoLearningEvent> {
+
+    VideoLearningEvent read(Calendar timestamp, String androidId, String packageName, String videoTitle) throws DataAccessException;
+}

src/main/java/ai/elimu/dao/jpa/VideoLearningEventDaoJpa.java

@@ -0,0 +1,31 @@
+package ai.elimu.dao.jpa;
+
+import ai.elimu.dao.VideoLearningEventDao;
+import ai.elimu.model.analytics.VideoLearningEvent;
+import java.util.Calendar;
+import javax.persistence.NoResultException;
+import org.springframework.dao.DataAccessException;
+
+public class VideoLearningEventDaoJpa extends GenericDaoJpa<VideoLearningEvent> implements VideoLearningEventDao {
+
+    @Override
+    public VideoLearningEvent read(Calendar timestamp, String androidId, String packageName, String videoTitle) throws DataAccessException {
+        try {
+            return (VideoLearningEvent) em.createQuery(
+                "SELECT event " +
+                "FROM VideoLearningEvent event " +
+                "WHERE event.timestamp = :timestamp " +
+                "AND event.androidId = :androidId " + 
+                "AND event.packageName = :packageName " + 
+                "AND event.videoTitle = :videoTitle")
+                .setParameter("timestamp", timestamp)
+                .setParameter("androidId", androidId)
+                .setParameter("packageName", packageName)
+                .setParameter("videoTitle", videoTitle)
+                .getSingleResult();
+        } catch (NoResultException e) {
+            logger.info("VideoLearningEvent (" + timestamp.getTimeInMillis() + ", " + androidId + ", " + packageName + ", \"" + videoTitle + "\") was not found");
+            return null;
+        }
+    }
+}

src/main/java/ai/elimu/model/analytics/LearningEvent.java

@@ -65,12 +65,7 @@ public void setTimestamp(Calendar timestamp) {
     }
 
     public String getAndroidId() {
-        if (!androidId.contains("***")) {
-            // Hide parts of the Android ID, e.g. "7161a85a0e4751cd" --> "7161***51cd"
-            return androidId.substring(0, 4) + "***" + androidId.substring(12);
-        } else {
-            return androidId;
-        }
+        return androidId;
     }
 
     public void setAndroidId(String androidId) {

src/main/java/ai/elimu/tasks/analytics/VideoLearningEventImportScheduler.java

@@ -0,0 +1,102 @@
+package ai.elimu.tasks.analytics;
+
+import java.io.File;
+import java.util.List;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+
+import ai.elimu.dao.VideoLearningEventDao;
+import ai.elimu.model.analytics.VideoLearningEvent;
+import ai.elimu.model.v2.enums.Language;
+import ai.elimu.rest.v2.analytics.VideoLearningEventsRestController;
+import ai.elimu.util.ConfigHelper;
+import ai.elimu.util.csv.CsvAnalyticsExtractionHelper;
+
+/**
+ * Extracts learning events from CSV files previously received by the 
+ * {@link VideoLearningEventsRestController}, and imports them into the database.
+ * <p />
+ * 
+ * Expected folder structure:
+ * <pre>
+ * ├── lang-ENG
+ * │   ├── analytics
+ * │   │   ├── android-id-e387e38700000001
+ * │   │   │   └── version-code-3001018
+ * │   │   │       └── video-learning-events
+ * │   │   │           ├── e387e38700000001_3001018_video-learning-events_2024-10-09.csv
+ * │   │   │           ├── e387e38700000001_3001018_video-learning-events_2024-10-10.csv
+ * │   │   │           ├── e387e38700000001_3001018_video-learning-events_2024-10-11.csv
+ * │   │   │           ├── e387e38700000001_3001018_video-learning-events_2024-10-14.csv
+ * │   │   │           ├── e387e38700000001_3001018_video-learning-events_2024-10-18.csv
+ * │   │   │           └── e387e38700000001_3001018_video-learning-events_2024-10-20.csv
+ * │   │   ├── android-id-e387e38700000002
+ * │   │   │   └── version-code-3001018
+ * │   │   │       └── video-learning-events
+ * │   │   │           ├── e387e38700000002_3001018_video-learning-events_2024-10-09.csv
+ * │   │   │           ├── e387e38700000002_3001018_video-learning-events_2024-10-10.csv
+ * │   │   │           ├── e387e38700000002_3001018_video-learning-events_2024-10-11.csv
+ * </pre>
+ */
+@Service
+public class VideoLearningEventImportScheduler {
+
+    private Logger logger = LogManager.getLogger();
+
+    @Autowired
+    private VideoLearningEventDao videoLearningEventDao;
+
+    @Scheduled(cron="00 30 * * * *") // Half past every hour
+    public synchronized void execute() {
+        logger.info("execute");
+
+        // Lookup CSV files stored on the filesystem
+        File elimuAiDir = new File(System.getProperty("user.home"), ".elimu-ai");
+        File languageDir = new File(elimuAiDir, "lang-" + Language.valueOf(ConfigHelper.getProperty("content.language")));
+        File analyticsDir = new File(languageDir, "analytics");
+        logger.info("analyticsDir: " + analyticsDir);
+        analyticsDir.mkdirs();
+        for (File analyticsDirFile : analyticsDir.listFiles()) {
+            if (analyticsDirFile.getName().startsWith("android-id-")) {
+                File androidIdDir = new File(analyticsDir, analyticsDirFile.getName());
+                for (File androidIdDirFile : androidIdDir.listFiles()) {
+                    if (androidIdDirFile.getName().startsWith("version-code-")) {
+                        File versionCodeDir = new File(androidIdDir, androidIdDirFile.getName());
+                        for (File versionCodeDirFile : versionCodeDir.listFiles()) {
+                            if (versionCodeDirFile.getName().equals("video-learning-events")) {
+                                File videoLearningEventsDir = new File(versionCodeDir, versionCodeDirFile.getName());
+                                for (File csvFile : videoLearningEventsDir.listFiles()) {
+                                    logger.info("csvFile: " + csvFile);
+
+                                    // Convert from CSV to Java
+                                    List<VideoLearningEvent> events = CsvAnalyticsExtractionHelper.extractVideoLearningEvents(csvFile);
+                                    logger.info("events.size(): " + events.size());
+
+                                    // Store in database
+                                    for (VideoLearningEvent event : events) {
+                                        // Check if the event has already been stored in the database
+                                        VideoLearningEvent existingVideoLearningEvent = videoLearningEventDao.read(event.getTimestamp(), event.getAndroidId(), event.getPackageName(), event.getVideoTitle());
+                                        if (existingVideoLearningEvent != null) {
+                                            logger.warn("The event has already been stored in the database. Skipping data import.");
+                                            continue;
+                                        }
+
+                                        // Store the event in the database
+                                        videoLearningEventDao.create(event);
+                                        logger.info("Stored event in database with ID " + event.getId());
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        logger.info("execute complete");
+    }
+}

src/main/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelper.java

@@ -4,7 +4,10 @@
 import ai.elimu.dao.StoryBookDao;
 import ai.elimu.model.admin.Application;
 import ai.elimu.model.analytics.StoryBookLearningEvent;
+import ai.elimu.model.analytics.VideoLearningEvent;
+import ai.elimu.model.analytics.WordLearningEvent;
 import ai.elimu.model.content.StoryBook;
+import ai.elimu.model.content.Word;
 import ai.elimu.model.v2.enums.analytics.LearningEventType;
 import ai.elimu.rest.v2.analytics.StoryBookLearningEventsRestController;
 import ai.elimu.web.analytics.StoryBookLearningEventCsvExportController;
@@ -15,13 +18,15 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Calendar;
 import java.util.List;
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVParser;
 import org.apache.commons.csv.CSVRecord;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.springframework.http.HttpStatus;
 
 public class CsvAnalyticsExtractionHelper {
 
@@ -99,4 +104,68 @@ public static List<StoryBookLearningEvent> getStoryBookLearningEventsFromCsvBack
 
         return storyBookLearningEvents;
     }
+
+    public static List<VideoLearningEvent> extractVideoLearningEvents(File csvFile) {
+        logger.info("extractVideoLearningEvents");
+
+        List<VideoLearningEvent> videoLearningEvents = new ArrayList<>();
+
+        // Iterate each row in the CSV file
+        Path csvFilePath = Paths.get(csvFile.toURI());
+        logger.info("csvFilePath: " + csvFilePath);
+        try {
+            Reader reader = Files.newBufferedReader(csvFilePath);
+            CSVFormat csvFormat = CSVFormat.DEFAULT
+                    .withHeader(
+                            "id", // The Android database ID
+                            "timestamp",
+                            "android_id",
+                            "package_name",
+                            "video_id",
+                            "video_title",
+                            "learning_event_type",
+                            "additional_data"
+                    )
+                    .withSkipHeaderRecord();
+            logger.info("header: " + Arrays.toString(csvFormat.getHeader()));
+            CSVParser csvParser = new CSVParser(reader, csvFormat);
+            for (CSVRecord csvRecord : csvParser) {
+                logger.info("csvRecord: " + csvRecord);
+
+                // Convert from CSV to Java
+
+                VideoLearningEvent videoLearningEvent = new VideoLearningEvent();
+
+                long timestampInMillis = Long.valueOf(csvRecord.get("timestamp"));
+                Calendar timestamp = Calendar.getInstance();
+                timestamp.setTimeInMillis(timestampInMillis);
+                videoLearningEvent.setTimestamp(timestamp);
+
+                String androidId = csvRecord.get("android_id");
+                videoLearningEvent.setAndroidId(androidId);
+
+                String packageName = csvRecord.get("package_name");
+                videoLearningEvent.setPackageName(packageName);
+
+                Long videoId = Long.valueOf(csvRecord.get("video_id"));
+                videoLearningEvent.setVideoId(videoId);
+
+                String videoTitle = csvRecord.get("video_title");
+                videoLearningEvent.setVideoTitle(videoTitle);
+
+                LearningEventType learningEventType = LearningEventType.valueOf(csvRecord.get("learning_event_type"));
+                videoLearningEvent.setLearningEventType(learningEventType);
+
+                String additionalData = csvRecord.get("additional_data");
+                videoLearningEvent.setAdditionalData(additionalData);
+
+                videoLearningEvents.add(videoLearningEvent);
+            }
+            csvParser.close();
+        } catch (IOException ex) {
+            logger.error(ex);
+        }
+
+        return videoLearningEvents;
+    }
 }

src/main/webapp/WEB-INF/spring/applicationContext-jpa.xml

@@ -74,6 +74,7 @@
     <bean id="storyBookParagraphDao" class="ai.elimu.dao.jpa.StoryBookParagraphDaoJpa" />
     <bean id="syllableDao" class="ai.elimu.dao.jpa.SyllableDaoJpa" />
     <bean id="videoDao" class="ai.elimu.dao.jpa.VideoDaoJpa" />
+    <bean id="videoLearningEventDao" class="ai.elimu.dao.jpa.VideoLearningEventDaoJpa" />
     <bean id="wordDao" class="ai.elimu.dao.jpa.WordDaoJpa" />
     <bean id="wordContributionEventDao" class="ai.elimu.dao.jpa.WordContributionEventDaoJpa" />
     <bean id="wordLearningEventDao" class="ai.elimu.dao.jpa.WordLearningEventDaoJpa" />

src/test/java/ai/elimu/dao/VideoLearningEventDaoJpaTest.java

@@ -0,0 +1,53 @@
+package ai.elimu.dao;
+
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.util.Calendar;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.test.context.junit.jupiter.SpringJUnitConfig;
+
+import ai.elimu.model.analytics.VideoLearningEvent;
+
+@SpringJUnitConfig(locations = {
+    "file:src/main/webapp/WEB-INF/spring/applicationContext.xml",
+    "file:src/main/webapp/WEB-INF/spring/applicationContext-jpa.xml"
+})
+public class VideoLearningEventDaoJpaTest {
+
+    @Autowired
+    private VideoLearningEventDao videoLearningEventDao;
+
+    @Test
+    public void testRead() {
+        Calendar timestamp = Calendar.getInstance();
+        String androidId = "e387e38700000001";
+        String packageName = "ai.elimu.filamu";
+        String videoTitle = "akili and me - the rectangle song";
+
+        VideoLearningEvent existingEvent = videoLearningEventDao.read(
+            timestamp,
+            androidId,
+            packageName,
+            videoTitle
+        );
+        assertNull(existingEvent);
+
+        VideoLearningEvent event = new VideoLearningEvent();
+        event.setTimestamp(timestamp);
+        event.setAndroidId(androidId);
+        event.setPackageName(packageName);
+        event.setVideoTitle(videoTitle);
+        videoLearningEventDao.create(event);
+
+        existingEvent = videoLearningEventDao.read(
+            timestamp,
+            androidId,
+            packageName,
+            videoTitle
+        );
+        assertNotNull(existingEvent);
+    }
+}

src/test/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelperTest.java

@@ -0,0 +1,55 @@
+package ai.elimu.util.csv;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.jupiter.api.Test;
+import org.springframework.core.io.ClassRelativeResourceLoader;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.ResourceLoader;
+
+import ai.elimu.model.analytics.VideoLearningEvent;
+import ai.elimu.model.v2.enums.analytics.LearningEventType;
+
+public class CsvAnalyticsExtractionHelperTest {
+
+    private Logger logger = LogManager.getLogger();
+
+    @Test
+    public void testExtractVideoLearningEvents() throws IOException {
+        ResourceLoader resourceLoader = new ClassRelativeResourceLoader(CsvAnalyticsExtractionHelper.class);
+        Resource resource = resourceLoader.getResource("e387e38700000001_3001018_video-learning-events_2024-10-09.csv");
+        File csvFile = resource.getFile();
+        logger.debug("csvFile: " + csvFile);
+
+        List<VideoLearningEvent> videoLearningEvents = CsvAnalyticsExtractionHelper.extractVideoLearningEvents(csvFile);
+        assertEquals(6, videoLearningEvents.size());
+
+        // Test the 1st row of data
+        VideoLearningEvent event1st = videoLearningEvents.get(0);
+        assertEquals(1728486312687L, event1st.getTimestamp().getTimeInMillis());
+        assertEquals("e387e38700000001", event1st.getAndroidId());
+        assertEquals("ai.elimu.analytics", event1st.getPackageName());
+        assertEquals(13, event1st.getVideoId());
+        assertEquals("akili and me - the rectangle song", event1st.getVideoTitle());
+        assertEquals(LearningEventType.VIDEO_OPENED, event1st.getLearningEventType());
+        assertEquals("", event1st.getAdditionalData());
+
+        // Test the 2nd row of data
+        VideoLearningEvent event2nd = videoLearningEvents.get(1);
+        assertEquals(1728486319885L, event2nd.getTimestamp().getTimeInMillis());
+        assertEquals("e387e38700000001", event2nd.getAndroidId());
+        assertEquals("ai.elimu.analytics", event2nd.getPackageName());
+        assertEquals(13, event2nd.getVideoId());
+        assertEquals("akili and me - the rectangle song", event2nd.getVideoTitle());
+        assertEquals(LearningEventType.VIDEO_CLOSED_BEFORE_COMPLETION, event2nd.getLearningEventType());
+        assertEquals("{'video_playback_position_ms': 58007}", event2nd.getAdditionalData());
+    }
+}

src/test/resources/ai/elimu/util/csv/e387e38700000001_3001018_video-learning-events_2024-10-09.csv

@@ -0,0 +1,7 @@
+id,timestamp,android_id,package_name,video_id,video_title,learning_event_type,additional_data
+0,1728486312687,e387e38700000001,ai.elimu.analytics,13,akili and me - the rectangle song,VIDEO_OPENED,
+0,1728486319885,e387e38700000001,ai.elimu.analytics,13,akili and me - the rectangle song,VIDEO_CLOSED_BEFORE_COMPLETION,{'video_playback_position_ms': 58007}
+0,1728486312687,e387e38700000001,ai.elimu.analytics,6,akili and me - letter f,VIDEO_OPENED,
+0,1728486316065,e387e38700000001,ai.elimu.analytics,6,akili and me - letter f,VIDEO_CLOSED_BEFORE_COMPLETION,{'video_playback_position_ms': 6831}
+0,1728486312687,e387e38700000001,ai.elimu.analytics,34,akili and me - letter j,VIDEO_OPENED,
+0,1728486363791,e387e38700000001,ai.elimu.analytics,34,akili and me - letter j,VIDEO_CLOSED_BEFORE_COMPLETION,{'video_playback_position_ms': 32243}