traffic-a22-forecast: add historical import funcionality

noi-techpark · Sep 20, 2023 · 1f8c098 · 1f8c098
1 parent eaecfac
commit 1f8c098
Show file tree

Hide file tree

Showing 5 changed files with 66 additions and 10 deletions.
diff --git a/data-collectors/traffic-a22-forecast/.env.example b/data-collectors/traffic-a22-forecast/.env.example
@@ -3,6 +3,12 @@ A22_URL=
 A22_USER=
 A22_PASSWORD=
 
+FORECAST_MONTHS=3
+
+HISTORY_IMPORT=false
+HISTORY_START_DATE=01.2023
+HISTORY_DATE_FORMAT=mm.yyyy
+
 SCHEDULER_CRON=0 */10 * * * *
 
 SCHEDULER_POOL_SIZE=1

diff --git a/...traffic-a22-forecast/src/main/java/com/opendatahub/traffic/a22/forecast/JobScheduler.java b/...traffic-a22-forecast/src/main/java/com/opendatahub/traffic/a22/forecast/JobScheduler.java
@@ -6,12 +6,18 @@
 
 import org.slf4j.LoggerFactory;
 
-import java.util.Arrays;
+import java.time.YearMonth;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.GregorianCalendar;
 import java.util.List;
-import java.util.Map;
+
+import javax.annotation.PostConstruct;
 
 import org.slf4j.Logger;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Lazy;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
@@ -22,11 +28,19 @@
 import com.opendatahub.traffic.a22.forecast.services.OdhClient;
 import com.opendatahub.traffic.a22.forecast.config.ProvenanceConfig;
 
-
 @Service
 public class JobScheduler {
     private static final Logger LOG = LoggerFactory.getLogger(JobScheduler.class);
 
+    @Value("${historyimport.enabled}")
+    private boolean historyEnabled;
+
+    @Value("#{new java.text.SimpleDateFormat('${historyimport.dateformat}').parse('${historyimport.startdate}')}")
+    private YearMonth historyStartDate;
+
+    @Value("${forecast.months}")
+    private int forecastMonths;
+
     @Lazy
     @Autowired
     private OdhClient odhClient;
@@ -44,12 +58,36 @@ public class JobScheduler {
     @Autowired
     private ProvenanceConfig provC;
 
+    @PostConstruct
+    public void postConstruct() {
+        if (historyEnabled) {
+            LOG.info("Historical import job started...");
+            syncData(historyStartDate, YearMonth.now());
+            LOG.info("Historical import job successful.");
+        }
+    }
+
     @Scheduled(cron = "${scheduler.job}")
-    public void collectBikeBoxData() {
-        LOG.info("Cron job started");
+    public void forecastImport() {
+        LOG.info("Cron job stations started...");
+        YearMonth currentDate = YearMonth.now();
+        syncData(currentDate, currentDate.plusMonths(forecastMonths));
+        LOG.info("Cron job successful.");
+
+    }
+
+    public void syncData(YearMonth from, YearMonth to) {
+        LOG.info("Sync started from {}...", historyStartDate.toString());
+
+        List<ForecastDto> forecasts = new ArrayList<>();
+
+        while (historyStartDate.isBefore(to)) {
+            forecasts.add(a22Service.getForecasts(from));
+            from = from.plusMonths(1);
+        }
 
-        ForecastDto forecasts = a22Service.getForecasts("2023", "9");
-        LOG.info("DONE");
+        // sync with Open Data Hub
 
+        LOG.info("Sync done. Imported {} months.", forecasts.size());
     }
 }
diff --git a/...2-forecast/src/main/java/com/opendatahub/traffic/a22/forecast/config/A22ClientConfig.java b/...2-forecast/src/main/java/com/opendatahub/traffic/a22/forecast/config/A22ClientConfig.java
@@ -11,6 +11,7 @@
 
 @Configuration
 public class A22ClientConfig {
+
         @Value("${a22.url}")
         private String url;
 

diff --git a/...-a22-forecast/src/main/java/com/opendatahub/traffic/a22/forecast/services/A22Service.java b/...-a22-forecast/src/main/java/com/opendatahub/traffic/a22/forecast/services/A22Service.java
@@ -4,6 +4,8 @@
 
 package com.opendatahub.traffic.a22.forecast.services;
 
+import java.time.YearMonth;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -23,13 +25,13 @@ public class A22Service {
     @Qualifier("a22Client") // avoid overlap with webClient in bdp-core
     private WebClient client;
 
-    public ForecastDto getForecasts(String year, String month) {
+    public ForecastDto getForecasts(YearMonth date) {
         return client.post()
                 .uri(u -> u.path(ENDPOINT_FORECAST).build())
                 .accept(MediaType.ALL)
                 .contentType(MediaType.APPLICATION_JSON)
-                .header("month", month)
-                .header("year", year)
+                .header("month", String.valueOf(date.getMonthValue()))
+                .header("year", String.valueOf(date.getYear()))
                 .header("User-Agent", "NOI/A22TrafficForecastConnector")
                 .retrieve()
                 .bodyToMono(ForecastDto.class)

diff --git a/data-collectors/traffic-a22-forecast/src/main/resources/application.properties b/data-collectors/traffic-a22-forecast/src/main/resources/application.properties
@@ -2,6 +2,15 @@ a22.url=${A22_URL}
 a22.username=${A22_USER}
 a22.password=${A22_PASSWORD}
 
+# how many months of forecast will be imported
+forecast.months=${FORECAST_MONTHS:3}
+
+# import of historical data
+# can only be imported once on database with no date after HISTORY_START_DATE
+historyimport.enabled=${HISTORY_IMPORT:false}
+historyimport.startdate=${HISTORY_START_DATE:01.2023}
+historyimport.dateformat=${HISTORY_DATE_FORMAT:mm.yyyy}
+
 # Cron definition to start jobs
 scheduler.job=${SCHEDULER_CRON:*/10 * * * * *}