Merge pull request #123 from RADAR-base/release-0.5.2

Release 0.5.2

.github/workflows/codeql.yml

@@ -0,0 +1,89 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "master", "dev" ]
+  pull_request:
+    branches: [ "master", "dev" ]
+  schedule:
+    - cron: '24 21 * * 0'
+
+jobs:
+  analyze:
+    name: Analyze
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners
+    # Consider using larger runners for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'java-kotlin' ]
+        # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ]
+        # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin' # See 'Supported distributions' for available options
+        java-version: '17'
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+
+    # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v3
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #     echo "Run, Build Application using script"
+    #     ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/main.yml

@@ -25,7 +25,7 @@ jobs:
       - uses: actions/setup-java@v3
         with:
           distribution: temurin
-          java-version: 11
+          java-version: 17
 
       - name: Setup Gradle
         uses: gradle/gradle-build-action@v2

.github/workflows/oura.yml

@@ -25,7 +25,7 @@ jobs:
       - uses: actions/setup-java@v3
         with:
           distribution: temurin
-          java-version: 11
+          java-version: 17
 
       - name: Setup Gradle
         uses: gradle/gradle-build-action@v2

.github/workflows/release-fitbit.yml

@@ -20,7 +20,7 @@ jobs:
       - uses: actions/setup-java@v3
         with:
           distribution: temurin
-          java-version: 11
+          java-version: 17
 
       - name: Gradle cache
         uses: actions/cache@v3

.github/workflows/release-oura.yml

@@ -20,7 +20,7 @@ jobs:
       - uses: actions/setup-java@v3
         with:
           distribution: temurin
-          java-version: 11
+          java-version: 17
 
       - name: Gradle cache
         uses: actions/cache@v3

README.md

@@ -8,7 +8,7 @@ Fitbit in particular. The documentation of the Kafka Connect REST source still n
 ### Installation
 
 This repository relies on a recent version of docker and docker-compose as well as an installation
-of Java 11 or later.
+of Java 17 or later.
 
 ### Usage
 

buildSrc/build.gradle.kts

@@ -1,5 +1,5 @@
 plugins {
-    kotlin("jvm") version "1.9.10"
+    kotlin("jvm") version "1.9.22"
 }
 
 repositories {

buildSrc/src/main/kotlin/Versions.kt

@@ -1,33 +1,33 @@
 @Suppress("ConstPropertyName", "MemberVisibilityCanBePrivate")
 object Versions {
-    const val project = "0.5.1"
+    const val project = "0.5.2"
 
-    const val java = 11
+    const val java = 17
     const val kotlin = "1.9.10"
     const val wrapper = "8.4"
 
-    const val radarCommons = "1.1.1"
-    const val confluent = "7.5.0"
+    const val radarCommons = "1.1.2"
+    const val confluent = "7.6.0"
     const val kafka = "$confluent-ce"
-    const val avro = "1.11.0"
+    const val avro = "1.11.3"
 
     const val managementPortal = "2.0.0"
 
     // From image
-    const val jackson = "2.14.2"
+    const val jackson = "2.16.1"
 
     const val log4j2 = "2.20.0"
     const val slf4j = "2.0.9"
 
-    const val okhttp = "4.11.0"
+    const val okhttp = "4.12.0"
 
     const val firebaseAdmin = "9.1.0"
-    const val radarSchemas = "0.8.6"
+    const val radarSchemas = "0.8.7-hotfix"
     const val ktor = "2.3.5"
 
     const val junit = "5.9.3"
     const val wiremock = "2.27.2"
     const val mockito = "5.3.1"
 
-    const val kotlinVersion = "1.8.21"
+    const val kotlinVersion = "1.9.10"
 }

kafka-connect-fitbit-source/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM --platform=$BUILDPLATFORM gradle:8.4-jdk11 as builder
+FROM --platform=$BUILDPLATFORM gradle:8.4-jdk17 as builder
 
 RUN mkdir /code
 WORKDIR /code

kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/FitbitRestSourceConnectorConfig.java

@@ -106,6 +106,11 @@ public class FitbitRestSourceConnectorConfig extends RestSourceConnectorConfig {
   private static final String FITBIT_INTRADAY_HEART_RATE_VARIABILITY_TOPIC_DISPLAY = "Intraday heart rate variability topic";
   private static final String FITBIT_INTRADAY_HEART_RATE_VARIABILITY_TOPIC_DEFAULT = "connect_fitbit_intraday_heart_rate_variability";
 
+  private static final String FITBIT_INTRADAY_SPO2_TOPIC_CONFIG = "fitbit.intraday.spo2.topic";
+  private static final String FITBIT_INTRADAY_SPO2_TOPIC_DOC = "Topic for Fitbit intraday intraday_spo2";
+  private static final String FITBIT_INTRADAY_SPO2_TOPIC_DISPLAY = "Intraday spo2 topic";
+  private static final String FITBIT_INTRADAY_SPO2_TOPIC_DEFAULT = "connect_fitbit_intraday_spo2";
+
   private static final String FITBIT_BREATHING_RATE_TOPIC_CONFIG = "fitbit.breathing.rate.topic";
   private static final String FITBIT_BREATHING_RATE_TOPIC_DOC = "Topic for Fitbit breathing rate";
   private static final String FITBIT_BREATHING_RATE_TOPIC_DISPLAY = "Breathing rate topic";
@@ -343,6 +348,17 @@ public String toString() {
             Width.SHORT,
             FITBIT_INTRADAY_HEART_RATE_VARIABILITY_TOPIC_DISPLAY)
 
+        .define(FITBIT_INTRADAY_SPO2_TOPIC_CONFIG,
+            Type.STRING,
+            FITBIT_INTRADAY_SPO2_TOPIC_DEFAULT,
+            nonControlChar,
+            Importance.LOW,
+            FITBIT_INTRADAY_SPO2_TOPIC_DOC,
+            group,
+            ++orderInGroup,
+            Width.SHORT,
+            FITBIT_INTRADAY_SPO2_TOPIC_DISPLAY)
+
         .define(FITBIT_BREATHING_RATE_TOPIC_CONFIG,
             Type.STRING,
             FITBIT_BREATHING_RATE_TOPIC_DEFAULT,
@@ -511,6 +527,10 @@ public String getFitbitIntradayHeartRateVariabilityTopic() {
     return getString(FITBIT_INTRADAY_HEART_RATE_VARIABILITY_TOPIC_CONFIG);
   }
 
+  public String getFitbitIntradaySpo2Topic() {
+    return getString(FITBIT_INTRADAY_SPO2_TOPIC_CONFIG);
+  }
+
   public String getFitbitBreathingRateTopic() {
     return getString(FITBIT_BREATHING_RATE_TOPIC_CONFIG);
   }

kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitBreathingRateAvroConverter.java

@@ -26,9 +26,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.time.Instant;
-import java.time.LocalDateTime;
-import java.time.ZonedDateTime;
+import java.time.*;
 import java.util.stream.Stream;
 
 import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull;
@@ -58,23 +56,22 @@ protected Stream<TopicData> processRecords(FitbitRestRequest request, JsonNode r
 
         return iterableToStream(br)
             .filter(m -> m != null && m.isObject())
-            .flatMap(FitbitAvroConverter::iterableToStream)
             .map(tryOrNull(m -> parseBr(m, startDate, timeReceived),
                 (a, ex) -> logger.warn("Failed to convert breathing rate from request {}, {}", request, a, ex)));
     }
 
     private TopicData parseBr(JsonNode data, ZonedDateTime startDate, double timeReceived) {
-      Instant time = startDate.with(LocalDateTime.parse(data.get("dateTime").asText())).toInstant();
+      Instant time = LocalDate.parse(data.get("dateTime").asText()).atStartOfDay(ZoneOffset.UTC).toInstant();
       JsonNode value = data.get("value");
       if (value == null || !value.isObject()) {
         return null;
       }
       FitbitBreathingRate fitbitBr = new FitbitBreathingRate(time.toEpochMilli() / 1000d,
               timeReceived,
-              (float) value.get("deepSleepBrSummary").get("breathingRate").asDouble(),
-              (float) value.get("remSleepBrSummary").get("breathingRate").asDouble(),
-              (float) value.get("fullSleepBrSummary").get("breathingRate").asDouble(),
-              (float) value.get("lightSleepBrSummary").get("breathingRate").asDouble());
+              (float) value.get("deepSleepSummary").get("breathingRate").asDouble(),
+              (float) value.get("remSleepSummary").get("breathingRate").asDouble(),
+              (float) value.get("fullSleepSummary").get("breathingRate").asDouble(),
+              (float) value.get("lightSleepSummary").get("breathingRate").asDouble());
       return new TopicData(time, breathingRateTopic, fitbitBr);
     }
 }

kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayHeartRateVariabilityAvroConverter.java

@@ -73,7 +73,7 @@ private TopicData parseHrv(JsonNode minuteData, ZonedDateTime startDate, double
       }
       FitbitIntradayHeartRateVariability fitbitHrv = new FitbitIntradayHeartRateVariability(time.toEpochMilli() / 1000d,
               timeReceived,
-              (float) value.get("dailyRmssd").asDouble(),
+              (float) value.get("rmssd").asDouble(),
               (float) value.get("coverage").asDouble(),
               (float) value.get("hf").asDouble(),
               (float) value.get("lf").asDouble());

kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradaySpo2AvroConverter.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2018 The Hyve
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.radarbase.connect.rest.fitbit.converter;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import io.confluent.connect.avro.AvroData;
+import org.radarbase.connect.rest.RestSourceConnectorConfig;
+import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig;
+import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest;
+import org.radarcns.connector.fitbit.FitbitIntradaySpo2;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZonedDateTime;
+import java.util.stream.Stream;
+
+import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull;
+
+public class FitbitIntradaySpo2AvroConverter extends FitbitAvroConverter {
+    private static final Logger logger = LoggerFactory.getLogger(FitbitIntradaySpo2AvroConverter.class);
+    private String spo2Topic;
+
+    public FitbitIntradaySpo2AvroConverter(AvroData avroData) {
+        super(avroData);
+    }
+
+    @Override
+    public void initialize(RestSourceConnectorConfig config) {
+        spo2Topic = ((FitbitRestSourceConnectorConfig) config).getFitbitIntradaySpo2Topic();
+        logger.info("Using intraday spo2 topic {}", spo2Topic);
+    }
+
+    @Override
+    protected Stream<TopicData> processRecords(FitbitRestRequest request, JsonNode root, double timeReceived) {
+        JsonNode spo2 = root;
+        if (spo2 == null || !spo2.isArray()) {
+            logger.warn("No Spo2 is provided for {}: {}", request, root);
+            return Stream.empty();
+        }
+        ZonedDateTime startDate = request.getDateRange().end();
+
+        return iterableToStream(spo2)
+                .filter(m -> m != null && m.isObject())
+                .map(m -> m.get("minutes"))
+                .filter(minutes -> minutes != null && minutes.isArray())
+                .flatMap(FitbitAvroConverter::iterableToStream)
+                .map(tryOrNull(minuteData -> parseSpo2(minuteData, startDate, timeReceived),
+                        (a, ex) -> logger.warn("Failed to convert spo2 from request {}, {}", request, a, ex)));
+    }
+
+    private TopicData parseSpo2(JsonNode minuteData, ZonedDateTime startDate, double timeReceived) {
+      Instant time = startDate.with(LocalDateTime.parse(minuteData.get("minute").asText())).toInstant();
+      Float value = (float) minuteData.get("value").asDouble();
+      if (value == null) {
+        return null;
+      }
+      FitbitIntradaySpo2 fitbitSpo2 = new FitbitIntradaySpo2(time.toEpochMilli() / 1000d,
+              timeReceived,
+              (float) value);
+      return new TopicData(time, spo2Topic, fitbitSpo2);
+    }
+}