Merge pull request #60 from RADAR-base/release-0.3.1

Release 0.3.1

Dockerfile

@@ -40,7 +40,7 @@ COPY ./kafka-connect-fitbit-source/src/ /code/kafka-connect-fitbit-source/src
 
 RUN ./gradlew jar
 
-FROM confluentinc/cp-kafka-connect-base:5.1.0
+FROM confluentinc/cp-kafka-connect-base:5.5.0
 
 MAINTAINER Joris Borgdorff <[email protected]>
 

README.md

@@ -63,6 +63,12 @@ your Fitbit App client ID and client secret. The following tables shows the poss
 <tr>
 <td>fitbit.user.repository.url</td></td><td>URL for webservice containing user credentials. Only used if a webservice-based user repository is configured.</td></td><td>string</td></td><td>""</td></td><td></td></td><td>low</td></td></tr>
 <tr>
+<td>fitbit.user.repository.client.id</td></td><td>Client ID for connecting to the service repository.</td></td><td>string</td></td><td>""</td></td><td></td></td><td>medium</td></td></tr>
+<tr>
+<td>fitbit.user.repository.client.secret</td></td><td>Client secret for connecting to the service repository.</td></td><td>string</td></td><td>""</td></td><td></td></td><td>medium</td></td></tr>
+<tr>
+<td>fitbit.user.repository.oauth2.token.url</td></td><td>OAuth 2.0 token url for retrieving client credentials.</td></td><td>string</td></td><td>""</td></td><td></td></td><td>medium</td></td></tr>
+<tr>
 <td>fitbit.intraday.steps.topic</td></td><td>Topic for Fitbit intraday steps</td></td><td>string</td></td><td>connect_fitbit_intraday_steps</td></td><td>non-empty string without control characters</td></td><td>low</td></td></tr>
 <tr>
 <td>fitbit.intraday.heart.rate.topic</td></td><td>Topic for Fitbit intraday heart_rate</td></td><td>string</td></td><td>connect_fitbit_intraday_heart_rate</td></td><td>non-empty string without control characters</td></td><td>low</td></td></tr>
@@ -76,8 +82,26 @@ your Fitbit App client ID and client secret. The following tables shows the poss
 <td>fitbit.activity.log.topic</td></td><td>Topic for Fitbit activity log.</td></td><td>string</td></td><td>connect_fitbit_activity_log</td></td><td>non-empty string without control characters</td></td><td>low</td></td></tr>
 <tr>
 <td>fitbit.intraday.calories.topic</td></td><td>Topic for Fitbit intraday calories</td></td><td>string</td></td><td>connect_fitbit_intraday_calories</td></td><td>non-empty string without control characters</td></td><td>low</td></td></tr>
+<tr>
+<td>fitbit.user.firebase.collection.fitbit.name</td></td><td>Firestore Collection for retrieving Fitbit Auth details. Only used when a Firebase based user repository is used.</td></td><td>string</td></td><td>fitbit</td></td><td></td></td><td>low</td></td></tr>
+<tr>
+<td>fitbit.user.firebase.collection.user.name</td></td><td>Firestore Collection for retrieving User details. Only used when a Firebase based user repository is used.</td></td><td>string</td></td><td>users</td></td><td></td></td><td>low</td></td></tr>
 </tbody></table>
 
+If the ManagementPortal is used to authenticate against the user repository, please add an OAuth client to ManagementPortal with the following properties:
+
+```
+Client ID: fitbit.user.repository.client.id
+Client Secret: fitbit.user.repository.client.secret
+Scope: SUBJECT.READ
+Resources: res_restAuthorizer
+Grant types: client_credentials
+Access Token validity: 600
+Refresh Token validity: 0
+```
+
+Finally set the `fitbit.user.repository.oauth.token.url` to `http://managementportal-app:8080/managementportal/oauth/token`.
+
 Now you can run a full Kafka stack using
 
 ```shell

build.gradle

@@ -2,16 +2,16 @@ description = 'kafka-connect-rest-source'
 
 subprojects {
     ext {
-        kafkaVersion = '2.2.0-cp2'
-        confluentVersion = '5.2.1'
-        jacksonVersion = '2.9.9'
+        kafkaVersion = '2.5.0'
+        confluentVersion = '5.5.0'
+        jacksonVersion = '2.11.0'
     }
 
     apply plugin: 'java'
     apply plugin: 'java-library'
 
     group = 'org.radarbase'
-    version = '0.3.0'
+    version = '0.3.1'
 
     sourceCompatibility = 1.8
     targetCompatibility = 1.8
@@ -21,12 +21,13 @@ subprojects {
         maven { url "https://packages.confluent.io/maven/" }
         maven { url "https://repo.maven.apache.org/maven2" }
         jcenter()
+        maven { url  "https://dl.bintray.com/radar-cns/org.radarcns" }
         maven { url  'https://oss.jfrog.org/artifactory/oss-snapshot-local/' }
     }
 }
 
 wrapper {
-    gradleVersion '5.4.1'
+    gradleVersion '6.4.1'
 }
 
 evaluationDependsOnChildren()

docker/kafka-wait

@@ -18,12 +18,11 @@ max_timeout=32
 tries=10
 timeout=1
 while true; do
-    ZOOKEEPER_CHECK=$(zookeeper-shell ${CONNECT_ZOOKEEPER_CONNECT} <<< "ls /brokers/ids" | tail -2 | head -1)
+    ZOOKEEPER_CHECK=$(zookeeper-shell ${CONNECT_ZOOKEEPER_CONNECT} <<< "ls /brokers/ids" | tail -n 1)
     echo "Zookeeper response: ${ZOOKEEPER_CHECK}"
-    # ZOOKEEPER_CHECK="${ZOOKEEPER_CHECK##*$'\n'}"
     ZOOKEEPER_CHECK="$(echo -e "${ZOOKEEPER_CHECK}" | tr -d '[:space:]'  | tr -d '['  | tr -d ']')"
 
-    IFS=',' read -r -a array <<< ${ZOOKEEPER_CHECK}
+    IFS=',' read -r -a array <<< "${ZOOKEEPER_CHECK}"
     LENGTH=${#array[@]}
     if [ "$LENGTH" -eq "$KAFKA_BROKERS" ]; then
         echo "Kafka brokers available."

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-5.4.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.4.1-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradlew

@@ -7,7 +7,7 @@
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -82,6 +82,7 @@ esac
 
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 
+
 # Determine the Java command to use to start the JVM.
 if [ -n "$JAVA_HOME" ] ; then
     if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
@@ -125,10 +126,11 @@ if $darwin; then
     GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
 fi
 
-# For Cygwin, switch paths to Windows format before running java
-if $cygwin ; then
+# For Cygwin or MSYS, switch paths to Windows format before running java
+if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+
     JAVACMD=`cygpath --unix "$JAVACMD"`
 
     # We build the pattern for arguments to be converted via cygpath
@@ -154,19 +156,19 @@ if $cygwin ; then
         else
             eval `echo args$i`="\"$arg\""
         fi
-        i=$((i+1))
+        i=`expr $i + 1`
     done
     case $i in
-        (0) set -- ;;
-        (1) set -- "$args0" ;;
-        (2) set -- "$args0" "$args1" ;;
-        (3) set -- "$args0" "$args1" "$args2" ;;
-        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
-        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
-        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
-        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
-        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
-        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+        0) set -- ;;
+        1) set -- "$args0" ;;
+        2) set -- "$args0" "$args1" ;;
+        3) set -- "$args0" "$args1" "$args2" ;;
+        4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
     esac
 fi
 
@@ -175,14 +177,9 @@ save () {
     for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
     echo " "
 }
-APP_ARGS=$(save "$@")
+APP_ARGS=`save "$@"`
 
 # Collect all arguments for the java command, following the shell quoting and substitution rules
 eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
 
-# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
-if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
-  cd "$(dirname "$0")"
-fi
-
 exec "$JAVACMD" "$@"

gradlew.bat

@@ -5,7 +5,7 @@
 @rem you may not use this file except in compliance with the License.
 @rem You may obtain a copy of the License at
 @rem
-@rem      http://www.apache.org/licenses/LICENSE-2.0
+@rem      https://www.apache.org/licenses/LICENSE-2.0
 @rem
 @rem Unless required by applicable law or agreed to in writing, software
 @rem distributed under the License is distributed on an "AS IS" BASIS,
@@ -29,6 +29,9 @@ if "%DIRNAME%" == "" set DIRNAME=.
 set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
+@rem Resolve any "." and ".." in APP_HOME to make it shorter.
+for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
+
 @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
 set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
 
@@ -81,6 +84,7 @@ set CMD_LINE_ARGS=%*
 
 set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
 
+
 @rem Execute Gradle
 "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
 

kafka-connect-fitbit-source/build.gradle

@@ -3,6 +3,7 @@ dependencies {
     api group: 'io.confluent', name: 'kafka-connect-avro-converter', version: confluentVersion
     api group: 'org.radarcns', name: 'radar-schemas-commons', version: '0.5.3'
 
+    implementation group: 'org.radarcns', name: 'oauth-client-util', version: '0.5.8'
 
     implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: jacksonVersion
     implementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310', version: jacksonVersion

kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/FitbitRestSourceConnectorConfig.java

@@ -20,6 +20,8 @@
 import static org.apache.kafka.common.config.ConfigDef.NO_DEFAULT_VALUE;
 
 import java.lang.reflect.InvocationTargetException;
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -122,6 +124,19 @@ public class FitbitRestSourceConnectorConfig extends RestSourceConnectorConfig {
   private static final String FITBIT_INTRADAY_CALORIES_TOPIC_DISPLAY = "Intraday calories topic";
   private static final String FITBIT_INTRADAY_CALORIES_TOPIC_DEFAULT = "connect_fitbit_intraday_calories";
 
+
+  public static final String FITBIT_USER_REPOSITORY_CLIENT_ID_CONFIG = "fitbit.user.repository.client.id";
+  private static final String FITBIT_USER_REPOSITORY_CLIENT_ID_DOC = "Client ID for connecting to the service repository.";
+  private static final String FITBIT_USER_REPOSITORY_CLIENT_ID_DISPLAY = "Client ID for user repository.";
+
+  public static final String FITBIT_USER_REPOSITORY_CLIENT_SECRET_CONFIG = "fitbit.user.repository.client.secret";
+  private static final String FITBIT_USER_REPOSITORY_CLIENT_SECRET_DOC = "Client secret for connecting to the service repository.";
+  private static final String FITBIT_USER_REPOSITORY_CLIENT_SECRET_DISPLAY = "Client Secret for user repository.";
+
+  public static final String FITBIT_USER_REPOSITORY_TOKEN_URL_CONFIG = "fitbit.user.repository.oauth2.token.url";
+  private static final String FITBIT_USER_REPOSITORY_TOKEN_URL_DOC = "OAuth 2.0 token url for retrieving client credentials.";
+  private static final String FITBIT_USER_REPOSITORY_TOKEN_URL_DISPLAY = "OAuth 2.0 token URL.";
+
   public static final String FITBIT_USER_REPOSITORY_FIRESTORE_FITBIT_COLLECTION_CONFIG = "fitbit.user.firebase.collection.fitbit.name";
   private static final String FITBIT_USER_REPOSITORY_FIRESTORE_FITBIT_COLLECTION_DOC = "Firestore Collection for retrieving Fitbit Auth details. Only used when a Firebase based user repository is used.";
   private static final String FITBIT_USER_REPOSITORY_FIRESTORE_FITBIT_COLLECTION_DISPLAY = "Firebase Fitbit collection name.";
@@ -242,6 +257,37 @@ public String toString() {
             Width.SHORT,
             FITBIT_USER_REPOSITORY_URL_DISPLAY)
 
+
+        .define(FITBIT_USER_REPOSITORY_CLIENT_ID_CONFIG,
+            Type.STRING,
+            "",
+            Importance.MEDIUM,
+            FITBIT_USER_REPOSITORY_CLIENT_ID_DOC,
+            group,
+            ++orderInGroup,
+            Width.SHORT,
+            FITBIT_USER_REPOSITORY_CLIENT_ID_DISPLAY)
+
+        .define(FITBIT_USER_REPOSITORY_CLIENT_SECRET_CONFIG,
+            Type.PASSWORD,
+            "",
+            Importance.MEDIUM,
+            FITBIT_USER_REPOSITORY_CLIENT_SECRET_DOC,
+            group,
+            ++orderInGroup,
+            Width.SHORT,
+            FITBIT_USER_REPOSITORY_CLIENT_SECRET_DISPLAY)
+
+        .define(FITBIT_USER_REPOSITORY_TOKEN_URL_CONFIG,
+            Type.STRING,
+            "",
+            Importance.MEDIUM,
+            FITBIT_USER_REPOSITORY_TOKEN_URL_DOC,
+            group,
+            ++orderInGroup,
+            Width.SHORT,
+            FITBIT_USER_REPOSITORY_TOKEN_URL_DISPLAY)
+
         .define(FITBIT_INTRADAY_STEPS_TOPIC_CONFIG,
             Type.STRING,
             FITBIT_INTRADAY_STEPS_TOPIC_DEFAULT,
@@ -447,4 +493,25 @@ public String getFitbitUserRepositoryFirestoreFitbitCollection() {
   public String getFitbitUserRepositoryFirestoreUserCollection() {
     return getString(FITBIT_USER_REPOSITORY_FIRESTORE_USER_COLLECTION_CONFIG);
   }
+
+  public String getFitbitUserRepositoryClientId() {
+    return getString(FITBIT_USER_REPOSITORY_CLIENT_ID_CONFIG);
+  }
+
+  public String getFitbitUserRepositoryClientSecret() {
+    return getPassword(FITBIT_USER_REPOSITORY_CLIENT_SECRET_CONFIG).value();
+  }
+
+  public URL getFitbitUserRepositoryTokenUrl() {
+    String value = getString(FITBIT_USER_REPOSITORY_TOKEN_URL_CONFIG);
+    if (value == null || value.isEmpty()) {
+      return null;
+    } else {
+      try {
+        return new URL(getString(FITBIT_USER_REPOSITORY_TOKEN_URL_CONFIG));
+      } catch (MalformedURLException e) {
+        throw new ConfigException("Fitbit user repository token URL is invalid.");
+      }
+    }
+  }
 }

kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitActivityLogAvroConverter.java

@@ -85,7 +85,7 @@ private FitbitActivityLogRecord getRecord(JsonNode s, OffsetDateTime startTime)
         .setLogType(optString(s, "logType").orElse(null))
         .setType(optLong(s, "activityType").orElse(null))
         .setSpeed(optDouble(s, "speed").orElse(null))
-        .setDistance(optDouble(s, "distance").map(Float::new).orElse(null))
+        .setDistance(optDouble(s, "distance").map(Double::floatValue).orElse(null))
         .setSteps(optInt(s, "steps").orElse(null))
         .setEnergy(optInt(s, "calories").map(e -> e * FOOD_CAL_TO_KJOULE_FACTOR)
             .orElse(null))

kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitPollingRoute.java

@@ -25,6 +25,7 @@
 import static org.radarbase.connect.rest.converter.PayloadToSourceRecordConverter.MIN_INSTANT;
 import static org.radarbase.connect.rest.converter.PayloadToSourceRecordConverter.TIMESTAMP_OFFSET_KEY;
 import static org.radarbase.connect.rest.converter.PayloadToSourceRecordConverter.nearFuture;
+import static org.radarbase.connect.rest.request.PollingRequestRoute.max;
 
 import java.io.IOException;
 import java.time.Duration;
@@ -316,7 +317,7 @@ protected Instant nextPoll(User user) {
     } else {
       Instant nextPoll = lastPollPerUser.getOrDefault(user.getId(), MIN_INSTANT)
           .plus(getPollIntervalPerUser());
-      return PollingRequestRoute.max(offset.plus(getLookbackTime()), nextPoll);
+      return max(offset.plus(getLookbackTime()), nextPoll);
     }
   }