Full dockerization of UDMI local setup (#932)

.github/workflows/testing.yml

@@ -184,7 +184,7 @@ jobs:
         run: more out/pubber.log* pubber/out/*.json | cat
       - name: udmis log
         if: ${{ !cancelled() }}
-        run: cat out/udmis*.log || true
+        run: cat /tmp/udmis.log || true
       - name: itemized test post-process
         if: ${{ !cancelled() }}
         run: egrep ' test .* after .*s ' out/sequencer.log-* | tee out/timing_itemized.out
@@ -204,7 +204,7 @@ jobs:
     name: Baseline Tests
     runs-on: ubuntu-latest
     needs: pretest
-    timeout-minutes: 10
+    timeout-minutes: 15
     if: vars.TARGET_PROJECT != ''
     env:
       TARGET_PROJECT: ${{ vars.TARGET_PROJECT }}
@@ -280,7 +280,7 @@ jobs:
           sudo more /etc/mosquitto/* /etc/mosquitto/conf.d/* || true
       - name: udmis log
         if: ${{ !cancelled() }}
-        run: cat out/udmis*.log
+        run: cat /tmp/udmis.log
       - name: mosquitto logs
         if: ${{ !cancelled() }}
         run: cat /var/log/mosquitto/mosquitto.log || true
@@ -298,7 +298,7 @@ jobs:
     name: Endpoint Redirection
     runs-on: ubuntu-latest
     needs: [ pretest, baseline ]
-    timeout-minutes: 10
+    timeout-minutes: 15
     if: vars.TARGET_PROJECT != ''
     env:
       TARGET_PROJECT: ${{ vars.TARGET_PROJECT }}

.gitignore

@@ -25,8 +25,6 @@ credentials.json
 /udmis/var/
 /udmis/profile/
 /udmis/.idea/libraries/
-/udmis/udmi_bin/
-/udmis/udmi_etc/
 /selfie/build/
 /pubber/build/
 /pubber/out/

bin/container

@@ -22,7 +22,7 @@ git_repo=${git_prefix#*.com:}
 
 GCP_PROJECT=$(gcloud config get project)
 REPOSITORY=ghcr.io/${git_repo}
-TEMPLATES=$(cd etc; ls k8s_*.yaml)
+TEMPLATES=$(cd etc; ls k8s_*.yaml) || true
 
 NOCHECK=
 [[ ${1:-} == "--no-check" ]] && NOCHECK=1 && shift
@@ -91,7 +91,7 @@ if [[ -n $prep ]]; then
     rm -rf var tmp && mkdir -p var tmp
 
     cp -f etc/*.json var/ || true
-    cp -a ../schema tmp/
+    cp -a ../schema build/
 
     bin/build
 

bin/keygen

@@ -21,12 +21,19 @@ else
     SUBJ=client
 fi
 
+TARGET_HOST=localhost
+if [[ $type =~ ^CERT/ || $type =~ ^CA/ ]]; then
+    TARGET_HOST=${type#*/}
+    type=${type%/*}
+    echo Generating $type with altname $TARGET_HOST
+fi
+
 CA_CRT=$CA_DIR/ca.crt
 CA_SRL=$CA_DIR/ca.srl
 CA_KEY=$CA_DIR/rsa_private.pem
 OPTS_509=/tmp/x509opts.txt
 
-echo "subjectAltName=DNS:localhost, IP:127.0.0.1" > $OPTS_509
+echo "subjectAltName=DNS:${TARGET_HOST}, IP:127.0.0.1, DNS:localhost" > $OPTS_509
 x509opts="-days 3650 -extfile $OPTS_509"
 
 if [[ $type == CA ]]; then

bin/mosquctl_log

@@ -1,12 +1,11 @@
 #!/bin/bash -e
 
 UDMI_ROOT=$(dirname $0)/..
+source $UDMI_ROOT/etc/shell_common.sh
 cd $UDMI_ROOT
 
 LOG_FILE=/var/log/mosquitto/mosquitto.log
 
-source $UDMI_ROOT/etc/shell_common.sh
-
 if [[ $# != 1 ]]; then
     echo Usage: $0 client_id_prefix
     false

bin/pubber

@@ -61,7 +61,7 @@ echo Cleaning output directory $UDMI_ROOT/pubber/out/$serial_no
 rm -rf $UDMI_ROOT/pubber/out/$serial_no
 
 echo Building pubber...
-$UDMI_ROOT/pubber/bin/build
+[[ ! -d $UDMI_ROOT/pubber/src ]] || $UDMI_ROOT/pubber/bin/build
 
 echo Running tools version $udmi_version
 
@@ -172,6 +172,13 @@ fi
 
 jq --slurpfile endpoint $ENDPOINT_TMP '.endpoint = $endpoint[0]' $tmp_config | sponge $tmp_config
 
+if [[ -f ${site_model:-}/reflector/ca.crt && -n ${project_target:-} ]]; then
+    echo Checking for signed device certificate...
+    certs=(${site_model}/devices/${target_id}/*.crt) || true
+    [[ -f $certs ]] || $UDMI_ROOT/bin/keygen CERT/${project_target} ${site_model}/devices/${target_id}/
+    ls -l ${site_model}/devices/${target_id}/*.crt
+fi
+
 # Run in the background to force new process group and get PID
 $UDMI_ROOT/pubber/bin/run $tmp_config &
 PUB_PID=$!

bin/sequencer

@@ -172,7 +172,7 @@ fi
 
 cat $SEQUENCER_CONFIG
 
-validator/bin/build
+[[ ! -d $UDMI_ROOT/validator/src ]] || $UDMI_ROOT/validator/bin/build
 
 echo $JAVA_CMD
 result=0
@@ -192,5 +192,3 @@ fgrep RESULT $site_model/out/devices/$device_id/RESULT.log | \
 more $site_model/out/devices/$device_id/RESULT.log $SCHEMA_OUT | cat
 
 bin/sequencer_report $site_model $device_id
-
-bin/check_version

bin/setup_base

@@ -3,7 +3,16 @@
 ROOT_DIR=$(realpath $(dirname $0)/..)
 cd $ROOT_DIR
 
-sudo apt-get install -y moreutils expect mosquitto mosquitto-clients
+if [[ -n $(which apt-get) ]]; then
+    update_cmd="apt-get install -y"
+    extra_packages=""
+else
+    update_cmd="apk add"
+    # Some extra packages are necessary for alpine based docker builds.
+    extra_packages="linux-headers g++ gcc"
+fi
+
+sudo $update_cmd $extra_packages moreutils expect python3-dev mosquitto mosquitto-clients
 
 python3 --version
 python3 -m venv venv

bin/setup_ca

@@ -6,22 +6,19 @@ source $UDMI_ROOT/etc/shell_common.sh
 CERT_DIR=/etc/mosquitto/certs/
 sudo rm -f $CERT_DIR/*
 
-[[ $# != 1 ]] && fail $0 site_model
+[[ $# != 2 ]] && fail $0 site_model hostname
 
 site_model=$(realpath $1)
+target=$2
 
 cd $UDMI_ROOT
 
-bin/keygen CA $site_model/reflector
-bin/keygen CERT $site_model/reflector
+bin/keygen CA/$target $site_model/reflector
+bin/keygen CERT/$target $site_model/reflector
 
 echo Importing server certs from $site_model...
 sudo cp $site_model/reflector/ca.* $CERT_DIR/
 sudo cp $site_model/reflector/rsa_private.* $CERT_DIR/
 sudo chgrp -R mosquitto $CERT_DIR
 sudo chmod 0640 $CERT_DIR/rsa_private.*
 ls -l $CERT_DIR
-
-
-
-

bin/start_etcd

@@ -2,6 +2,7 @@
 
 IMAGE=quay.io/coreos/etcd:v3.5.13
 BINDIR=/tmp/etcd
+ETCD_LOG=/tmp/etcd.log
 
 echo pwd: $(pwd)
 
@@ -11,11 +12,11 @@ $BINDIR/etcd -version
 
 $BINDIR/etcd -listen-client-urls=http://0.0.0.0:2379 \
              -advertise-client-urls=http://127.0.0.1:2379 \
-             > etcd.log 2>&1 &
+             > $ETCD_LOG 2>&1 &
 ETCD_PID=$!
 
-echo Waiting 10s for etcd to start...
+echo Waiting 10s for etcd to start, log in $ETCD_LOG
 sleep 10
-[[ -d /proc/$ETCD_PID ]] || (cat etcd.log && fail starting etcd)
+[[ -d /proc/$ETCD_PID ]] || (cat $ETCD_LOG && echo error starting etcd && false)
 
 echo Completed etcd startup.

bin/start_local

@@ -25,27 +25,34 @@ if [[ -d var/ && ! -d etc/ ]]; then
 fi
 
 cd $UDMI_ROOT
+UDMIS_LOG=/tmp/udmis.log
+mkdir -p out
+date > $UDMIS_LOG
 
 if [[ ! $project_spec =~ ^//mqtt/ ]]; then
-    echo Not a local setup, doing nothing!
+    echo Not a local setup, doing nothing! || tee -a $UDMIS_LOG
     exit 0
 fi
 
+project_target=${project_spec##*/}
 site_config=$site_model/cloud_iot_config.json
 registry_id=$(jq -r .registry_id $site_config)${UDMI_REGISTRY_SUFFIX:-}
 
+echo Starting etcd... | tee -a $UDMIS_LOG
 bin/start_etcd
 
 source $UDMI_ROOT/etc/mosquitto_ctrl.sh
 mkdir -p $CERT_DIR
 
-bin/setup_ca $site_model
+echo Starting mosquitto on server $project_target | tee -a $UDMIS_LOG
+bin/setup_ca $site_model $project_target
 bin/start_mosquitto
 
 $MOSQUITTO_CTRL deleteClient $SERV_USER
 $MOSQUITTO_CTRL createClient $SERV_USER -p $SERV_PASS # No client_id to allow multiple backend connections.
 $MOSQUITTO_CTRL addClientRole $SERV_USER service
 
+echo Starting initializing site $site_model | tee -a $UDMIS_LOG
 bin/mosquctl_site $site_model
 
 if [[ -n ${UDMI_ALT_REGISTRY:-} ]]; then
@@ -55,6 +62,7 @@ fi
 # Global access to logs for connection tracking by UDMIS.
 sudo chmod a+r /var/log/mosquitto/mosquitto.log
 
+echo Starting udmis proper... | tee -a $UDMIS_LOG
 bin/start_udmis
 
 echo Done with local server setup.

bin/start_mosquitto

@@ -15,8 +15,9 @@ GROUP=mosquitto
 UDMI_FILE=$ETC_DIR/conf.d/udmi.conf
 PASS_FILE=$ETC_DIR/mosquitto.passwd
 DYN_FILE=$ETC_DIR/dynamic_security.json
+MOSQUITTO_LOG=/var/log/mosquitto/mosquitto.log
 
-mkdir -p /var/log/mosquitto/
+sudo mkdir -p $(dirname $MOSQUITTO_LOG)
 
 if [[ ! -f $UDMI_FILE ]]; then
     echo Creating new $UDMI_FILE from template...
@@ -51,11 +52,11 @@ if [[ -n $(which systemctl) ]]; then
     sudo systemctl restart mosquitto
 else
     # Raw mode for running (e.g.) in a docker container
-    mosquitto -c $ETC_DIR/mosquitto.conf > mosquitto.log 2>&1 &
+    mosquitto -c $ETC_DIR/mosquitto.conf > $MOSQUITTO_LOG 2>&1 &
     MOSQUITTO_PID=$!
     echo Waiting 10s for background mosquitto to start...
     sleep 10
-    [[ -d /proc/$MOSQUITTO_PID ]] || (cat mosquitto.log && fail starting mosquitto)
+    [[ -d /proc/$MOSQUITTO_PID ]] || (cat $MOSQUITTO_LOG && fail starting mosquitto)
     echo Completed mosquitto startup.
 fi
 

bin/start_udmis

@@ -22,8 +22,7 @@ fi
 
 bin/container udmis prep --no-check || true
 
-mkdir -p out
-LOGFILE=out/udmis.log
+LOGFILE=/tmp/udmis.log
 date > $LOGFILE
 
 export ETCD_CLUSTER=localhost
@@ -32,7 +31,7 @@ export SSL_SECRETS_DIR=/etc/mosquitto/certs
 UDMIS_DIR=udmis
 [[ -d $UDMIS_DIR ]] || UDMIS_DIR=..
 
-sudo PATH=$PATH -E $UDMIS_DIR/bin/run $UDMIS_DIR/etc/local_pod.json > $LOGFILE 2>&1 &
+sudo PATH=$PATH -E $UDMIS_DIR/bin/run $UDMIS_DIR/etc/local_pod.json >> $LOGFILE 2>&1 &
 
 PID=$!
 

bin/test_mosquitto

@@ -26,7 +26,7 @@ CLNT_USER=kiwi
 CLNT_PASS=possum
 CLNT_ID=$registry_id/client
 
-sudo chmod a+r $CERT_DIR/rsa_private.*
+sudo chmod a+r $CERT_DIR/rsa_private.* || true
 bin/keygen CERT $CLIENT_DIR
 
 CLIENT_OPTS="-i $CLNT_ID -u $CLNT_USER -P $CLNT_PASS --cafile $CA_CERT --cert $CLIENT_DIR/rsa_private.crt --key $CLIENT_DIR/rsa_private.pem"
@@ -51,7 +51,7 @@ mosquitto_pub $CLIENT_OPTS -t $topic -m "Hello Racket"
 sleep 1
 
 echo Checking received message
-fgrep Racket out/mosquitto.sub
+fgrep Racket out/mosquitto.sub || fail did not find expected message
 
 hash_pass=$(sha256sum < $site_path/devices/$device_id/rsa_private.pkcs8)
 dev_pass=${hash_pass:0:8}
@@ -113,7 +113,7 @@ echo Received messages:
 cat out/mosquitto.sub | cut -c -120
 
 echo Checking received end message
-fgrep Rudolph out/mosquitto.sub
+fgrep Rudolph out/mosquitto.sub || fail did not find expected message
 
 echo Checking messages for matching serial no $serial_no
 received_no=$(fgrep operational out/mosquitto.sub | head -n 1 | sed -E 's/^[^{]+//' | jq -r .system.serial_no)

bin/toolrun

@@ -11,7 +11,7 @@ OUT_DIR=$UDMI_ROOT/out
 mkdir -p $OUT_DIR
 rm -f $OUT_DIR/$util_name.log
 
-$UDMI_ROOT/validator/bin/build
+[[ ! -d $UDMI_ROOT/validator/src ]] || $UDMI_ROOT/validator/bin/build
 
 JAVA_CLASS=com.google.daq.mqtt.util.Dispatcher
 

bin/validator

@@ -79,5 +79,3 @@ cat $site_file
 echo Running tools version $udmi_version
 
 $UDMI_ROOT/validator/bin/validate $site_file $* 2>&1 | tee $OUT_DIR/validator.log
-
-bin/check_version