diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 9628cce113..8be16b260d 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -184,7 +184,7 @@ jobs: run: more out/pubber.log* pubber/out/*.json | cat - name: udmis log if: ${{ !cancelled() }} - run: cat out/udmis*.log || true + run: cat /tmp/udmis.log || true - name: itemized test post-process if: ${{ !cancelled() }} run: egrep ' test .* after .*s ' out/sequencer.log-* | tee out/timing_itemized.out @@ -204,7 +204,7 @@ jobs: name: Baseline Tests runs-on: ubuntu-latest needs: pretest - timeout-minutes: 10 + timeout-minutes: 15 if: vars.TARGET_PROJECT != '' env: TARGET_PROJECT: ${{ vars.TARGET_PROJECT }} @@ -280,7 +280,7 @@ jobs: sudo more /etc/mosquitto/* /etc/mosquitto/conf.d/* || true - name: udmis log if: ${{ !cancelled() }} - run: cat out/udmis*.log + run: cat /tmp/udmis.log - name: mosquitto logs if: ${{ !cancelled() }} run: cat /var/log/mosquitto/mosquitto.log || true @@ -298,7 +298,7 @@ jobs: name: Endpoint Redirection runs-on: ubuntu-latest needs: [ pretest, baseline ] - timeout-minutes: 10 + timeout-minutes: 15 if: vars.TARGET_PROJECT != '' env: TARGET_PROJECT: ${{ vars.TARGET_PROJECT }} diff --git a/.gitignore b/.gitignore index 0753e3b6d4..a564f419b4 100644 --- a/.gitignore +++ b/.gitignore @@ -25,8 +25,6 @@ credentials.json /udmis/var/ /udmis/profile/ /udmis/.idea/libraries/ -/udmis/udmi_bin/ -/udmis/udmi_etc/ /selfie/build/ /pubber/build/ /pubber/out/ diff --git a/bin/container b/bin/container index ed46f43115..7e90e78f31 100755 --- a/bin/container +++ b/bin/container @@ -22,7 +22,7 @@ git_repo=${git_prefix#*.com:} GCP_PROJECT=$(gcloud config get project) REPOSITORY=ghcr.io/${git_repo} -TEMPLATES=$(cd etc; ls k8s_*.yaml) +TEMPLATES=$(cd etc; ls k8s_*.yaml) || true NOCHECK= [[ ${1:-} == "--no-check" ]] && NOCHECK=1 && shift @@ -91,7 +91,7 @@ if [[ -n $prep ]]; then rm -rf var tmp && mkdir -p var tmp cp -f etc/*.json var/ || true - cp -a ../schema tmp/ + cp -a ../schema build/ bin/build diff --git a/bin/docker_udmis b/bin/docker_udmis deleted file mode 100755 index a142b5412c..0000000000 --- a/bin/docker_udmis +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -e - -UDMI_ROOT=$(dirname $0)/.. -source $UDMI_ROOT/etc/shell_common.sh - -[[ $# == 1 ]] || usage site_model - -site_model=$(realpath $1) -shift - -cd $UDMI_ROOT - -docker rm udmis || true - -[[ -f $site_model/cloud_iot_config.json ]] || fail missing $site_model/cloud_iot_config.json - -IMAGE_TAG=ghcr.io/faucetsdn/udmi:latest -docker inspect -f '{{.Id}}' --type=image udmis:latest && IMAGE_TAG=udmis:latest - -cmd="docker run -d --name udmis -p 8883:8883 \ - -v $site_model:/root/site \ - -v $PWD/var/etcd:/root/default.etcd \ - -v $PWD/var/mosquitto:/etc/mosquitto \ - $IMAGE_TAG udmi/bin/start_local block site/ //mqtt/localhost" - -echo exec: $cmd -$cmd - -echo Waiting 30s for container startup... -sleep 30 - -echo :::::::::::::: docker logs udmis -docker logs udmis - -echo :::::::::::::: docker ps -docker ps - -echo :::::::::::::: diff --git a/bin/keygen b/bin/keygen index 4d53f9f5ed..76902e3f05 100755 --- a/bin/keygen +++ b/bin/keygen @@ -21,12 +21,19 @@ else SUBJ=client fi +TARGET_HOST=localhost +if [[ $type =~ ^CERT/ || $type =~ ^CA/ ]]; then + TARGET_HOST=${type#*/} + type=${type%/*} + echo Generating $type with altname $TARGET_HOST +fi + CA_CRT=$CA_DIR/ca.crt CA_SRL=$CA_DIR/ca.srl CA_KEY=$CA_DIR/rsa_private.pem OPTS_509=/tmp/x509opts.txt -echo "subjectAltName=DNS:localhost, IP:127.0.0.1" > $OPTS_509 +echo "subjectAltName=DNS:${TARGET_HOST}, IP:127.0.0.1, DNS:localhost" > $OPTS_509 x509opts="-days 3650 -extfile $OPTS_509" if [[ $type == CA ]]; then diff --git a/bin/mosquctl_log b/bin/mosquctl_log index 7686e16b49..b692e3868d 100755 --- a/bin/mosquctl_log +++ b/bin/mosquctl_log @@ -1,12 +1,11 @@ #!/bin/bash -e UDMI_ROOT=$(dirname $0)/.. +source $UDMI_ROOT/etc/shell_common.sh cd $UDMI_ROOT LOG_FILE=/var/log/mosquitto/mosquitto.log -source $UDMI_ROOT/etc/shell_common.sh - if [[ $# != 1 ]]; then echo Usage: $0 client_id_prefix false diff --git a/bin/pubber b/bin/pubber index 3904538ddc..dc43669362 100755 --- a/bin/pubber +++ b/bin/pubber @@ -61,7 +61,7 @@ echo Cleaning output directory $UDMI_ROOT/pubber/out/$serial_no rm -rf $UDMI_ROOT/pubber/out/$serial_no echo Building pubber... -$UDMI_ROOT/pubber/bin/build +[[ ! -d $UDMI_ROOT/pubber/src ]] || $UDMI_ROOT/pubber/bin/build echo Running tools version $udmi_version @@ -172,6 +172,13 @@ fi jq --slurpfile endpoint $ENDPOINT_TMP '.endpoint = $endpoint[0]' $tmp_config | sponge $tmp_config +if [[ -f ${site_model:-}/reflector/ca.crt && -n ${project_target:-} ]]; then + echo Checking for signed device certificate... + certs=(${site_model}/devices/${target_id}/*.crt) || true + [[ -f $certs ]] || $UDMI_ROOT/bin/keygen CERT/${project_target} ${site_model}/devices/${target_id}/ + ls -l ${site_model}/devices/${target_id}/*.crt +fi + # Run in the background to force new process group and get PID $UDMI_ROOT/pubber/bin/run $tmp_config & PUB_PID=$! diff --git a/bin/sequencer b/bin/sequencer index 6f0753c694..ee5c473a5a 100755 --- a/bin/sequencer +++ b/bin/sequencer @@ -172,7 +172,7 @@ fi cat $SEQUENCER_CONFIG -validator/bin/build +[[ ! -d $UDMI_ROOT/validator/src ]] || $UDMI_ROOT/validator/bin/build echo $JAVA_CMD result=0 @@ -192,5 +192,3 @@ fgrep RESULT $site_model/out/devices/$device_id/RESULT.log | \ more $site_model/out/devices/$device_id/RESULT.log $SCHEMA_OUT | cat bin/sequencer_report $site_model $device_id - -bin/check_version diff --git a/bin/setup_base b/bin/setup_base index 002cf8fca0..8705c17656 100755 --- a/bin/setup_base +++ b/bin/setup_base @@ -3,7 +3,16 @@ ROOT_DIR=$(realpath $(dirname $0)/..) cd $ROOT_DIR -sudo apt-get install -y moreutils expect mosquitto mosquitto-clients +if [[ -n $(which apt-get) ]]; then + update_cmd="apt-get install -y" + extra_packages="" +else + update_cmd="apk add" + # Some extra packages are necessary for alpine based docker builds. + extra_packages="linux-headers g++ gcc" +fi + +sudo $update_cmd $extra_packages moreutils expect python3-dev mosquitto mosquitto-clients python3 --version python3 -m venv venv diff --git a/bin/setup_ca b/bin/setup_ca index 5b10d7aa10..4116a0130d 100755 --- a/bin/setup_ca +++ b/bin/setup_ca @@ -6,14 +6,15 @@ source $UDMI_ROOT/etc/shell_common.sh CERT_DIR=/etc/mosquitto/certs/ sudo rm -f $CERT_DIR/* -[[ $# != 1 ]] && fail $0 site_model +[[ $# != 2 ]] && fail $0 site_model hostname site_model=$(realpath $1) +target=$2 cd $UDMI_ROOT -bin/keygen CA $site_model/reflector -bin/keygen CERT $site_model/reflector +bin/keygen CA/$target $site_model/reflector +bin/keygen CERT/$target $site_model/reflector echo Importing server certs from $site_model... sudo cp $site_model/reflector/ca.* $CERT_DIR/ @@ -21,7 +22,3 @@ sudo cp $site_model/reflector/rsa_private.* $CERT_DIR/ sudo chgrp -R mosquitto $CERT_DIR sudo chmod 0640 $CERT_DIR/rsa_private.* ls -l $CERT_DIR - - - - diff --git a/bin/start_etcd b/bin/start_etcd index 055c37dc36..57da1488f1 100755 --- a/bin/start_etcd +++ b/bin/start_etcd @@ -2,6 +2,7 @@ IMAGE=quay.io/coreos/etcd:v3.5.13 BINDIR=/tmp/etcd +ETCD_LOG=/tmp/etcd.log echo pwd: $(pwd) @@ -11,11 +12,11 @@ $BINDIR/etcd -version $BINDIR/etcd -listen-client-urls=http://0.0.0.0:2379 \ -advertise-client-urls=http://127.0.0.1:2379 \ - > etcd.log 2>&1 & + > $ETCD_LOG 2>&1 & ETCD_PID=$! -echo Waiting 10s for etcd to start... +echo Waiting 10s for etcd to start, log in $ETCD_LOG sleep 10 -[[ -d /proc/$ETCD_PID ]] || (cat etcd.log && fail starting etcd) +[[ -d /proc/$ETCD_PID ]] || (cat $ETCD_LOG && echo error starting etcd && false) echo Completed etcd startup. diff --git a/bin/start_local b/bin/start_local index 03bf8988b1..6c85ca6071 100755 --- a/bin/start_local +++ b/bin/start_local @@ -25,27 +25,34 @@ if [[ -d var/ && ! -d etc/ ]]; then fi cd $UDMI_ROOT +UDMIS_LOG=/tmp/udmis.log +mkdir -p out +date > $UDMIS_LOG if [[ ! $project_spec =~ ^//mqtt/ ]]; then - echo Not a local setup, doing nothing! + echo Not a local setup, doing nothing! || tee -a $UDMIS_LOG exit 0 fi +project_target=${project_spec##*/} site_config=$site_model/cloud_iot_config.json registry_id=$(jq -r .registry_id $site_config)${UDMI_REGISTRY_SUFFIX:-} +echo Starting etcd... | tee -a $UDMIS_LOG bin/start_etcd source $UDMI_ROOT/etc/mosquitto_ctrl.sh mkdir -p $CERT_DIR -bin/setup_ca $site_model +echo Starting mosquitto on server $project_target | tee -a $UDMIS_LOG +bin/setup_ca $site_model $project_target bin/start_mosquitto $MOSQUITTO_CTRL deleteClient $SERV_USER $MOSQUITTO_CTRL createClient $SERV_USER -p $SERV_PASS # No client_id to allow multiple backend connections. $MOSQUITTO_CTRL addClientRole $SERV_USER service +echo Starting initializing site $site_model | tee -a $UDMIS_LOG bin/mosquctl_site $site_model if [[ -n ${UDMI_ALT_REGISTRY:-} ]]; then @@ -55,6 +62,7 @@ fi # Global access to logs for connection tracking by UDMIS. sudo chmod a+r /var/log/mosquitto/mosquitto.log +echo Starting udmis proper... | tee -a $UDMIS_LOG bin/start_udmis echo Done with local server setup. diff --git a/bin/start_mosquitto b/bin/start_mosquitto index 9a4de3142e..75dabbec7c 100755 --- a/bin/start_mosquitto +++ b/bin/start_mosquitto @@ -15,8 +15,9 @@ GROUP=mosquitto UDMI_FILE=$ETC_DIR/conf.d/udmi.conf PASS_FILE=$ETC_DIR/mosquitto.passwd DYN_FILE=$ETC_DIR/dynamic_security.json +MOSQUITTO_LOG=/var/log/mosquitto/mosquitto.log -mkdir -p /var/log/mosquitto/ +sudo mkdir -p $(dirname $MOSQUITTO_LOG) if [[ ! -f $UDMI_FILE ]]; then echo Creating new $UDMI_FILE from template... @@ -51,11 +52,11 @@ if [[ -n $(which systemctl) ]]; then sudo systemctl restart mosquitto else # Raw mode for running (e.g.) in a docker container - mosquitto -c $ETC_DIR/mosquitto.conf > mosquitto.log 2>&1 & + mosquitto -c $ETC_DIR/mosquitto.conf > $MOSQUITTO_LOG 2>&1 & MOSQUITTO_PID=$! echo Waiting 10s for background mosquitto to start... sleep 10 - [[ -d /proc/$MOSQUITTO_PID ]] || (cat mosquitto.log && fail starting mosquitto) + [[ -d /proc/$MOSQUITTO_PID ]] || (cat $MOSQUITTO_LOG && fail starting mosquitto) echo Completed mosquitto startup. fi diff --git a/bin/start_udmis b/bin/start_udmis index 295ab2a125..57d16edfe5 100755 --- a/bin/start_udmis +++ b/bin/start_udmis @@ -22,8 +22,7 @@ fi bin/container udmis prep --no-check || true -mkdir -p out -LOGFILE=out/udmis.log +LOGFILE=/tmp/udmis.log date > $LOGFILE export ETCD_CLUSTER=localhost @@ -32,7 +31,7 @@ export SSL_SECRETS_DIR=/etc/mosquitto/certs UDMIS_DIR=udmis [[ -d $UDMIS_DIR ]] || UDMIS_DIR=.. -sudo PATH=$PATH -E $UDMIS_DIR/bin/run $UDMIS_DIR/etc/local_pod.json > $LOGFILE 2>&1 & +sudo PATH=$PATH -E $UDMIS_DIR/bin/run $UDMIS_DIR/etc/local_pod.json >> $LOGFILE 2>&1 & PID=$! diff --git a/bin/test_mosquitto b/bin/test_mosquitto index b6913a0da9..f6581fa274 100755 --- a/bin/test_mosquitto +++ b/bin/test_mosquitto @@ -26,7 +26,7 @@ CLNT_USER=kiwi CLNT_PASS=possum CLNT_ID=$registry_id/client -sudo chmod a+r $CERT_DIR/rsa_private.* +sudo chmod a+r $CERT_DIR/rsa_private.* || true bin/keygen CERT $CLIENT_DIR CLIENT_OPTS="-i $CLNT_ID -u $CLNT_USER -P $CLNT_PASS --cafile $CA_CERT --cert $CLIENT_DIR/rsa_private.crt --key $CLIENT_DIR/rsa_private.pem" @@ -51,7 +51,7 @@ mosquitto_pub $CLIENT_OPTS -t $topic -m "Hello Racket" sleep 1 echo Checking received message -fgrep Racket out/mosquitto.sub +fgrep Racket out/mosquitto.sub || fail did not find expected message hash_pass=$(sha256sum < $site_path/devices/$device_id/rsa_private.pkcs8) dev_pass=${hash_pass:0:8} @@ -113,7 +113,7 @@ echo Received messages: cat out/mosquitto.sub | cut -c -120 echo Checking received end message -fgrep Rudolph out/mosquitto.sub +fgrep Rudolph out/mosquitto.sub || fail did not find expected message echo Checking messages for matching serial no $serial_no received_no=$(fgrep operational out/mosquitto.sub | head -n 1 | sed -E 's/^[^{]+//' | jq -r .system.serial_no) diff --git a/bin/toolrun b/bin/toolrun index b81112fd59..b6070ed0da 100755 --- a/bin/toolrun +++ b/bin/toolrun @@ -11,7 +11,7 @@ OUT_DIR=$UDMI_ROOT/out mkdir -p $OUT_DIR rm -f $OUT_DIR/$util_name.log -$UDMI_ROOT/validator/bin/build +[[ ! -d $UDMI_ROOT/validator/src ]] || $UDMI_ROOT/validator/bin/build JAVA_CLASS=com.google.daq.mqtt.util.Dispatcher diff --git a/bin/validator b/bin/validator index 02bedde55e..29bf78bd63 100755 --- a/bin/validator +++ b/bin/validator @@ -79,5 +79,3 @@ cat $site_file echo Running tools version $udmi_version $UDMI_ROOT/validator/bin/validate $site_file $* 2>&1 | tee $OUT_DIR/validator.log - -bin/check_version diff --git a/docs/udmis/local_docker.md b/docs/udmis/local_docker.md index c1c858ad7a..4275fc2cd8 100644 --- a/docs/udmis/local_docker.md +++ b/docs/udmis/local_docker.md @@ -1,27 +1,102 @@ -[**UDMI**](../../) / [**Docs**](../) / [UDMIS](.) / [Local Docker](#) +[**UDMI**](../../) / [**Docs**](../) / [**UDMIS**](.) / [Local Docker](#) # Local docker UDMIS setup and execution -* Identify a site model and parameters - * For default reference udmi site model: - * `bin/clone_model` - * `site_model=sites/udmi_site_model` - * `device_id=AHU-1` - * `serial_no=21874812` -* Run the standard docker image: - * `bin/docker_udmis ${site_model}` -* Persistent DB files are mapped locally - * `ls -l var/etcd var/mosquitto` -* Register your site with the server - * `bin/registrar ${site_model} //mqtt/localhost` -* Optionally test with pubber instance - * `sudo bin/keygen CERT ${site_model}/devices/${device_id}/` - * `bin/pubber ${site_model} //mqtt/localhost ${device_id} ${serial_no}` - -# Container build - -General notes on how to build/push the upstream docker image. - -* `bin/container udmis push` -* `docker tag udmis:latest ghcr.io/faucetsdn/udmi:latest` -* `docker push ghcr.io/faucetsdn/udmi:latest` +This document describes how to setup and run all the UDMI components locally using docker, so no +specific install required. All the commands here should be direct cut-and-paste into a suitable +shell window. + +_Note:_ This is still in 'alpha' form. It works, but there's likely going to be some kinks to work out +(likely to do with authentication, because authentication is annoying). + +## Configuration Setup + +These setup variables can be customized for any particular setup. It's recommended to first +get everything working with the built-in _pubber_ as shown, and then try with a real device after! +``` +site_model=udmi_site_model +device_id=AHU-1 +serial_no=8127324 +``` + +## Environment Setup + +One-time setup pieces to setup docker and a test site model, if necessary. +``` +docker inspect -f ok udminet || docker network create udminet --subnet 192.168.99.0/24 +[[ ${site_model} != udmi_site_model ]] || git clone https://github.com/faucetsdn/udmi_site_model.git +``` + +## UDMIS Container Startup + +Starts the background UDMIS locally. The output of this command is uninteresting (assuming +it works), so see the next section for some sample diagnostic commands. It will take some time +(less than 30s) to get up and going before the other commands will work. +``` +docker run -d --rm --net udminet --name udmis -p 8883:8883 \ + -v $(realpath $site_model):/root/site \ + -v $PWD/var/tmp:/tmp \ + -v $PWD/var/etcd:/root/udmi/default.etcd \ + -v $PWD/var/mosquitto:/etc/mosquitto \ + ghcr.io/faucetsdn/udmi:udmis-latest udmi/bin/start_local block site/ //mqtt/udmis +``` + +## UDMIS Startup Diagnostics + +Some sample commands to inspect and diagnose what's going on with the UDMIS startup. See +the [sample UDMIS output](udmis_output.md) for what this might look like. Real diagnosing +will require a bit more investigation than shown here, but if you see the same output +then you can assume that the system is working! +``` +docker logs udmis 2>&1 | fgrep udmis +fgrep pod_ready var/tmp/udmis.log +ls -l var/tmp/pod_ready.txt +``` + +## Registrar Run + +After startup, the site model needs to be registered as per standard UDMI practice. This only +needs to be done once per site model (or after any significant changes). The +[sample registrar output](registrar_output.md) shows what a successful run looks like. +``` +docker run --rm --net udminet --name registrar -v $(realpath $site_model):/root/site \ + ghcr.io/faucetsdn/udmi:validator-latest bin/registrar site/ //mqtt/udmis +``` + +## Pubber Instance + +For initial install testing, it's recommended to try first with the standard _pubber_ utility. +See the [sample pubber output](pubber_output.md) for the beginning of what this run looks like. + +``` +docker run -d --rm --net udminet --name pubber -v $(realpath $site_model):/root/site \ + ghcr.io/faucetsdn/udmi:pubber-latest bin/pubber site/ //mqtt/udmis ${device_id} ${serial_no} +``` + +## Sequencer Testing + +Sequencer can be run directly as per normal too. See the [sample sequencer output](sequencer_output.md) +for what the beginning of a successful run looks like. +``` +docker run --rm --net udminet --name sequencer -v $(realpath $site_model):/root/site \ + ghcr.io/faucetsdn/udmi:validator-latest bin/sequencer site/ //mqtt/udmis ${device_id} ${serial_no} +``` + +The resulting output can be extracted from the site model, See the [sample report output](report_output.md) +for what this should look like. +``` +head ${site_model}/out/devices/${device_id}/results.md +``` + +# Container Build + +For development purposes, the following command will build (and push) the requisite docker images. This +requires a full UDMI install and appropriate permissions (to push). +``` +for image in udmis validator pubber; do + docker rmi ghcr.io/faucetsdn/udmi:$image-latest; \ + bin/container $image push \ + && docker tag $image:latest ghcr.io/faucetsdn/udmi:$image-latest \ + && docker push ghcr.io/faucetsdn/udmi:$image-latest +done +``` diff --git a/docs/udmis/pubber_output.md b/docs/udmis/pubber_output.md new file mode 100644 index 0000000000..b3a18081dd --- /dev/null +++ b/docs/udmis/pubber_output.md @@ -0,0 +1,154 @@ +[**UDMI**](../../) / [**Docs**](../) / [**UDMIS**](.) / [Pubber Output](#) + +Sample command to run pubber in a docker container: +``` +docker run -d --rm --net udminet --name pubber -v $(realpath $site_model):/root/site \ + ghcr.io/faucetsdn/udmi:pubber-latest bin/pubber site/ //mqtt/udmis ${device_id} ${serial_no} +``` + +`docker logs pubber` sample output: +``` +Unable to find image 'ghcr.io/faucetsdn/udmi:pubber-latest' locally +pubber-latest: Pulling from faucetsdn/udmi +ec99f8b99825: Already exists +efbe3ea3836c: Pull complete +f22a8d45822e: Pull complete +d1b89b9bf41b: Pull complete +170b8f92ca0f: Pull complete +25c526f5b443: Pull complete +Digest: sha256:91f4637277af1133074ba200f7ade2437cd21d060ad60544175ca5d9bc0af806 +Status: Downloaded newer image for ghcr.io/faucetsdn/udmi:pubber-latest +fatal: not a git repository (or any of the parent directories): .git +fatal: not a git repository (or any of the parent directories): .git +Constructing pubber config from command line args. +Cleaning output directory /root/pubber/out/8127324 +Building pubber... +Running tools version unknown +Attached to gateway null +Target is AHU-1 +Extracting hashed password from key file /root/site/devices/AHU-1/rsa_private.pkcs8 +Checking for signed device certificate... +Generating CERT with altname udmis +Generating self-signed cert from CA defined in /root/site/reflector +Generating cert for device keys in /root/site/devices/AHU-1 +Certificate request self-signature ok +subject=CN=client +Done with keygen. +-rw-r--r-- 1 root root 1123 Jul 19 04:21 /root/site/devices/AHU-1/rsa_private.crt +Original noPersist is null +java -Dorg.slf4j.simpleLogger.showThreadName=false -jar /root/pubber/build/libs/pubber-1.0-SNAPSHOT-all.jar /tmp/pubber_config.json +Waiting for pubber pid 38 to complete... +INFO daq.pubber.Pubber - Device start time is 2024-07-19T04:21:03Z +INFO daq.pubber.Pubber - State update defer -1721362861116ms +INFO daq.pubber.Pubber - State update defer -1721362861117ms +INFO daq.pubber.Pubber - Using addresses from default interface eth0 +INFO daq.pubber.Pubber - Family ipv4 address is 192.168.99.3 +INFO daq.pubber.Pubber - Family ether address is 02:42:c0:a8:63:03 +INFO daq.pubber.Pubber - State update defer -1721362861137ms +INFO daq.pubber.Pubber - State update defer -1721362861137ms +INFO daq.pubber.Pubber - Using addresses from default interface eth0 +INFO daq.pubber.Pubber - Family ipv4 address is 192.168.99.3 +INFO daq.pubber.Pubber - Family ether address is 02:42:c0:a8:63:03 +INFO daq.pubber.Pubber - State update defer -1721362861139ms +INFO daq.pubber.Pubber - State update defer -1721362861139ms +INFO daq.pubber.Pubber - Using addresses from default interface eth0 +INFO daq.pubber.Pubber - Family ipv4 address is 192.168.99.3 +INFO daq.pubber.Pubber - Family ether address is 02:42:c0:a8:63:03 +INFO daq.pubber.Pubber - State update defer -1721362861140ms +INFO daq.pubber.Pubber - State update defer -1721362861140ms +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Writing pubber feature file to /root/site/out/pubber_features.json +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Configured with auth_type RS256 +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z State update defer -1721362861212ms +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Resetting persistent store /root/site/out/devices/AHU-1/persistent_data.json +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Loading endpoint into persistent data from configuration +WARN daq.pubber.Pubber - 2024-07-19T04:21:03Z Updating persistent store: +{ + "endpoint" : { + "protocol" : "mqtt", + "transport" : "ssl", + "hostname" : "udmis", + "client_id" : "/r/ZZ-TRI-FECTA/d/AHU-1", + "topic_prefix" : "/r/ZZ-TRI-FECTA/d/AHU-1", + "auth_provider" : { + "basic" : { + "username" : "/r/ZZ-TRI-FECTA/d/AHU-1", + "password" : "38269d11" + } + } + }, + "restart_count" : 1 +} +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Starting pubber AHU-1, serial 8127324, mac null, gateway null, options noPersist +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z State update defer -1721362861218ms +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Loading device key bytes from /root/site/devices/AHU-1/rsa_private.pkcs8 +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z CA cert file: /root/site/reflector/ca.crt +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Device cert file: /root/site/devices/AHU-1/rsa_private.crt +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Private key file: /root/site/devices/AHU-1/rsa_private.pem +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Password sha256 6b5344e1 +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Publishing dirty state block +WARN daq.pubber.Pubber - 2024-07-19T04:21:03Z Starting connection 1721362863295 with 10 +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z State update defer -1721362861295ms +WARN daq.pubber.Pubber - 2024-07-19T04:21:03Z Creating new config latch for AHU-1 +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Update state 2024-07-19T04:21:03Z last_config 1970-01-01T00:00:00Z +WARN daq.pubber.Pubber - 2024-07-19T04:21:03Z Dropping state update until config received... +INFO daq.pubber.MqttPublisher - Creating new client to ssl://udmis:8883 as /r/ZZ-TRI-FECTA/d/AHU-1 +INFO daq.pubber.MqttPublisher - Auth using username /r/ZZ-TRI-FECTA/d/AHU-1 +INFO daq.pubber.MqttPublisher - Attempting connection to /r/ZZ-TRI-FECTA/d/AHU-1 +INFO daq.pubber.MqttPublisher - Subscribed to mqtt topic /r/ZZ-TRI-FECTA/d/AHU-1/config (qos 1) +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Log DEBUG* system.config.receive success 2024-07-19T04:21:03Z +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z State update defer -1721362861885ms +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Update state 2024-07-19T04:21:03Z last_config 1970-01-01T00:00:00Z +WARN daq.pubber.Pubber - 2024-07-19T04:21:03Z Dropping state update until config received... +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Log DEBUG* system.config.parse success 2024-07-19T04:21:03Z +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z State update defer -1721362861916ms +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Device AHU-1 config handler +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Update state 2024-07-19T04:21:03Z last_config 1970-01-01T00:00:00Z +WARN daq.pubber.Pubber - 2024-07-19T04:21:03Z Dropping state update until config received... +INFO daq.pubber.MqttPublisher - Subscribed to mqtt topic /r/ZZ-TRI-FECTA/d/AHU-1/errors (qos 0) +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Connection complete. +WARN daq.pubber.Pubber - 2024-07-19T04:21:03Z Start waiting 10s for config latch for AHU-1 +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Config update AHU-1: +{ + "timestamp" : "2024-07-19T04:20:13Z", + "version" : "1.4.2-248-gfeb3779c", + "system" : { + "min_loglevel" : 300, + "metrics_rate_sec" : 10, + "operation" : { } + }, + "localnet" : { + "families" : { + "ipv4" : { }, + "vendor" : { }, + "ether" : { } + } + }, + "pointset" : { + "points" : { + "filter_alarm_pressure_status" : { + "ref" : "BV11.present_value", + "units" : "No-units" + }, + "filter_differential_pressure_setpoint" : { + "units" : "Bars", + "set_value" : 98 + }, + "filter_differential_pressure_sensor" : { + "ref" : "AV12.present_value", + "units" : "Degrees-Celsius" + } + } + } +} +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z 2024-07-19T04:21:03Z received config 2024-07-19T04:20:13Z +WARN daq.pubber.Pubber - 2024-07-19T04:21:03Z Starting AHU-1 PointsetManager sender with delay 10s +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Restoring unknown point filter_alarm_pressure_status +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Restoring unknown point filter_differential_pressure_setpoint +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Restoring unknown point filter_differential_pressure_sensor +WARN daq.pubber.Pubber - 2024-07-19T04:21:03Z Starting AHU-1 SystemManager sender with delay 10s +INFO daq.pubber.MqttPublisher - Sending message to /r/ZZ-TRI-FECTA/d/AHU-1/events/system +WARN daq.pubber.Pubber - 2024-07-19T04:21:03Z Starting AHU-1 Pubber sender with delay 10s +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z State update defer -1721362861989ms +INFO daq.pubber.Pubber - 2024-07-19T04:21:03Z Update state 2024-07-19T04:21:03Z last_config 2024-07-19T04:20:13Z +``` +_...output continues indefinitely..._ diff --git a/docs/udmis/registrar_output.md b/docs/udmis/registrar_output.md new file mode 100644 index 0000000000..e2cbe95325 --- /dev/null +++ b/docs/udmis/registrar_output.md @@ -0,0 +1,136 @@ +[**UDMI**](../../) / [**Docs**](../) / [**UDMIS**](.) / [Registrar Output](#) + +Command for running registrar in a docker container: +``` +docker run --rm --net udminet --name registrar -v $(realpath $site_model):/root/site \ + ghcr.io/faucetsdn/udmi:validator-latest bin/registrar site/ //mqtt/udmis +``` + +Sample output: +``` +Unable to find image 'ghcr.io/faucetsdn/udmi:validator-latest' locally +validator-latest: Pulling from faucetsdn/udmi +ec99f8b99825: Already exists +8abafddf1ae3: Pull complete +f9eaaae8e4dc: Pull complete +5a2589c2150b: Pull complete +08b6c630112a: Pull complete +0a5909cd013c: Pull complete +eec9bf907dcf: Pull complete +93a71d99e987: Pull complete +31d56e6becc1: Pull complete +Digest: sha256:af97be37493df358aec09650715ed9f11c649cfb7bebcee0529ee1ecf00b8dca +Status: Downloaded newer image for ghcr.io/faucetsdn/udmi:validator-latest +fatal: not a git repository (or any of the parent directories): .git +fatal: not a git repository (or any of the parent directories): .git +udmi version unknown +date: unrecognized option: iso=s +BusyBox v1.36.1 (2024-06-10 07:11:47 UTC) multi-call binary. + +Usage: date [OPTIONS] [+FMT] [[-s] TIME] + +Display time (using +FMT), or set time + + -u Work in UTC (don't convert to local time) + [-s] TIME Set time to TIME + -d TIME Display TIME, not 'now' + -D FMT FMT (strptime format) for -s/-d TIME conversion + -r FILE Display last modification time of FILE + -R Output RFC-2822 date + -I[SPEC] Output ISO-8601 date + SPEC=date (default), hours, minutes, seconds or ns + +Recognized TIME formats: + @seconds_since_1970 + hh:mm[:ss] + [YYYY.]MM.DD-hh:mm[:ss] + YYYY-MM-DD hh:mm[:ss] + [[[[[YY]YY]MM]DD]hh]mm[.ss] + 'date TIME' form accepts MMDDhhmm[[YY]YY][.ss] instead +starting run at +java -cp /root/validator/build/libs/validator-1.0-SNAPSHOT-all.jar com.google.daq.mqtt.util.Dispatcher registrar site/ //mqtt/udmis +Writing reconciled configuration file to /root/out/registrar_conf.json +Using reflector iot client +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - Loaded key /root/site/reflector/rsa_private.pkcs8 as sha256 42a8a8f1287aa775 +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - ZZ-TRI-FECTA token expiration sec 3600 +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - Using hash-key username/password /r/UDMI-REFLECT/d/ZZ-TRI-FECTA 42a8a8f1 +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - CA cert file: /root/site/reflector/ca.crt +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - Device cert file: /root/site/reflector/rsa_private.crt +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - Private key file: /root/site/reflector/rsa_private.pem +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - Password sha256 0049165a +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - ZZ-TRI-FECTA creating client /r/UDMI-REFLECT/d/ZZ-TRI-FECTA on ssl://udmis:8883 +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - ZZ-TRI-FECTA creating new auth token for audience udmis +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - Using hash-key username/password /r/UDMI-REFLECT/d/ZZ-TRI-FECTA 42a8a8f1 +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - ZZ-TRI-FECTA connecting to mqtt server ssl://udmis:8883 +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - Subscribing with qos 1 to topic /r/UDMI-REFLECT/d/ZZ-TRI-FECTA/config +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - Subscribing with qos 1 to topic /r/UDMI-REFLECT/d/ZZ-TRI-FECTA/errors +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - Subscribing with qos 0 to topic /r/UDMI-REFLECT/d/ZZ-TRI-FECTA/commands/# +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - ZZ-TRI-FECTA done with setup connection +Subscribed to mqtt/udmis/us-central1/UDMI-REFLECT/ZZ-TRI-FECTA +Starting initial UDMI setup process +Ignoring initial config received timeout (config likely empty) +Setting state version unknown timestamp 2024-07-19T04:20:12Z +UDMI setting reflectorState: { + "version" : "unknown", + "udmi" : { + "setup" : { + "transaction_id" : "RC:d22034.0001" + } + }, + "timestamp" : "2024-07-19T04:20:12Z" +} +UDMI received reflectorConfig: { + "last_state" : "2024-07-19T04:20:12Z", + "setup" : { + "hostname" : "3d18e014852d", + "functions_min" : 13, + "functions_max" : 13, + "udmi_version" : "1.4.2-248-gfeb3779c", + "udmi_ref" : "ghcr.io/grafnu/udmi:udmis-gfeb3779cf", + "built_at" : "2024-07-19T04:12:00Z", + "built_by" : "testuser@testuser", + "transaction_id" : "RC:d22034.0001" + } +} +UDMI matching against expected state timestamp 2024-07-19T04:20:12Z +UDMI version mismatch: unknown +UDMI functions support versions 13:13 (required 13) +Created service for project udmis +Working with project udmis registry us-central1/ZZ-TRI-FECTA +Loading site_defaults.json +Finished loading 4 local devices. +Writing normalized /root/site/devices/GAT-123/out/metadata_norm.json +Writing normalized /root/site/devices/AHU-22/out/metadata_norm.json +Writing normalized /root/site/devices/AHU-1/out/metadata_norm.json +Writing normalized /root/site/devices/SNS-4/out/metadata_norm.json +Fetching devices from registry ZZ-TRI-FECTA... +Fetched 0 device models from cloud registry +Processing 4 new devices... +Waiting for device processing... +Waiting 61s for 4 tasks to complete... +Processed SNS-4 (4/4) in 0.045s (add) +Processed AHU-22 (3/4) in 0.046s (add) +Processed GAT-123 (1/4) in 0.091s (add) +Processed AHU-1 (2/4) in 0.160s (add) +Processed 4 (skipped 0) devices in 0.805s, 0.201s/d +Updating 0 existing devices... +Waiting for device processing... +Processed 0 (skipped 0) devices in 0.000s, 0.000s/d +Finished registering 4/4 devices. +Binding devices to GAT-123, already bound: +Binding 2 unbound devices to 1 gateways... +Waiting for device binding... +Binding AHU-22 to GAT-123 (1/2) +Waiting 61s for 2 tasks to complete... +Binding SNS-4 to GAT-123 (2/2) +Finished binding gateways in 0.175 +Updating site/devices/AHU-1/out/errors.map + +Summary: + Device Clean: 3 + Device Envelope: 1 + Device Validating: 1 +Out of 4 total. +Registration summary available in /root/site/out/registration_summary.json +Registration summary available in /root/site/out/registration_summary.csv +``` diff --git a/docs/udmis/report_output.md b/docs/udmis/report_output.md new file mode 100644 index 0000000000..47b449948c --- /dev/null +++ b/docs/udmis/report_output.md @@ -0,0 +1,20 @@ +[**UDMI**](../../) / [**Docs**](../) / [**UDMIS**](.) / [Report Output](#) + +The resulting report can be found in the site model under the target device: +``` +head ${site_model}/out/devices/${device_id}/results.md +``` + +With an expected output of something like: +``` +# AHU-1 + +- Start 2024-07-19T16:47:31Z +- End: 2024-07-19T17:04:28Z +- Status: Run completed + +## Device Identification + +| Device | AHU-1 | +|---|---| +``` diff --git a/docs/udmis/sequencer_output.md b/docs/udmis/sequencer_output.md new file mode 100644 index 0000000000..9b3351b376 --- /dev/null +++ b/docs/udmis/sequencer_output.md @@ -0,0 +1,165 @@ +[**UDMI**](../../) / [**Docs**](../) / [**UDMIS**](.) / [Sequencer Output](#) + +Command for running sequencer in a docker container: +``` +docker run --rm --net udminet --name sequencer -v $(realpath $site_model):/root/site \ + ghcr.io/faucetsdn/udmi:validator-latest bin/sequencer site/ //mqtt/udmis ${device_id} ${serial_no} +``` + +Sample output: +``` +fatal: not a git repository (or any of the parent directories): .git +fatal: not a git repository (or any of the parent directories): .git +Using site model directory /root/site +Writing config to /tmp/sequencer_config.json: +{ + "iot_provider": "mqtt", + "project_id": "udmis", + "bridge_host": null, + "udmi_namespace": null, + "site_model": "/root/site", + "device_id": "AHU-1", + "alt_registry": null, + "registry_suffix": null, + "shard_count": null, + "shard_index": null, + "serial_no": "8127324", + "log_level": "INFO", + "min_stage": "PREVIEW", + "udmi_version": "unknown", + "udmi_root": "/root", + "reflector_endpoint": null, + "sequences": null, + "key_file": "/root/site/reflector/rsa_private.pkcs8" +} +java -cp validator/build/libs/validator-1.0-SNAPSHOT-all.jar com.google.daq.mqtt.sequencer.SequenceRunner +Target sequence classes: + com.google.daq.mqtt.sequencer.sequences.BlobsetSequences + com.google.daq.mqtt.sequencer.sequences.ConfigSequences + com.google.daq.mqtt.sequencer.sequences.DiscoverySequences + com.google.daq.mqtt.sequencer.sequences.DiscoverySequences$1 + com.google.daq.mqtt.sequencer.sequences.GatewaySequences + com.google.daq.mqtt.sequencer.sequences.LocalnetSequences + com.google.daq.mqtt.sequencer.sequences.PointsetSequences + com.google.daq.mqtt.sequencer.sequences.ProxiedSequences + com.google.daq.mqtt.sequencer.sequences.SystemSequences + com.google.daq.mqtt.sequencer.sequences.WritebackSequences +Reading config file /tmp/sequencer_config.json +Found target methods: endpoint_connection_retry, endpoint_connection_success_reconnect, endpoint_connection_success_alternate, endpoint_connection_error, endpoint_redirect_and_restart, endpoint_failure_and_restart +Running target com.google.daq.mqtt.sequencer.sequences.BlobsetSequences#endpoint_connection_retry +Running target com.google.daq.mqtt.sequencer.sequences.BlobsetSequences#endpoint_connection_success_reconnect +Running target com.google.daq.mqtt.sequencer.sequences.BlobsetSequences#endpoint_connection_success_alternate +Running target com.google.daq.mqtt.sequencer.sequences.BlobsetSequences#endpoint_connection_error +Running target com.google.daq.mqtt.sequencer.sequences.BlobsetSequences#endpoint_redirect_and_restart +Running target com.google.daq.mqtt.sequencer.sequences.BlobsetSequences#endpoint_failure_and_restart +Checking for modified metadata file /root/site/out/devices/AHU-1/metadata_mod.json +Reading device metadata file /root/site/devices/AHU-1/metadata.json +Writing results to /root/site/out/devices/AHU-1/RESULT.log +Loading reflector key file from /root/site/reflector/rsa_private.pkcs8 +Validating against device AHU-1 serial 8127324 +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - Loaded key /root/site/reflector/rsa_private.pkcs8 as sha256 42a8a8f1287aa775 +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - ZZ-TRI-FECTA token expiration sec 3600 +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - Using hash-key username/password /r/UDMI-REFLECT/d/ZZ-TRI-FECTA 42a8a8f1 +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - CA cert file: /root/site/reflector/ca.crt +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - Device cert file: /root/site/reflector/rsa_private.crt +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - Private key file: /root/site/reflector/rsa_private.pem +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - Password sha256 0049165a +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - ZZ-TRI-FECTA creating client /r/UDMI-REFLECT/d/ZZ-TRI-FECTA on ssl://udmis:8883 +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - ZZ-TRI-FECTA creating new auth token for audience udmis +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - Using hash-key username/password /r/UDMI-REFLECT/d/ZZ-TRI-FECTA 42a8a8f1 +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - ZZ-TRI-FECTA connecting to mqtt server ssl://udmis:8883 +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - Subscribing with qos 1 to topic /r/UDMI-REFLECT/d/ZZ-TRI-FECTA/config +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - Subscribing with qos 1 to topic /r/UDMI-REFLECT/d/ZZ-TRI-FECTA/errors +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - Subscribing with qos 0 to topic /r/UDMI-REFLECT/d/ZZ-TRI-FECTA/commands/# +[main] INFO com.google.bos.iot.core.proxy.MqttPublisher - ZZ-TRI-FECTA done with setup connection +Subscribed to mqtt/udmis/us-central1/UDMI-REFLECT/ZZ-TRI-FECTA +Starting initial UDMI setup process +Setting state version unknown timestamp 2024-07-19T04:22:41Z +UDMI setting reflectorState: { + "version" : "unknown", + "udmi" : { + "setup" : { + "transaction_id" : "RC:8cc954.0001" + } + }, + "timestamp" : "2024-07-19T04:22:41Z" +} +UDMI received reflectorConfig: { + "last_state" : "2024-07-19T04:22:41Z", + "setup" : { + "hostname" : "3d18e014852d", + "functions_min" : 13, + "functions_max" : 13, + "udmi_version" : "1.4.2-248-gfeb3779c", + "udmi_ref" : "ghcr.io/grafnu/udmi:udmis-gfeb3779cf", + "built_at" : "2024-07-19T04:12:00Z", + "built_by" : "testuser@testuser", + "transaction_id" : "RC:8cc954.0001" + } +} +UDMI matching against expected state timestamp 2024-07-19T04:22:41Z +UDMI version mismatch: unknown +UDMI functions support versions 13:13 (required 13) +No alternate registry configured, disabling +2024-07-19T04:22:41Z INFO Cleaning test output dir /root/site/out/devices/AHU-1/tests/endpoint_connection_retry +2024-07-19T04:22:41Z INFO Cleaning test output dir /root/site/out/devices/AHU-1/tests/endpoint_connection_retry +2024-07-19T04:22:41Z NOTICE starting test endpoint_connection_retry ################################ +2024-07-19T04:22:41Z INFO Stage start waiting for starting test wrapper at 0s +2024-07-19T04:22:50Z INFO Stage start waiting for config sync at 9s +2024-07-19T04:22:50Z INFO Initial state #001: { +2024-07-19T04:22:50Z INFO Initial state #001: "timestamp" : "2024-07-19T04:22:50Z", +2024-07-19T04:22:50Z INFO Initial state #001: "version" : "1.5.1", +2024-07-19T04:22:50Z INFO Initial state #001: "system" : { +2024-07-19T04:22:50Z INFO Initial state #001: "last_config" : "2024-07-19T04:20:13Z", +2024-07-19T04:22:50Z INFO Initial state #001: "operation" : { +2024-07-19T04:22:50Z INFO Initial state #001: "operational" : true, +2024-07-19T04:22:50Z INFO Initial state #001: "last_start" : "2024-07-19T04:21:03Z", +2024-07-19T04:22:50Z INFO Initial state #001: "restart_count" : 1, +2024-07-19T04:22:50Z INFO Initial state #001: "mode" : "initial" +2024-07-19T04:22:50Z INFO Initial state #001: }, +2024-07-19T04:22:50Z INFO Initial state #001: "serial_no" : "8127324", +2024-07-19T04:22:50Z INFO Initial state #001: "hardware" : { +2024-07-19T04:22:50Z INFO Initial state #001: "make" : "BOS", +2024-07-19T04:22:50Z INFO Initial state #001: "model" : "pubber" +2024-07-19T04:22:50Z INFO Initial state #001: }, +2024-07-19T04:22:50Z INFO Initial state #001: "software" : { +2024-07-19T04:22:50Z INFO Initial state #001: "firmware" : "v1" +2024-07-19T04:22:50Z INFO Initial state #001: }, +2024-07-19T04:22:50Z INFO Initial state #001: "status" : { +2024-07-19T04:22:50Z INFO Initial state #001: "message" : "success", +2024-07-19T04:22:50Z INFO Initial state #001: "category" : "system.config.parse", +2024-07-19T04:22:50Z INFO Initial state #001: "timestamp" : "2024-07-19T04:22:50Z", +2024-07-19T04:22:50Z INFO Initial state #001: "level" : 100 +2024-07-19T04:22:50Z INFO Initial state #001: } +2024-07-19T04:22:50Z INFO Initial state #001: }, +2024-07-19T04:22:50Z INFO Initial state #001: "localnet" : { +2024-07-19T04:22:50Z INFO Initial state #001: "families" : { +2024-07-19T04:22:50Z INFO Initial state #001: "vendor" : { +2024-07-19T04:22:50Z INFO Initial state #001: "addr" : "28179023" +2024-07-19T04:22:50Z INFO Initial state #001: }, +2024-07-19T04:22:50Z INFO Initial state #001: "ether" : { +2024-07-19T04:22:50Z INFO Initial state #001: "addr" : "02:42:c0:a8:63:03" +2024-07-19T04:22:50Z INFO Initial state #001: }, +2024-07-19T04:22:50Z INFO Initial state #001: "ipv4" : { +2024-07-19T04:22:50Z INFO Initial state #001: "addr" : "192.168.99.3" +``` + +...this will go on for quite some time, until ultimately it should end with something like: +``` +Missing reference file: validator/sequences/pointset_publish/sequence.md +Missing reference file: validator/sequences/pointset_publish_interval/sequence.md +Missing reference file: validator/sequences/pointset_remove_point/sequence.md +Missing reference file: validator/sequences/pointset_request_extraneous/sequence.md +Missing reference file: validator/sequences/broken_config/sequence.md +Missing reference file: validator/sequences/config_logging/sequence.md +Missing reference file: validator/sequences/device_config_acked/sequence.md +Missing reference file: validator/sequences/extra_config/sequence.md +Missing reference file: validator/sequences/family_ether_addr/sequence.md +Missing reference file: validator/sequences/family_ipv4_addr/sequence.md +Missing reference file: validator/sequences/family_ipv6_addr/sequence.md +Missing reference file: validator/sequences/state_make_model/sequence.md +Missing reference file: validator/sequences/state_software/sequence.md +Missing reference file: validator/sequences/system_last_update/sequence.md +Missing reference file: validator/sequences/valid_serial_no/sequence.md +Report saved to: /root/site/out/devices/AHU-1/results.md +``` diff --git a/docs/udmis/udmis_output.md b/docs/udmis/udmis_output.md new file mode 100644 index 0000000000..69352ffa61 --- /dev/null +++ b/docs/udmis/udmis_output.md @@ -0,0 +1,28 @@ +[**UDMI**](../../) / [**Docs**](../) / [**UDMIS**](.) / [UDMIS Output](#) + +Commands to test if the UDMIS container is running properly: +``` +docker logs udmis 2>&1 | fgrep udmis +fgrep pod_ready var/tmp/udmis.log +ls -l var/tmp/pod_ready.txt +``` + +Sample output: +``` +export TARGET_PROJECT=//mqtt/udmis +bin/start_etcd: line 9: udmis/bin/etcdctl: No such file or directory +Starting mosquitto on server udmis +Generating CA with altname udmis +Generating CERT with altname udmis +Starting udmis proper... +bin/container: line 14: cd: /root/udmi/udmis: No such file or directory +Waiting for udmis startup 29... +Waiting for udmis startup 28... +Waiting for udmis startup 27... +Waiting for udmis startup 26... +Waiting for udmis startup 25... +::::::::: tail /tmp/udmis.log +udmis running in the background, pid 198 log in /tmp/udmis.log +2024-07-19T04:18:40Z xxxxxxxx N: UdmiServicePod Finished activation of container components, created /tmp/pod_ready.txt +-rw-r--r-- 1 root root 0 Jul 18 21:18 var/tmp/pod_ready.txt +``` diff --git a/etc/requirements.txt b/etc/requirements.txt index 66dcf68874..0a1f5123ca 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -14,8 +14,8 @@ google-auth-oauthlib==0.4.6 google-cloud-pubsub==2.12.1 googleapis-common-protos==1.54.0 grpc-google-iam-v1==0.12.3 -grpcio==1.53.2 -grpcio-status==1.48.2 +grpcio==1.65.1 +grpcio-status==1.65.1 htmlmin==0.1.12 httplib2==0.20.4 idna==3.7 @@ -31,15 +31,15 @@ mccabe==0.7.0 mypy-extensions==0.4.3 oauthlib==3.2.2 platformdirs==2.6.0 -proto-plus==1.22.2 -protobuf==3.19.5 +proto-plus==1.24.0 +protobuf==5.26.1 pyasn1==0.4.8 pyasn1-modules==0.2.8 Pygments==2.15.0 pylint==2.15.9 pyparsing==3.0.7 pytz==2021.3 -PyYAML==6.0 +PyYAML==6.0.1 requests==2.32.0 requests-oauthlib==1.3.1 rsa==4.8 diff --git a/pubber/Dockerfile.pubber b/pubber/Dockerfile.pubber new file mode 100644 index 0000000000..cb61302072 --- /dev/null +++ b/pubber/Dockerfile.pubber @@ -0,0 +1,15 @@ +FROM alpine:latest + +WORKDIR /root + +RUN apk add openjdk17 bash sudo gcompat curl jq python3 git moreutils openssl + +ADD build/libs pubber/build/libs/ + +ADD bin/ pubber/bin/ + +ADD build/udmi_etc/ etc/ + +ADD build/udmi_bin/ bin/ + +CMD ["/root/pubber/bin/run"] diff --git a/pubber/bin/build b/pubber/bin/build index 4dfadf203d..c17b937908 100755 --- a/pubber/bin/build +++ b/pubber/bin/build @@ -18,6 +18,14 @@ done rundir=$(dirname $0) cd $rundir/.. +echo Cleaning build/ +rm -rf build/ +mkdir build/ + +echo Mirroring udmi files... +cp -a ../bin build/udmi_bin/ +cp -a ../etc build/udmi_etc/ + newest=$(ls -rt `find src/ ../gencode/java -type f` | tail -n 1) jarfile=build/libs/pubber-1.0-SNAPSHOT-all.jar if [[ -z $check && -z $covg && -f $jarfile && $jarfile -nt $newest ]]; then @@ -26,8 +34,6 @@ if [[ -z $check && -z $covg && -f $jarfile && $jarfile -nt $newest ]]; then fi echo Building pubber in $PWD - -rm -rf build ./gradlew shadow $check $covg ls -l $jarfile diff --git a/pubber/etc/Dockerfile.pubber b/pubber/etc/Dockerfile.puller similarity index 100% rename from pubber/etc/Dockerfile.pubber rename to pubber/etc/Dockerfile.puller diff --git a/udmis/.idea/runConfigurations/UDMIS_Remote.xml b/udmis/.idea/runConfigurations/UDMIS_Remote.xml new file mode 100644 index 0000000000..b4935babbb --- /dev/null +++ b/udmis/.idea/runConfigurations/UDMIS_Remote.xml @@ -0,0 +1,16 @@ + + + + + \ No newline at end of file diff --git a/udmis/Dockerfile.udmis b/udmis/Dockerfile.udmis index 6a61524309..1c46ca3bea 100644 --- a/udmis/Dockerfile.udmis +++ b/udmis/Dockerfile.udmis @@ -2,11 +2,11 @@ FROM alpine:latest WORKDIR /root -RUN apk add bash openjdk17 gcompat curl jq sudo git mosquitto mosquitto-clients openssl +RUN apk add bash openjdk17 gcompat curl jq sudo git mosquitto mosquitto-clients openssl coreutils -ADD bin/ bin/ +ADD build/udmi_bin/ udmi/bin/ -ADD udmi_bin/ udmi/bin/ +ADD bin/ bin/ RUN bin/etcdctl version @@ -17,7 +17,7 @@ ADD build/libs/udmis-1.0-SNAPSHOT-all.jar build/libs/udmis-1.0-SNAPSHOT-all.jar ADD var/ var/ -ADD udmi_etc/ udmi/etc/ +ADD build/udmi_etc/ udmi/etc/ ENV CLEARBLADE_CONFIGURATION=/udmi/clearblade.json ENV SHUNT_CONFIGURATION=/udmi/shunt-config.json diff --git a/udmis/bin/build b/udmis/bin/build index 7c090f6131..b14d33700e 100755 --- a/udmis/bin/build +++ b/udmis/bin/build @@ -19,12 +19,14 @@ ROOT=$(dirname $0)/.. BASE=.. cd $ROOT -echo Mirroring udim/bin to udmi_bin... -rm -rf udmi_bin/ -cp -a ../bin/ udmi_bin/ -echo Mirroring udmi/etc to udmi_etc... -rm -rf udmi_etc/ -cp -a ../etc udmi_etc/ +echo Cleaning $ROOT/build/ +rm -rf build +mkdir build + +echo Mirroring udim files... +cp -a ../bin/ build/udmi_bin/ +cp -a ../etc/ build/udmi_etc/ +cp -a ../schema build/schema/ echo Copying default mosquitto config files... mkdir -p var/ @@ -43,7 +45,6 @@ export JAVA_HOME=$JAVA_HOME_11_X64 echo Building udmis in $PWD -rm -rf build ./gradlew shadow $check $covg $* ls -l $jarfile diff --git a/udmis/bin/run b/udmis/bin/run index 96d9510153..31b5dc6815 100755 --- a/udmis/bin/run +++ b/udmis/bin/run @@ -21,6 +21,7 @@ more /etc/os-release $POD_CONFIG | cat java -version POD_JAR=build/libs/udmis-1.0-SNAPSHOT-all.jar +DEBUG_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 gcloud_project=$(gcloud config get project || true) if [[ -z $GCP_PROJECT && -z $gcloud_project ]]; then @@ -78,4 +79,4 @@ env echo # Run with exec to replace shell so java receives SIGTERM signal. -exec java -jar $POD_JAR $POD_CONFIG +exec java $DEBUG_OPTS -jar $POD_JAR $POD_CONFIG diff --git a/udmis/src/main/java/com/google/bos/udmi/service/core/DistributorPipe.java b/udmis/src/main/java/com/google/bos/udmi/service/core/DistributorPipe.java index f2c6cb7df7..9685ff9731 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/core/DistributorPipe.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/core/DistributorPipe.java @@ -66,6 +66,10 @@ protected void defaultHandler(Object message) { */ public void publish(Envelope rawEnvelope, Object message, String source) { try { + if (!isEnabled()) { + warn("Not publishing to disabled distributor"); + return; + } Envelope envelope = deepCopy(rawEnvelope); String routeId = getRouteId(source); debug("Distributing %s for %s/%s as %s", message.getClass().getSimpleName(), diff --git a/udmis/src/main/java/com/google/bos/udmi/service/core/ReflectProcessor.java b/udmis/src/main/java/com/google/bos/udmi/service/core/ReflectProcessor.java index 6069ee17dc..1df3776b6c 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/core/ReflectProcessor.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/core/ReflectProcessor.java @@ -295,7 +295,7 @@ private void reflectStateHandler(Envelope envelope, UdmiState toolState) { envelope.gatewayId = envelope.source; ifNotNullThen(distributor, d -> catchToElse(() -> d.publish(envelope, toolState, containerId), - e -> error("Error handling awareness: " + friendlyStackTrace(e)))); + e -> error("Error handling update: %s %s", friendlyStackTrace(e), envelope.transactionId))); updateAwareness(envelope, toolState); UdmiConfig udmiConfig = UdmiServicePod.getUdmiConfig(toolState); diff --git a/validator/.idea/jarRepositories.xml b/validator/.idea/jarRepositories.xml index 1da9a525d2..5ecc50afc0 100644 --- a/validator/.idea/jarRepositories.xml +++ b/validator/.idea/jarRepositories.xml @@ -31,5 +31,10 @@