Initial commit

.github/workflows/audit.yml

@@ -0,0 +1,13 @@
+name: Security Audit
+
+on: [push, pull_request]
+
+jobs:
+  audit:
+    name: Audit
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: rustsec/[email protected]
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -0,0 +1,127 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  lint:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        rust: [stable]
+        os: [ubuntu-latest]
+
+    steps:
+      - name: Install system packages
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libusb-1.0-0-dev libftdi1-dev libudev-dev
+
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Cache target
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ matrix.os }}-cargo--${{ matrix.rust }}-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Install toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy, rustfmt
+          toolchain: ${{ matrix.rust }}
+
+      - name: Clippy
+        run: cargo clippy --all-features -- -W clippy::all -D warnings
+
+      - name: Format
+        run: cargo fmt --all -- --check
+
+      - name: Doc Generation
+        run: cargo doc --bins --examples --all-features --no-deps
+
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        rust: [stable]
+        os: [ubuntu-latest, windows-latest, macos-latest]
+
+    steps:
+      - name: Install system packages (Linux)
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libusb-1.0-0-dev libftdi1-dev libudev-dev
+
+      - name: Install system packages (Windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: |
+          vcpkg install libftdi1:x64-windows libusb:x64-windows
+
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Cache target
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ matrix.os }}-cargo--${{ matrix.rust }}-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Install toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy
+          toolchain: ${{ matrix.rust }}
+
+      - name: Build debug binary
+        run: cargo build
+
+      - name: Build release binary
+        run: cargo build --release
+
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        rust: [stable]
+        os: [ubuntu-latest, windows-latest, macos-latest]
+
+    steps:
+      - name: Install system packages (Linux)
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libusb-1.0-0-dev libftdi1-dev libudev-dev
+
+      - name: Install system packages (Windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: |
+          vcpkg install libftdi1:x64-windows libusb:x64-windows
+
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Cache target
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ matrix.os }}-cargo--${{ matrix.rust }}-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Install toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy
+          toolchain: ${{ matrix.rust }}
+
+      - name: Test
+        run: cargo test --all-features -- --test-threads=1

.github/workflows/integration_tests.yml

@@ -0,0 +1,205 @@
+name: Integration Tests
+on:
+  push:
+  schedule:
+    # Run every week at 12am PST (8am UTC) Saturday
+    - cron: '0 8 * * SAT'
+  workflow_dispatch:
+    inputs:
+      package_channel:
+        description: 'Modality package channel (stable/nightly)'
+        required: true
+        default: 'stable'
+        type: choice
+        options:
+        - stable
+        - nightly
+
+env:
+  MODALITY_URL: "http://localhost:14181/v1"
+  MODALITY_WORKSPACE: "ci-tests"
+  RENODE_CI_MODE: YES
+
+jobs:
+  demo:
+    name: Run, collect, and test
+    timeout-minutes: 30
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Print Environment
+        run: |
+          echo "GITHUB_WORKFLOW=$GITHUB_WORKFLOW"
+          echo "GITHUB_RUN_ID=$GITHUB_RUN_ID"
+          echo "GITHUB_RUN_NUMBER=$GITHUB_RUN_NUMBER"
+          echo "GITHUB_JOB=$GITHUB_JOB"
+          echo "GITHUB_ACTION=$GITHUB_ACTION"
+          echo "GITHUB_ACTOR=$GITHUB_ACTOR"
+          echo "GITHUB_REF=$GITHUB_REF"
+          echo "GITHUB_SHA=$GITHUB_SHA"
+          echo "PACKAGE_CHANNEL=${{github.event.inputs.package_channel}}"
+          docker --version
+
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Login to github container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.AUXON_GHCR_TOKEN }}
+
+      - name: Install system packages
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libusb-1.0-0-dev libftdi1-dev libudev-dev bridge-utils gcc-arm-none-eabi binutils-arm-none-eabi
+
+      - name: Install renode (latest)
+        run: |
+          cd /opt
+          sudo wget --quiet --output-document renode.tar.gz https://github.com/renode/renode/releases/download/v1.15.0/renode-1.15.0.linux-portable.tar.gz
+          sudo mkdir renode
+          sudo tar xf renode.tar.gz -C renode --strip-components 1
+          sudo pip install -r /opt/renode/tests/requirements.txt
+          echo "PATH=/opt/renode:${PATH}" >> $GITHUB_ENV
+
+      - name: Install toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy, rustfmt
+          toolchain: stable
+          targets: thumbv7em-none-eabihf
+
+      - name: Install Rust tools
+        run: cargo install renode-run flip-link
+
+      - name: Build release binary
+        run: cargo build --release
+
+      - name: Install Auxon client packages
+        env:
+          DEBIAN_FRONTEND: noninteractive
+        run: |
+          wget --no-verbose --quiet https://get.keygen.sh/auxon-io/auxon.deb -O /tmp/auxon.deb
+          sudo apt-get install -y /tmp/auxon.deb
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends modality-client conform deviant
+
+      - name: Start Modality server
+        run: |
+          image="ghcr.io/auxoncorp/modalityd-nightly:latest"
+          if [ "${{github.event.inputs.package_channel}}" == "stable" ]; then
+            image="ghcr.io/auxoncorp/modalityd:latest"
+          fi
+          mkdir -p modalityd_data
+          docker run \
+            --name modalityd \
+            --network=host \
+            -v "$(pwd)/modalityd_data:/data-dir" \
+            -e MODALITY_ACCEPT_EULA=Y \
+            -e MODALITY_LICENSE_KEY="${{secrets.MODALITY_LICENSE_KEY}}" \
+            -e NO_TLS=Y \
+            -d --rm \
+            ${image}
+
+          curl --retry-max-time 30 --retry 10 --retry-connrefused ${{env.MODALITY_URL}}/alive
+          docker logs modalityd
+
+      - name: Setup initial Modality configuration
+        run: |
+          modality --version
+          modality config --modalityd ${{env.MODALITY_URL}}
+          modality user create --use ci
+          echo "MODALITY_AUTH_TOKEN=$(modality user auth-token)" >> $GITHUB_ENV
+          modality workspace create --use ${{env.MODALITY_WORKSPACE}} test_system/config/workspace.toml
+
+      - name: Update reflector plugins
+        run: |
+          sudo mkdir -p /usr/lib/modality-reflector-plugins/importers
+          sudo mkdir -p /usr/lib/modality-reflector-plugins/collectors
+          sudo cp target/release/modality-defmt-importer /usr/lib/modality-reflector-plugins/importers/
+          sudo cp target/release/modality-defmt-rtt-collector /usr/lib/modality-reflector-plugins/collectors/
+
+          # NOTE: We're going to run the importer as a collector
+          # this will change once we add import support to our reflector docker image
+          sudo cp target/release/modality-defmt-importer /usr/lib/modality-reflector-plugins/collectors/
+
+      - name: Create specs
+        run: |
+          conform spec create --file test_system/specs/tests.speqtr tests
+
+      - name: Build test system
+        working-directory: test_system
+        run: cargo build --release
+
+      - name: Run test system
+        working-directory: test_system
+        run: cargo run --release
+
+      - name: Import data
+        working-directory: test_system
+        run: |
+          image="ghcr.io/auxoncorp/modality-reflector-nightly:latest"
+          if [ "${{github.event.inputs.package_channel}}" == "stable" ]; then
+            image="ghcr.io/auxoncorp/modality-reflector:latest"
+          fi
+          docker run \
+            --name reflector \
+            --network=host \
+            -e RUST_LOG="modality_defmt=trace" \
+            -e INGEST_PROTOCOL_PARENT_URL="modality-ingest://127.0.0.1" \
+            -e MUTATION_PROTOCOL_PARENT_URL="modality-mutation://127.0.0.1" \
+            -v "/usr/lib/modality-reflector-plugins:/usr/lib/modality-reflector-plugins" \
+            -v "$(pwd)/config/reflector-config.toml:/reflector-config.toml" \
+            -v "$(pwd)/target/thumbv7em-none-eabihf/release/atsamd-rtic-firmware:/atsamd-rtic-firmware" \
+            -v "/tmp/rtt_log.bin:/rtt_log.bin" \
+            -e MODALITY_AUTH_TOKEN="${{env.MODALITY_AUTH_TOKEN}}" \
+            -e REFLECTOR_OPTS="--config /reflector-config.toml --collector defmt" \
+            -d --rm \
+            ${image}
+
+            # NOTE: this will go away once we switch to reflector import
+            sleep 2
+            modality wait-until --deadline '4m' 'panic@* aggregate count() = 1'
+            docker container stop reflector
+
+      - name: Inspect data
+        env:
+          MODALITY_AUTH_TOKEN: ${{env.MODALITY_AUTH_TOKEN}}
+        run: |
+          modality workspace list
+          modality workspace inspect ${{env.MODALITY_WORKSPACE}}
+          modality segment list
+
+      - name: Evaluate specs
+        env:
+          MODALITY_AUTH_TOKEN: ${{env.MODALITY_AUTH_TOKEN}}
+        run: |
+          conform spec eval --name tests
+
+      - name: Export Data
+        if: always()
+        env:
+          MODALITY_AUTH_TOKEN: ${{env.MODALITY_AUTH_TOKEN}}
+        run: |
+          datetime=$(date +'%Y-%m-%dT%H-%M-%SZ')
+          tarfile=defmt_test_system_modality_data_${datetime}.tar.gz
+          echo TARBALL_NAME=${tarfile} >> $GITHUB_ENV
+
+          time docker container stop -s SIGTERM -t 20 modalityd
+          modality user auth-token > modalityd_data/user_auth_token
+          cp /tmp/rtt_log.bin modalityd_data/
+          sudo rm -rf modalityd_data/index_invalidation_events
+          sudo rm -f modalityd_data/lcache
+          tar czvf ${tarfile} modalityd_data
+
+      - name: Upload Modality data
+        if: always()
+        uses: actions/upload-artifact@v4
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          name: ${{ env.TARBALL_NAME }}
+          path: ${{ env.TARBALL_NAME }}
+          retention-days: 7
+          if-no-files-found: error