Merge branch 'nasa-main'

.github/workflows/unit-test.yml

@@ -7,20 +7,29 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
 
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python
-      uses: actions/setup-python@v4
+    - uses: actions/checkout@v4
+    - name: Set up OnAIR test environment
+      uses: conda-incubator/setup-miniconda@v3
       with:
-        python-version: 3.11.2
+        activate-environment: onair
+        environment-file: environment.yml
+        python-version: ${{ matrix.python-version }}
+        auto-activate-base: false
     - name: Install dependencies
+      shell: bash -l {0}
       run: |
-        python -m pip install --upgrade pip
-        if [ -f requirements_pip.txt ]; then pip install -r requirements_pip.txt; fi
+        conda info
+        conda list
     - name: Test with pytest
+      shell: bash -l {0}
       run: python -m coverage run --branch --source=onair,plugins -m pytest ./test/
     - name: Coverage report
+      shell: bash -l {0}
       run: coverage report --skip-empty
     - name: Upload coverage reports to Codecov
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4

Dockerfile

@@ -39,12 +39,7 @@ RUN \
 
 # OnAIR Dependencies
 RUN \
-  apt-get install -y python3.9 && \
-  apt-get install -y python3.9-dev && \
-  apt-get install -y python3-pip
-
-# Add new packages to install here to prevent re-running previous instructions
-
+  apt-get install -y wget
 
 # Ensure that all packages are up to date after new packages have been added above
 RUN \
@@ -56,16 +51,26 @@ RUN \
 RUN adduser onair_dev sudo
 RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 
-# Make OnAir requirements file accessible by onair_dev user
-COPY requirements_pip.txt /home/onair_dev/requirements_onair.txt
-RUN chown onair_dev /home/onair_dev/requirements_onair.txt
-
 USER onair_dev
 
-# Python stuff is being installed for the local user
-ENV PATH="${PATH}:/home/onair_dev/.local/bin"
+# Install miniconda
+ENV CONDA_DIR /home/onair_dev/conda
+RUN \
+  mkdir -p $CONDA_DIR && \
+  wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \
+  bash ~/miniconda.sh -b -u -p $CONDA_DIR && \
+  rm -rf ~/miniconda.sh
+ENV PATH=$CONDA_DIR/bin:$PATH
 
-# Install OnAIR deps
-RUN python3.9 -m pip install --upgrade pip setuptools wheel
-RUN python3.9 -m pip install --user -r /home/onair_dev/requirements_onair.txt
+# Make OnAir requirements file accessible by onair_dev user
+COPY environment.yml /home/onair_dev/environment.yml
+RUN \
+  . $CONDA_DIR/etc/profile.d/conda.sh && \
+  conda init bash && \
+  . ~/.bashrc && \
+  conda env create -f /home/onair_dev/environment.yml && \
+  conda activate onair
 
+# Make sure that the onair conda environment is loaded
+RUN \
+  echo "conda activate onair" >> ~/.bashrc

README.md

@@ -12,7 +12,111 @@ It is intended to explore research concepts in autonomous operations in a simula
 
 Create a conda environment with the necessary packages
 
-    conda create --name onair --file requirements_pip.txt
+    conda env create -f environment.yml
+
+## Redis example
+
+Using the redis_adapter.py as the DataSource, telemetry can be received through multiple Redis channels and inserted into the full data frame.
+
+### OnAIR config file (.ini)
+
+The redis_example.ini uses a very basic setup:
+ - meta : redis_example_CONFIG.json
+ - parser : onair/data_handling/redis_adapter.py
+ - plugins : one of each type, all of them 'generic' that do nothing
+
+### Telemetry config file (.json)
+
+The telemetry file defines the subscribed channels, data frame and subsystems.
+ - subscriptions : defines channel names where telemetry will be received
+ - order : designation of where each 'channel.telemetry_item' is to be put in full data frame (the data header)
+ - subsystems : data for each specific telemetry item (descriptions only for redis example)
+
+
+### Receipt of telemetry
+
+The Redis adapter expects any published telemetry on a channel to include:
+ - time
+ - every telemetry_item as described under "order" as 'channel.telemetry_item'
+
+All messages sent must be json format (key to value) and will warn when it is not then discard the message (outputting what was received first). Keys should match the required telemetry_item names with the addition of "time." Values should be floats.
+
+### Running the example
+
+If not already running, start a Redis server on 'localhost', port:6379 (typical defaults)
+```
+redis-server
+```
+
+Start up OnAIR with the redis_example.ini file:
+```
+python driver.py onair/config/example.ini
+```
+You should see:
+```
+Redis Adapter ignoring file
+
+---- Redis adapter connecting to server...
+
+---- ... connected!
+
+---- Subscribing to channel: state_0
+
+---- Subscribing to channel: state_1
+
+---- Subscribing to channel: state_2
+
+---- Redis adapter: channel 'state_0' received message type: subscribe.
+
+---- Redis adapter: channel 'state_1' received message type: subscribe.
+
+---- Redis adapter: channel 'state_2' received message type: subscribe.
+
+***************************************************
+************    SIMULATION STARTED     ************
+***************************************************
+```
+
+In another process run the experimental publisher:
+```
+python redis-experiment-publisher.py
+```
+This will send telemetry every 2 seconds, one channel at random until all 3 channels have recieved data then repeat for a total of 9 times (all of which can be changed in the file). Its output should be similar to this:
+```
+Published data to state_0, [0, 0.1, 0.2]
+Published data to state_1, [1, 1.1, 1.2]
+Published data to state_2, [2, 2.1, 2.2]
+Completed 1 loops
+Published data to state_2, [3, 3.1, 3.2]
+Published data to state_1, [4, 4.1, 4.2]
+```
+And OnAir should begin receiving data similarly to this:
+```
+--------------------- STEP 1 ---------------------
+
+CURRENT DATA: [0, 0.1, 0.2, '-', '-', '-', '-']
+INTERPRETED SYSTEM STATUS: ---
+
+--------------------- STEP 2 ---------------------
+
+CURRENT DATA: [1, 0.1, 0.2, 1.1, 1.2, '-', '-']
+INTERPRETED SYSTEM STATUS: ---
+
+--------------------- STEP 3 ---------------------
+
+CURRENT DATA: [2, 0.1, 0.2, 1.1, 1.2, 2.1, 2.2]
+INTERPRETED SYSTEM STATUS: ---
+
+--------------------- STEP 4 ---------------------
+
+CURRENT DATA: [3, 0.1, 0.2, 1.1, 1.2, 3.1, 3.2]
+INTERPRETED SYSTEM STATUS: ---
+
+--------------------- STEP 5 ---------------------
+
+CURRENT DATA: [4, 0.1, 0.2, 4.1, 4.2, 3.1, 3.2]
+INTERPRETED SYSTEM STATUS: ---
+```
 
 ## Running unit tests
 
@@ -36,12 +140,12 @@ python driver.py -t
 #### A few optional settings for the driver.py file
 Options that may be added to the driver.py test run. Use these at your own discretion.
 
-`--conftest-seed=###` - set the random values seed for this run  
-`--randomly-seed=###` - set the random order seed for this run  
-`--verbose` or `-v` - set verbosity level, also -vv, -vvv, etc.  
-`-k KEYWORD` - only run tests that match the KEYWORD (see `pytest --help`)  
+`--conftest-seed=###` - set the random values seed for this run
+`--randomly-seed=###` - set the random order seed for this run
+`--verbose` or `-v` - set verbosity level, also -vv, -vvv, etc.
+`-k KEYWORD` - only run tests that match the KEYWORD (see `pytest --help`)
 
-NOTE: Running tests will output results using provided seeds, but each seed is random when not set directly.  
+NOTE: Running tests will output results using provided seeds, but each seed is random when not set directly.
 Example start of test output:
 ```
 Using --conftest-seed=1691289424
@@ -60,33 +164,33 @@ python -m coverage run --branch --source=onair,plugins -m pytest ./test/
 
 #### Command breakdown:
 
-`python -m` - invokes the python runtime on the library following the -m  
-`coverage run` - runs coverage data collection during testing, wrapping itself on the test runner used  
-`--branch` - includes code branching information in the coverage report  
-`--source=onair,plugins` - tells coverage where the code under test exists for reporting line hits  
-`-m pytest` - tells coverage what test runner (framework) to wrap  
-`./test` - run all tests found in this directory and subdirectories  
+`python -m` - invokes the python runtime on the library following the -m
+`coverage run` - runs coverage data collection during testing, wrapping itself on the test runner used
+`--branch` - includes code branching information in the coverage report
+`--source=onair,plugins` - tells coverage where the code under test exists for reporting line hits
+`-m pytest` - tells coverage what test runner (framework) to wrap
+`./test` - run all tests found in this directory and subdirectories
 
 #### A few optional settings for the command line
 Options that may be added to the command line test run. Use these at your own discretion.
 
-`--disable-warnings` - removes the warning reports, but displays count (i.e., 124 passed, 1 warning in 0.65s)  
+`--disable-warnings` - removes the warning reports, but displays count (i.e., 124 passed, 1 warning in 0.65s)
 `-p no:randomly` - ONLY required to stop random order testing IFF pytest-randomly installed
-`--conftest-seed=###` - set the random values seed for this run  
-`--randomly-seed=###` - set the random order seed for this run  
-`--verbose` or `-v` - set verbosity level, also -vv, -vvv, etc.  
-`-k KEYWORD` - only run tests that match the KEYWORD (see `pytest --help`)  
+`--conftest-seed=###` - set the random values seed for this run
+`--randomly-seed=###` - set the random order seed for this run
+`--verbose` or `-v` - set verbosity level, also -vv, -vvv, etc.
+`-k KEYWORD` - only run tests that match the KEYWORD (see `pytest --help`)
 
 NOTE: see note about seeds in driver.py section above
 
 ### To view testing line coverage after run:
 NOTE: you may or may not need the `python -m` to run coverage report or html
 
-`coverage report` - prints basic results in terminal  
+`coverage report` - prints basic results in terminal
 or
 `coverage html` - creates htmlcov/index.html, automatic when using driver.py for testing
 
-then  
+then
 `<browser_here> htmlcov/index.html` - browsable coverage (i.e., `firefox htmlcov/index.html`)
 
 ## License and Copyright

environment.yml

@@ -0,0 +1,14 @@
+name: onair
+channels:
+  - default
+  - conda-forge
+dependencies:
+  - python>=3.8,<3.13
+  - numpy
+  - coverage
+  - pytest
+  - pytest-mock
+  - pytest-randomly
+  - pip
+  - pip:
+    - redis

onair/config/redis_example.ini

@@ -2,16 +2,14 @@
 TelemetryFilePath = onair/data/raw_telemetry_data/data_physics_generation/Errors
 TelemetryFile = 700_crash_to_earth_1.csv
 MetaFilePath = onair/data/telemetry_configs/
-MetaFile = data_physics_generation_CONFIG.json
-
-[DATA_HANDLING]
-DataSourceFile = onair/data_handling/redis_adapter.py
+MetaFile = redis_example_CONFIG.json
+ParserFileName = onair/data_handling/redis_adapter.py
 
 [PLUGINS]
-KnowledgeRepPluginDict = {'generic':'plugins/generic/__init__.py'}
-LearnersPluginDict = {'generic':'plugins/generic/__init__.py'}
-PlannersPluginDict = {'generic':'plugins/generic/__init__.py'}
-ComplexPluginDict = {'generic':'plugins/generic/__init__.py'}
+KnowledgeRepPluginDict = {'knowledge':'plugins/generic/__init__.py'}
+LearnersPluginDict = {'learner':'plugins/generic/__init__.py'}
+PlannersPluginDict = {'planner':'plugins/generic/__init__.py'}
+ComplexPluginDict = {'complex':'plugins/generic/__init__.py'}
 
 [OPTIONS]
 IO_Enabled = true

onair/data/telemetry_configs/redis_example_CONFIG.json

@@ -0,0 +1,41 @@
+{
+  "subsystems": {
+    "STATES": {
+      "time": {
+        "description": "Time of latest receipt of values"
+      },
+      "state_0.x": {
+        "description": "Vehicle 0's current state of x"
+      },
+      "state_0.y": {
+        "description": "Vehicle 0's current state of y"
+      },
+      "state_1.x": {
+        "description": "Vehicle 1's current state of x"
+      },
+      "state_1.y": {
+        "description": "Vehicle 1's current state of y"
+      },
+      "state_2.x": {
+        "description": "Vehicle 2's current state of x"
+      },
+      "state_2.y": {
+        "description": "Vehicle 2's current state of y"
+      }
+    }
+  },
+  "redis_subscriptions": [
+    "state_0",
+    "state_1",
+    "state_2"
+  ],
+  "order": [
+    "time",
+    "state_0.x",
+    "state_0.y",
+    "state_1.x",
+    "state_1.y",
+    "state_2.x",
+    "state_2.y"
+  ]
+}

onair/data_handling/on_air_data_source.py

@@ -8,7 +8,10 @@
 # See "NOSA GSC-19165-1 OnAIR.pdf"
 
 from abc import ABC, abstractmethod
-from .parser_util import * 
+from .parser_util import *
+
+class ConfigKeyError(KeyError):
+    pass
 
 class OnAirDataSource(ABC):
     def __init__(self, data_file, meta_file, ss_breakdown = False):