Merge pull request #820 from projecthorus/testing

v1.7.1 - Bugfixes, RS41 Mainboard type

auto_rx/auto_rx.py

@@ -891,7 +891,10 @@ def main():
     # Start our exporter options
     # Telemetry Logger
     if config["per_sonde_log"]:
-        _logger = TelemetryLogger(log_directory=logging_path)
+        _logger = TelemetryLogger(
+            log_directory=logging_path,
+            save_cal_data=config["save_cal_data"]
+        )
         exporter_objects.append(_logger)
         exporter_functions.append(_logger.add)
 

auto_rx/autorx/__init__.py

@@ -12,7 +12,7 @@
 # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus.
 # PATCH - Small changes, or minor feature additions.
 
-__version__ = "1.7.0"
+__version__ = "1.7.1"
 
 
 # Global Variables

auto_rx/autorx/config.py

@@ -165,6 +165,7 @@ def read_auto_rx_config(filename, no_sdr_test=False):
         "save_raw_hex": False,
         "save_system_log": False,
         "enable_debug_logging": False,
+        "save_cal_data": False,
         # URL for the Habitat DB Server.
         # As of July 2018 we send via sondehub.org, which will allow us to eventually transition away
         # from using the habhub.org tracker, and leave it for use by High-Altitude Balloon Hobbyists.
@@ -775,6 +776,17 @@ def read_auto_rx_config(filename, no_sdr_test=False):
             )
             auto_rx_config["wideband_sondes"] = False
 
+        # 1.7.1 - Save RS41 Calibration Data
+        try:
+            auto_rx_config["save_cal_data"] = config.getboolean(
+                "logging", "save_cal_data"
+            )
+        except:
+            logging.warning(
+                "Config - Missing save_cal_data option (new in v1.7.1), using default (False)"
+            )
+            auto_rx_config["save_cal_data"] = False
+
         # If we are being called as part of a unit test, just return the config now.
         if no_sdr_test:
             return auto_rx_config

auto_rx/autorx/decode.py

@@ -246,6 +246,9 @@ def __init__(
         self.imet_prev_time = None
         self.imet_prev_frame = None
 
+        # Keep a record of which RS41 serials we have uploaded complete subframe data for.
+        self.rs41_subframe_uploads = []
+
         # This will become our decoder thread.
         self.decoder = None
 
@@ -394,7 +397,7 @@ def generate_decoder_command(self):
             if self.save_decode_audio:
                 decode_cmd += f" tee {self.save_decode_audio_path} |"
 
-            decode_cmd += "./rs41mod --ptu2 --json 2>/dev/null"
+            decode_cmd += "./rs41mod --ptu2 --json --jsnsubfrm1 2>/dev/null"
 
         elif self.sonde_type == "RS92":
             # Decoding a RS92 requires either an ephemeris or an almanac file.
@@ -830,7 +833,7 @@ def generate_decoder_command_experimental(self):
                 _baud_rate,
             )
 
-            decode_cmd = f"./rs41mod --ptu2 --json --softin -i {self.raw_file_option} 2>/dev/null"
+            decode_cmd = f"./rs41mod --ptu2 --json --jsnsubfrm1 --softin -i {self.raw_file_option} 2>/dev/null"
 
             # RS41s transmit pulsed beacons - average over the last 2 frames, and use a peak-hold
             demod_stats = FSKDemodStats(averaging_time=2.0, peak_hold=True)
@@ -1602,8 +1605,9 @@ def handle_decoder_line(self, data):
             # which is most likely an Ozone sensor (though could be something different!)
             # We append -Ozone to the sonde type field to indicate this.
             # TODO: Decode device ID from aux field to indicate what the aux payload actually is?
-            if "aux" in _telemetry:
-               _telemetry["type"] += "-Ozone"
+            # 2023-10 - disabled this addition. Can be too misleading. -XDATA now appended on the web interface only.
+            # if "aux" in _telemetry:
+            #    _telemetry["type"] += "-Ozone"
 
             # iMet Specific actions
             if self.sonde_type == "IMET":
@@ -1704,6 +1708,19 @@ def handle_decoder_line(self, data):
                     "%Y-%m-%dT%H:%M:%SZ"
                 )
 
+            # RS41 Subframe Data Actions
+            # We only upload the subframe data once.
+            if 'rs41_calconf51x16' in _telemetry:
+                # Remove subframe data if we have already uploaded it once.
+                if _telemetry['id'] in self.rs41_subframe_uploads:
+                    _telemetry.pop('rs41_calconf51x16')
+                else:
+                    self.rs41_subframe_uploads.append(_telemetry['id'])
+                    self.log_info(f"Received complete calibration dataset for {_telemetry['id']}.")
+                    _telemetry['rs41_subframe'] = _telemetry['rs41_calconf51x16']
+                    _telemetry.pop('rs41_calconf51x16')
+
+
 
             # Grab a snapshot of modem statistics, if we are using an experimental decoder.
             if self.demod_stats is not None:

auto_rx/autorx/logger.py

@@ -5,6 +5,7 @@
 #   Copyright (C) 2018  Mark Jessop <[email protected]>
 #   Released under GNU GPL v3 or later
 #
+import codecs
 import datetime
 import glob
 import logging
@@ -50,7 +51,9 @@ class TelemetryLogger(object):
 
     LOG_HEADER = "timestamp,serial,frame,lat,lon,alt,vel_v,vel_h,heading,temp,humidity,pressure,type,freq_mhz,snr,f_error_hz,sats,batt_v,burst_timer,aux_data\n"
 
-    def __init__(self, log_directory="./log"):
+    def __init__(self, 
+                 log_directory="./log",
+                 save_cal_data=False):
         """ Initialise and start a sonde logger.
         
         Args:
@@ -59,6 +62,7 @@ def __init__(self, log_directory="./log"):
         """
 
         self.log_directory = log_directory
+        self.save_cal_data = save_cal_data
 
         # Dictionary to contain file handles.
         # Each sonde id is added as a unique key. Under each key are the contents:
@@ -199,6 +203,9 @@ def write_telemetry(self, telemetry):
         _id = telemetry["id"]
         _type = telemetry["type"]
 
+        if 'aux' in telemetry:
+            _type += "-XDATA"
+
         # If there is no log open for the current ID check to see if there is an existing (closed) log file, and open it.
         if _id not in self.open_logs:
             _search_string = os.path.join(self.log_directory, "*%s_*_sonde.log" % (_id))
@@ -211,6 +218,7 @@ def write_telemetry(self, telemetry):
                 self.open_logs[_id] = {
                     "log": open(_log_file_name, "a"),
                     "last_time": time.time(),
+                    "subframe_saved": False
                 }
             else:
                 # Create a new log file.
@@ -226,6 +234,7 @@ def write_telemetry(self, telemetry):
                 self.open_logs[_id] = {
                     "log": open(_log_file_name, "a"),
                     "last_time": time.time(),
+                    "subframe_saved": False
                 }
 
                 # Write in a header line.
@@ -241,6 +250,15 @@ def write_telemetry(self, telemetry):
         self.open_logs[_id]["last_time"] = time.time()
         self.log_debug("Wrote line: %s" % _log_line.strip())
 
+        # Save out RS41 subframe data once, if we have it.
+        if ('rs41_subframe' in telemetry) and self.save_cal_data:
+            if self.open_logs[_id]['subframe_saved'] == False:
+                self.open_logs[_id]['subframe_saved'] = self.write_rs41_subframe(telemetry)
+
+
+
+
+
     def cleanup_logs(self):
         """ Close any open logs that have not had telemetry added in X seconds. """
 
@@ -259,6 +277,42 @@ def cleanup_logs(self):
             except Exception as e:
                 self.log_error("Error closing log for %s - %s" % (_id, str(e)))
 
+    def write_rs41_subframe(self, telemetry):
+        """ Write RS41 subframe data to disk """
+
+        _id = telemetry["id"]
+        _type = telemetry["type"]
+
+        if 'aux' in telemetry:
+            _type += "-XDATA"
+
+        _subframe_log_suffix = "%s_%s_%s_%d_subframe.bin" % (
+            datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S"),
+            _id,
+            _type,
+            int(telemetry["freq_float"] * 1e3),  # Convert frequency to kHz
+        )
+        _log_file_name = os.path.join(self.log_directory, _subframe_log_suffix)
+
+
+        try:
+            _subframe_data = codecs.decode(telemetry['rs41_subframe'], 'hex')
+        except Exception as e:
+            self.log_error("Error parsing RS41 subframe data")
+
+        if _subframe_data:
+            _subframe_file = open(_log_file_name, 'wb')
+            _subframe_file.write(_subframe_data)
+            _subframe_file.close()
+
+            self.log_info(f"Wrote subframe data for {telemetry['id']} to {_subframe_log_suffix}")
+            return True
+        else:
+            return False
+
+
+
+
     def close(self):
         """ Close input processing thread. """
         self.input_processing_running = False

auto_rx/autorx/scan.py

@@ -533,11 +533,13 @@ def detect_sonde(
         )
         _sonde_type = "IMET"
     elif "IMET1" in _type:
+        # This could actually be a wideband iMet sonde. We treat this as a IMET4.
         logging.debug(
-            "Scanner (%s) - Detected a iMet Sonde! (Type %s - Unsupported) (Score: %.2f)"
+            "Scanner (%s) - Possible detection of a Wideband iMet Sonde! (Type %s) (Score: %.2f)"
             % (_sdr_name, _type, _score)
         )
-        _sonde_type = "IMET1"
+        # Override the type to IMET4.
+        _sonde_type = "IMET"
     elif "IMETafsk" in _type:
         logging.debug(
             "Scanner (%s) - Detected a iMet Sonde! (Type %s - Unsupported) (Score: %.2f)"

auto_rx/autorx/sondehub.py

@@ -10,6 +10,8 @@
 #   Released under GNU GPL v3 or later
 #
 import autorx
+import base64
+import codecs
 import datetime
 import glob
 import gzip
@@ -291,6 +293,22 @@ def reformat_data(self, telemetry):
         if "ref_datetime" in telemetry:
             _output["ref_datetime"] = telemetry["ref_datetime"]
 
+        if "rs41_mainboard" in telemetry:
+            _output["rs41_mainboard"] = telemetry["rs41_mainboard"]
+
+        if "rs41_mainboard_fw" in telemetry:
+            _output["rs41_mainboard_fw"] = str(telemetry["rs41_mainboard_fw"])
+
+        if 'rs41_subframe' in telemetry:
+            # RS41 calibration subframe data.
+            # We try to base64 encode this.
+            try:
+                _calbytes = codecs.decode(telemetry['rs41_subframe'], 'hex')
+                _output['rs41_subframe'] = base64.b64encode(_calbytes).decode()
+            except Exception as e:
+                self.log_error(f"Error handling RS41 subframe data.")
+
+
         # Handle the additional SNR and frequency estimation if we have it
         if "snr" in telemetry:
             _output["snr"] = telemetry["snr"]

auto_rx/autorx/templates/index.html

@@ -709,6 +709,14 @@
 
                         // Add data into the 'other' field.
                         sonde_id_data.other = "";
+
+                        if(sonde_id_data.hasOwnProperty('rs41_mainboard')){
+                            // Only print mainboard type if it's not the 'original' mainboard.
+                            if(sonde_id_data.rs41_mainboard !== 'RSM412'){
+                                sonde_id_data.other += sonde_id_data.rs41_mainboard + " ";
+                            }
+                        }
+
                         // Burst timer for RS41s
                         if (sonde_id_data.hasOwnProperty('bt')){
                             if ((sonde_id_data.bt >= 0) && (sonde_id_data.bt < 65535)) {
@@ -719,6 +727,11 @@
                             sonde_id_data.other += sonde_id_data.batt.toFixed(1) + " V";
                         }
 
+                        if (sonde_id_data.hasOwnProperty('aux')){
+                            sonde_id_data.type += "-XDATA";
+                        }
+
+
                         telem_data.push(sonde_id_data);
                     });
                 }

auto_rx/autorx/utils.py

@@ -196,6 +196,8 @@ def short_type_lookup(type_name):
         return "Lockheed Martin LMS6-1680"
     elif type_name == "IMET":
         return "Intermet Systems iMet-1/4"
+    elif type_name == "IMET-XDATA":
+        return "Intermet Systems iMet-1/4 + XDATA"
     elif type_name == "IMET5":
         return "Intermet Systems iMet-5x"
     elif type_name == "MEISEI":
@@ -238,6 +240,8 @@ def short_short_type_lookup(type_name):
         return "LMS6-1680"
     elif type_name == "IMET":
         return "iMet-1/4"
+    elif type_name == "IMET-XDATA":
+        return "iMet-1/4"
     elif type_name == "IMET5":
         return "iMet-5x"
     elif type_name == "MEISEI":

auto_rx/station.cfg.example

@@ -458,6 +458,9 @@ save_system_log = False
 # auto_rx operational issues.
 enable_debug_logging = False
 
+# Enable logging of RS41 Calibration data ('subframe' data)
+# This is saved as a binary file with file suffix _subframe.bin
+save_cal_data = False
 
 ###########################
 # WEB INTERFACE SETTINNGS #

auto_rx/station.cfg.example.network

@@ -458,6 +458,10 @@ save_system_log = False
 # auto_rx operational issues.
 enable_debug_logging = False
 
+# Enable logging of RS41 Calibration data ('subframe' data)
+# This is saved as a binary file with file suffix _subframe.bin
+save_cal_data = False
+
 ###########################
 # WEB INTERFACE SETTINNGS #
 ###########################