From a96c60d7eec8cf34ef6cc98cd077e591a36046b6 Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Fri, 3 Nov 2023 09:06:31 +1030 Subject: [PATCH 01/42] Restart always_on decoders after 6 hours of no packets. Give additional error messages if spyserver configured incorrectly. --- auto_rx/auto_rx.py | 2 +- auto_rx/autorx/__init__.py | 2 +- auto_rx/autorx/sdr_wrappers.py | 8 +++++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/auto_rx/auto_rx.py b/auto_rx/auto_rx.py index 03e7c5a3..bcd03c40 100644 --- a/auto_rx/auto_rx.py +++ b/auto_rx/auto_rx.py @@ -244,7 +244,7 @@ def start_decoder(freq, sonde_type, continuous=False): _exp_sonde_type = sonde_type if continuous: - _timeout = 0 + _timeout = 3600*6 # 6 hours before a 'continuous' decoder gets restarted automatically. else: _timeout = config["rx_timeout"] diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index 427e6c17..07c5bdf5 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -12,7 +12,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.7.1" +__version__ = "1.7.2-beta1" # Global Variables diff --git a/auto_rx/autorx/sdr_wrappers.py b/auto_rx/autorx/sdr_wrappers.py index 2b187ad2..6fc2afe0 100644 --- a/auto_rx/autorx/sdr_wrappers.py +++ b/auto_rx/autorx/sdr_wrappers.py @@ -71,7 +71,7 @@ def test_sdr( f"{ss_iq_path} " f"-f {check_freq} " f"-s 48000 " - f"-r {sdr_hostname} -q {sdr_port} -n 48000 - > /dev/null 2> /dev/null" + f"-r {sdr_hostname} -q {sdr_port} -n 48000 - > /dev/null" ) logging.debug(f"SpyServer - Testing using command: {_cmd}") @@ -85,6 +85,12 @@ def test_sdr( logging.critical( f"SpyServer ({sdr_hostname}:{sdr_port}) - ss_iq call failed with return code {e.returncode}." ) + # Look at the error output in a bit more details. + _output = e.output.decode("ascii") + if "outside currently allowed range" in _output: + logging.critical( + f"SpyServer ({sdr_hostname}:{sdr_port}) - SpyServer does not cover required frequency {check_freq}, please check your SpyServer configuration!" + ) return False return True From 24b6835331f3ed9b4f98377f2cde9800a3e1fa90 Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Fri, 10 Nov 2023 08:42:53 +1030 Subject: [PATCH 02/42] Some intial tests of KA9Q-Radio support (no spectrum yet) --- auto_rx/auto_rx.py | 9 +- auto_rx/autorx/config.py | 4 +- auto_rx/autorx/ka9q.py | 129 ++++++++++++++++++++++++++++ auto_rx/autorx/scan.py | 8 +- auto_rx/autorx/sdr_wrappers.py | 107 ++++++++++++++++++++--- auto_rx/station.cfg.example | 10 +-- auto_rx/station.cfg.example.network | 10 +-- 7 files changed, 250 insertions(+), 27 deletions(-) create mode 100644 auto_rx/autorx/ka9q.py diff --git a/auto_rx/auto_rx.py b/auto_rx/auto_rx.py index bcd03c40..99481433 100644 --- a/auto_rx/auto_rx.py +++ b/auto_rx/auto_rx.py @@ -445,7 +445,8 @@ def clean_task_list(): else: # Shutdown the SDR, if required for the particular SDR type. - shutdown_sdr(config["sdr_type"], _task_sdr) + if _key != 'SCAN': + shutdown_sdr(config["sdr_type"], _task_sdr, sdr_hostname=config["sdr_hostname"], frequency=_key) # Release its associated SDR. autorx.sdr_list[_task_sdr]["in_use"] = False autorx.sdr_list[_task_sdr]["task"] = None @@ -505,6 +506,12 @@ def stop_all(): for _task in autorx.task_list.keys(): try: autorx.task_list[_task]["task"].stop() + + # Release the SDR channel if necessary + _task_sdr = autorx.task_list[_task]["device_idx"] + if _task != 'SCAN': + shutdown_sdr(config["sdr_type"], _task_sdr, sdr_hostname=config["sdr_hostname"], frequency=_task) + except Exception as e: logging.error("Error stopping task - %s" % str(e)) diff --git a/auto_rx/autorx/config.py b/auto_rx/autorx/config.py index 07503a81..bfc45a89 100644 --- a/auto_rx/autorx/config.py +++ b/auto_rx/autorx/config.py @@ -867,7 +867,7 @@ def read_auto_rx_config(filename, no_sdr_test=False): return None for _n in range(1, auto_rx_config["sdr_quantity"] + 1): - _sdr_name = f"KA9Q{_n:02d}" + _sdr_name = f"KA9Q-{_n:02d}" auto_rx_config["sdr_settings"][_sdr_name] = { "ppm": 0, "gain": 0, @@ -876,8 +876,6 @@ def read_auto_rx_config(filename, no_sdr_test=False): "task": None, } - logging.critical("Config - KA9Q SDR Support not implemented yet - exiting.") - return None else: logging.critical(f"Config - Unknown SDR Type {auto_rx_config['sdr_type']} - exiting.") diff --git a/auto_rx/autorx/ka9q.py b/auto_rx/autorx/ka9q.py new file mode 100644 index 00000000..f4ca66f1 --- /dev/null +++ b/auto_rx/autorx/ka9q.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# +# radiosonde_auto_rx - SDR Abstraction - KA9Q-Radio +# +# Copyright (C) 2022 Mark Jessop +# Released under GNU GPL v3 or later +# + +import logging +import os.path +import platform +import subprocess +from .utils import timeout_cmd + + +def ka9q_setup_channel( + sdr_hostname, + frequency, + sample_rate +): + # tune --samprate 48000 --frequency 404m09 --mode iq --ssrc 404090000 --radio sonde.local + _cmd = ( + f"{timeout_cmd()} 5 " # Add a timeout, because connections to non-existing servers block for ages + f"tune " + f"--samprate {int(sample_rate)} " + f"--mode iq " + f"--frequency {int(frequency)} " + f"--ssrc {int(frequency)} " + f"--radio {sdr_hostname}" + ) + + logging.debug(f"KA9Q - Starting channel at {frequency} Hz, with command: {_cmd}") + + try: + _output = subprocess.check_output( + _cmd, shell=True, stderr=subprocess.STDOUT + ) + except subprocess.CalledProcessError as e: + # Something went wrong... + + if e.returncode == 124: + logging.critical( + f"KA9Q ({sdr_hostname}) - tune call failed while opening channel with a timeout. Is the server running?" + ) + elif e.returncode == 127: + logging.critical( + f"KA9Q ({sdr_hostname}) - Could not find KA9Q-Radio 'tune' binary! This may need to be compiled and installed." + ) + else: + logging.critical( + f"KA9Q ({sdr_hostname}) - tune call failed while opening channel with return code {e.returncode}." + ) + # Look at the error output in a bit more details. + #_output = e.output.decode("ascii") + + # TODO - see if we can look in the output for any error messages. + return False + + return True + + +def ka9q_close_channel( + sdr_hostname, + frequency +): + + _cmd = ( + f"{timeout_cmd()} 5 " # Add a timeout, because connections to non-existing servers block for ages + f"tune " + f"--samprate 48000 " + f"--mode iq " + f"--frequency 0 " + f"--ssrc {int(frequency)} " + f"--radio {sdr_hostname}" + ) + + logging.debug(f"KA9Q - Closing channel at {frequency} Hz, with command: {_cmd}") + + try: + _output = subprocess.check_output( + _cmd, shell=True, stderr=subprocess.STDOUT + ) + except subprocess.CalledProcessError as e: + # Something went wrong... + + if e.returncode == 124: + logging.critical( + f"KA9Q ({sdr_hostname}) - tune call failed while closing channel with a timeout. Is the server running?" + ) + elif e.returncode == 127: + logging.critical( + f"KA9Q ({sdr_hostname}) - Could not find KA9Q-Radio 'tune' binary! This may need to be compiled and installed." + ) + else: + logging.critical( + f"KA9Q ({sdr_hostname}) - tune call failed while closing chanel with return code {e.returncode}." + ) + # Look at the error output in a bit more details. + #_output = e.output.decode("ascii") + + # TODO - see if we can look in the output for any error messages. + return False + + return True + + +def ka9q_get_iq_cmd( + sdr_hostname, + frequency, + sample_rate +): + + # We need to setup a channel before we can use it! + _setup_success = ka9q_setup_channel(sdr_hostname, frequency, sample_rate) + + if not _setup_success: + logging.critical(f"KA9Q ({sdr_hostname}) - Could not setup rx channel! Decoder will likely timeout.") + + # Get the 'PCM' version of the server name, where as assume -pcm is added to the first part of the hostname. + _pcm_host = sdr_hostname.split('.')[0] + "-pcm." + ".".join(sdr_hostname.split(".")[1:]) + + # pcmcat -2 -s 404090000 sonde-pcm.local + _cmd = ( + f"pcmcat -2 " + f"-s {int(frequency)} " + f"{_pcm_host} |" + ) + + return _cmd diff --git a/auto_rx/autorx/scan.py b/auto_rx/autorx/scan.py index cc2d791e..c1085dd5 100644 --- a/auto_rx/autorx/scan.py +++ b/auto_rx/autorx/scan.py @@ -26,7 +26,7 @@ peak_decimation, timeout_cmd ) -from .sdr_wrappers import test_sdr, reset_sdr, get_sdr_name, get_sdr_iq_cmd, get_sdr_fm_cmd, get_power_spectrum +from .sdr_wrappers import test_sdr, reset_sdr, get_sdr_name, get_sdr_iq_cmd, get_sdr_fm_cmd, get_power_spectrum, shutdown_sdr try: @@ -434,6 +434,10 @@ def detect_sonde( ret_output = subprocess.check_output(rx_test_command, shell=True, stderr=FNULL) FNULL.close() ret_output = ret_output.decode("utf8") + + # Release the SDR channel if necessary + shutdown_sdr(sdr_type, rtl_device_idx, sdr_hostname, frequency) + except subprocess.CalledProcessError as e: # dft_detect returns a code of 1 if no sonde is detected. # logging.debug("Scanner - dfm_detect return code: %s" % e.returncode) @@ -452,7 +456,7 @@ def detect_sonde( except Exception as e: # Something broke when running the detection function. logging.error( - f"Scanner ({_sdr_name}) - Error when running dft_detect - {sdr(e)}" + f"Scanner ({_sdr_name}) - Error when running dft_detect - {str(e)}" ) return (None, 0.0) diff --git a/auto_rx/autorx/sdr_wrappers.py b/auto_rx/autorx/sdr_wrappers.py index 6fc2afe0..e874b939 100644 --- a/auto_rx/autorx/sdr_wrappers.py +++ b/auto_rx/autorx/sdr_wrappers.py @@ -12,6 +12,7 @@ import numpy as np from .utils import rtlsdr_test, reset_rtlsdr_by_serial, reset_all_rtlsdrs, timeout_cmd +from .ka9q import * def test_sdr( @@ -51,13 +52,86 @@ def test_sdr( elif sdr_type == "KA9Q": - # To be implemented - _ok = False + # Test that a KA9Q server is working by attempting to start up a new narrowband channel on it. + + # Check for presence of KA9Q-radio binaries that we need + # if not os.path.isfile('tune'): + # logging.critical("Could not find KA9Q-Radio 'tune' binary! This may need to be compiled and installed.") + # return False + # if not os.path.isfile('pcmcat'): + # logging.critical("Could not find KA9Q-Radio 'pcmcat' binary! This may need to be compiled and installed.") + # return False + # TBD - whatever we need for spectrum use. + # if not os.path.isfile('TBD'): + # logging.critical("Could not find KA9Q-Radio 'tune' binary! This may need to be compiled and installed.") + # return False + + + # Try and configure a channel at check_freq Hz + # tune --samprate 48000 --frequency 404m09 --mode iq --ssrc 404090000 --radio sonde.local + _cmd = ( + f"{timeout_cmd()} 5 " # Add a timeout, because connections to non-existing servers block for ages + f"tune " + f"--samprate 48000 --mode iq " + f"--frequency {int(check_freq)} " + f"--ssrc {int(check_freq)} " + f"--radio {sdr_hostname}" + ) - if not _ok: - logging.error(f"KA9Q Server {sdr_hostname}:{sdr_port} non-functional.") + logging.debug(f"KA9Q - Testing using command: {_cmd}") - return _ok + try: + _output = subprocess.check_output( + _cmd, shell=True, stderr=subprocess.STDOUT + ) + except subprocess.CalledProcessError as e: + # Something went wrong... + + if e.returncode == 124: + logging.critical( + f"KA9Q ({sdr_hostname}) - tune call failed with a timeout. Is the server running?" + ) + elif e.returncode == 127: + logging.critical( + f"KA9Q ({sdr_hostname}) - Could not find KA9Q-Radio 'tune' binary! This may need to be compiled and installed." + ) + else: + logging.critical( + f"KA9Q ({sdr_hostname}) - tune call failed with return code {e.returncode}." + ) + # Look at the error output in a bit more details. + #_output = e.output.decode("ascii") + + # TODO - see if we can look in the output for any error messages. + return False + + # Now close the channel we just opened by setting the frequency to 0 Hz. + _cmd = ( + f"{timeout_cmd()} 5 " # Add a timeout, because connections to non-existing servers block for ages + f"tune " + f"--samprate 48000 --mode iq " + f"--frequency 0 " + f"--ssrc {int(check_freq)} " + f"--radio {sdr_hostname}" + ) + + logging.debug(f"KA9Q - Closing testing channel using command: {_cmd}") + try: + _output = subprocess.check_output( + _cmd, shell=True, stderr=subprocess.STDOUT + ) + except subprocess.CalledProcessError as e: + # Something went wrong... + logging.critical( + f"KA9Q ({sdr_hostname}) - tune call (closing channel) failed with return code {e.returncode}." + ) + # Look at the error output in a bit more details. + #_output = e.output.decode("ascii") + + # TODO - see if we can look in the output for any error messages. + return False + + return True elif sdr_type == "SpyServer": # Test connectivity to a SpyServer by trying to grab some samples. @@ -156,7 +230,7 @@ def get_sdr_name( return f"RTLSDR {rtl_device_idx}" elif sdr_type == "KA9Q": - return f"KA9Q {sdr_hostname}:{sdr_port}" + return f"KA9Q {sdr_hostname}" elif sdr_type == "SpyServer": return f"SpyServer {sdr_hostname}:{sdr_port}" @@ -167,7 +241,9 @@ def get_sdr_name( def shutdown_sdr( sdr_type: str, - sdr_id: str + sdr_id: str, + sdr_hostname = "", + frequency: int = None ): """ Function to trigger shutdown/cleanup of some SDR types. @@ -178,8 +254,8 @@ def shutdown_sdr( """ if sdr_type == "KA9Q": - # TODO - KA9Q Server channel cleanup. - logging.debug(f"TODO - Cleanup for SDR type {sdr_type}") + logging.debug(f"KA9Q - Closing Channel for {sdr_hostname} @ {frequency} Hz.") + ka9q_close_channel(sdr_hostname, frequency) pass else: logging.debug(f"No shutdown action required for SDR type {sdr_type}") @@ -278,6 +354,14 @@ def get_sdr_iq_cmd( _cmd += _dc_remove return _cmd + + if sdr_type == "KA9Q": + _cmd = ka9q_get_iq_cmd(sdr_hostname, frequency, sample_rate) + + if dc_block: + _cmd += _dc_remove + + return _cmd else: logging.critical(f"IQ Source - Unsupported SDR type {sdr_type}") @@ -614,8 +698,9 @@ def get_power_spectrum( else: # Unsupported SDR Type - logging.critical(f"Get PSD - Unsupported SDR Type: {sdr_type}") - return (None, None, None) + logging.debug(f"Get PSD - Unsupported SDR Type: {sdr_type}") + return (np.array([0,1,2]),np.array([0,1,2]),1) + #return (None, None, None) if __name__ == "__main__": diff --git a/auto_rx/station.cfg.example b/auto_rx/station.cfg.example index edbde7f9..1588252a 100644 --- a/auto_rx/station.cfg.example +++ b/auto_rx/station.cfg.example @@ -21,8 +21,8 @@ # # EXPERIMENTAL / NOT IMPLEMENTED options: # SpyServer - Use an Airspy SpyServer -# KA9Q - Use a KA9Q SDR Server (Not yet implemented) -# WARNING: These are still under development and may not work. +# KA9Q - Use a KA9Q-Radio Server +# WARNING: These are still under development and may not work correctly. # sdr_type = RTLSDR @@ -43,9 +43,9 @@ sdr_quantity = 1 # # Network SDR Connection Details # -# If using either a KA9Q or SpyServer network server, the hostname and port -# of the server needs to be defined below. Usually this will be running on the -# same machine as auto_rx, so the defaults are usually fine. +# If using a spyserver, the hostname and port need to be defined below. +# Is using KA9Q-Radio, the hostname of the 'radio' server (e.g. sonde.local) needs to be +# defined, and the port number is unused. # sdr_hostname = localhost sdr_port = 5555 diff --git a/auto_rx/station.cfg.example.network b/auto_rx/station.cfg.example.network index 4cd46182..b94e0152 100644 --- a/auto_rx/station.cfg.example.network +++ b/auto_rx/station.cfg.example.network @@ -22,8 +22,8 @@ # # EXPERIMENTAL / NOT IMPLEMENTED options: # SpyServer - Use an Airspy SpyServer -# KA9Q - Use a KA9Q SDR Server (Not yet implemented) -# WARNING: These are still under development and may not work. +# KA9Q - Use a KA9Q-Radio Server +# WARNING: These are still under development and may not work correctly. # sdr_type = SpyServer @@ -44,9 +44,9 @@ sdr_quantity = 5 # # Network SDR Connection Details # -# If using either a KA9Q or SpyServer network server, the hostname and port -# of the server needs to be defined below. Usually this will be running on the -# same machine as auto_rx, so the defaults are usually fine. +# If using a spyserver, the hostname and port need to be defined below. +# Is using KA9Q-Radio, the hostname of the 'radio' server (e.g. sonde.local) needs to be +# defined, and the port number is unused. # sdr_hostname = localhost sdr_port = 5555 From 1d90d0c4b62051e3f3523377e2eda830dd79182a Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Fri, 10 Nov 2023 08:43:27 +1030 Subject: [PATCH 03/42] bump testing version --- auto_rx/autorx/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index 07c5bdf5..9dc1d8b1 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -12,7 +12,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.7.2-beta1" +__version__ = "1.7.2-beta2" # Global Variables From 2807232f23f0ec910993d739b9f520f08c5a437f Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Mon, 1 Jan 2024 10:25:38 +1030 Subject: [PATCH 04/42] Add Temp Block button on web controls --- auto_rx/autorx/__init__.py | 2 +- auto_rx/autorx/decode.py | 6 ++++- auto_rx/autorx/static/js/autorxapi.js | 38 +++++++++++++++++++++++++++ auto_rx/autorx/templates/index.html | 3 +++ auto_rx/autorx/web.py | 13 +++++++-- 5 files changed, 58 insertions(+), 4 deletions(-) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index 9dc1d8b1..206edf55 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -12,7 +12,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.7.2-beta2" +__version__ = "1.7.2-beta3" # Global Variables diff --git a/auto_rx/autorx/decode.py b/auto_rx/autorx/decode.py index 75679a28..ed4e0b71 100644 --- a/auto_rx/autorx/decode.py +++ b/auto_rx/autorx/decode.py @@ -1845,8 +1845,12 @@ def log_critical(self, line): f"Decoder ({_sdr_name}) {self.sonde_type} {self.sonde_freq/1e6:.3f} - {line}" ) - def stop(self, nowait=False): + def stop(self, nowait=False, temporary_lockout=False): """ Kill the currently running decoder subprocess """ + + if temporary_lockout: + self.exit_state = "TempBlock" + self.decoder_running = False if self.decoder is not None and (not nowait): diff --git a/auto_rx/autorx/static/js/autorxapi.js b/auto_rx/autorx/static/js/autorxapi.js index 340cfcff..7cf7c141 100644 --- a/auto_rx/autorx/static/js/autorxapi.js +++ b/auto_rx/autorx/static/js/autorxapi.js @@ -62,6 +62,7 @@ function disable_web_controls(){ $("#verify-password").prop('disabled', true); $("#start-decoder").prop('disabled', true); $("#stop-decoder").prop('disabled', true); + $("#stop-decoder-lockout").prop('disabled', true); $("#enable-scanner").prop('disabled', true); $("#disable-scanner").prop('disabled', true); $("#frequency-input").prop('disabled', true); @@ -75,6 +76,7 @@ function pause_web_controls() { $("#verify-password").prop('disabled', true); $("#start-decoder").prop('disabled', true); $("#stop-decoder").prop('disabled', true); + $("#stop-decoder-lockout").prop('disabled', true); $("#enable-scanner").prop('disabled', true); $("#disable-scanner").prop('disabled', true); $("#frequency-input").prop('disabled', true); @@ -86,6 +88,7 @@ function resume_web_controls() { $("#verify-password").prop('disabled', false); $("#start-decoder").prop('disabled', false); $("#stop-decoder").prop('disabled', false); + $("#stop-decoder-lockout").prop('disabled', false); $("#enable-scanner").prop('disabled', false); $("#disable-scanner").prop('disabled', false); $("#frequency-input").prop('disabled', false); @@ -235,6 +238,41 @@ function stop_decoder(){ }); } +function stop_decoder_lockout(){ + // Stop the decoder on the requested frequency, and lockout frequency + + // Re-verify the password. This will occur async, so wont stop the main request from going ahead, + // but will at least present an error for the user. + verify_password(); + + // Grab the password + _api_password = getCookie("password"); + + // Grab the selected frequency + _decoder = $('#stop-frequency-select').val(); + + // Do the request + $.post( + "stop_decoder", + {password: _api_password, freq: _decoder, lockout: 1}, + function(data){ + //console.log(data); + pause_web_controls(); + setTimeout(resume_web_controls,10000); + // Need to figure out where to put this data.. + } + ).fail(function(xhr, status, error){ + console.log(error); + // Otherwise, we probably got a 403 error (forbidden) which indicates the password was bad. + if(error == "FORBIDDEN"){ + $("#password-header").html("

Incorrect Password

"); + } else if (error == "NOT FOUND"){ + // Scanner isn't running. Don't do anything. + alert("Decoder on supplied frequency not running!"); + } + }); +} + function start_decoder(){ // Start a decoder on the requested frequency diff --git a/auto_rx/autorx/templates/index.html b/auto_rx/autorx/templates/index.html index 345c555c..2e43c1ad 100644 --- a/auto_rx/autorx/templates/index.html +++ b/auto_rx/autorx/templates/index.html @@ -1684,6 +1684,9 @@

Decoder Control

+
+ +

Scanner Control

Scanner

diff --git a/auto_rx/autorx/web.py b/auto_rx/autorx/web.py index 1455a3c0..67be9dab 100644 --- a/auto_rx/autorx/web.py +++ b/auto_rx/autorx/web.py @@ -464,7 +464,11 @@ def flask_start_decoder(): def flask_stop_decoder(): """ Request that a decoder process be halted. Example: - curl -d "freq=403250000" -X POST http://localhost:5000/stop_decoder + + curl -d "freq=403250000&password=foobar" -X POST http://localhost:5000/stop_decoder + + Stop decoder and lockout for temporary_block_time + curl -d "freq=403250000&password=foobar&lockout=1" -X POST http://localhost:5000/stop_decoder """ if request.method == "POST" and autorx.config.global_config["web_control"]: @@ -476,10 +480,15 @@ def flask_stop_decoder(): ): _freq = float(request.form["freq"]) + _lockout = False + if "lockout" in request.form: + if int(request.form["lockout"]) == 1: + _lockout = True + logging.info("Web - Got decoder stop request: %f" % (_freq)) if _freq in autorx.task_list: - autorx.task_list[_freq]["task"].stop(nowait=True) + autorx.task_list[_freq]["task"].stop(nowait=True, temporary_lockout=_lockout) return "OK" else: # If we aren't running a decoder, 404. From 5878ad433c64172d56b26aa85915fa4af02cc52d Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Sat, 6 Jan 2024 09:56:56 +1030 Subject: [PATCH 05/42] Update meisei demod to handle 2024 year --- auto_rx/autorx/__init__.py | 2 +- demod/mod/meisei100mod.c | 46 ++++++++++++++++++++++++++++++++------ 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index 206edf55..18db9eed 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -12,7 +12,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.7.2-beta3" +__version__ = "1.7.2-beta4" # Global Variables diff --git a/demod/mod/meisei100mod.c b/demod/mod/meisei100mod.c index 19eba26d..923c14cd 100644 --- a/demod/mod/meisei100mod.c +++ b/demod/mod/meisei100mod.c @@ -136,6 +136,7 @@ e.g. -b --br 2398 typedef struct { int frnr; int frnr1; + int ref_yr; int jahr; int monat; int tag; int std; int min; float sek; double lat; double lon; double alt; @@ -326,11 +327,27 @@ static int reset_gpx(gpx_t *gpx) { /* -------------------------------------------------------------------------- */ +static int est_year_ims100(int _y, int _yr) { + int yr_rollover = 20; // default: 2020..2029 + int yr_offset = 20; + if (_yr > 2003 && _yr < 2100) { + yr_rollover = _yr - 2004; + yr_offset = (yr_rollover / 10) * 10; + } + _y %= 10; + _y += yr_offset; + if (_y < yr_rollover) _y += 10; + return 2000+_y; +} + +/* -------------------------------------------------------------------------- */ + int main(int argc, char **argv) { int option_verbose = 0, option_raw = 0, + option_dbg = 0, option_inv = 0, option_ecc = 0, // BCH(63,51) option_jsn = 0; // JSON output (auto_rx) @@ -427,6 +444,7 @@ int main(int argc, char **argv) { return 0; } else if ( (strcmp(*argv, "-r") == 0) ) { option_raw = 1; } + else if ( (strcmp(*argv, "--dbg") == 0) ) { option_dbg = 1; } else if ( (strcmp(*argv, "-i") == 0) || (strcmp(*argv, "--invert") == 0) ) { option_inv = 1; // nicht noetig } @@ -445,7 +463,7 @@ int main(int argc, char **argv) { ++argv; if (*argv) { baudrate = atof(*argv); - if (baudrate < 2200 || baudrate > 2400) baudrate = 2400; // default: 2400 + if (baudrate < 2200 || baudrate > 2600) baudrate = 2400; // default: 2400 } else return -1; } @@ -507,6 +525,12 @@ int main(int argc, char **argv) { if (frq < 300000000) frq = -1; cfreq = frq; } + else if (strcmp(*argv, "--year") == 0) { + int _yr = 0; + ++argv; + if (*argv) _yr = atoi(*argv); else return -1; + if (_yr > 2003 && _yr < 2100) gpx.ref_yr = _yr; + } else if (strcmp(*argv, "-") == 0) { int sample_rate = 0, bits_sample = 0, channels = 0; ++argv; @@ -546,6 +570,9 @@ int main(int argc, char **argv) { if (cfreq > 0) gpx.jsn_freq = (cfreq+500)/1000; + // ims100: default ref. year + if (gpx.ref_yr < 2000) gpx.ref_yr = 2024; // -> 2020..2029 + #ifdef EXT_FSK if (!option_softin) { @@ -783,6 +810,10 @@ int main(int argc, char **argv) { | ( (w16[0]&0xFF00)>>8 | (w16[0]&0xFF)<<8 ); fw32 = f32e2(w32); + if (option_dbg) { + printf(" # [%02d] %08x : %.1f # ", counter % 64, w32, fw32); + } + if (err_blks == 0) // err_frm zu schwach { gpx.cfg[counter%64] = fw32; @@ -1016,6 +1047,10 @@ int main(int argc, char **argv) { w16[0] = bits2val(subframe_bits+HEADLEN+46*1 , 16); w16[1] = bits2val(subframe_bits+HEADLEN+46*1+17, 16); w32 = (w16[1]<<16) | w16[0]; + + if (option_dbg) { + printf(" # [%02d] %08x : %.1f # ", counter % 64, w32, *fcfg); + } // counter ok and w16[] ok (max 1 error) if (err_frm == 0 && block_err[0] < 2 && block_err[1] < 2) { @@ -1117,12 +1152,9 @@ int main(int argc, char **argv) { dat2 = bits2val(subframe_bits+HEADLEN, 16); gpx.tag = dat2/1000; gpx.monat = (dat2/10)%100; - _y = (dat2%10)+10; - if (_y < 14) _y += 10; // 2020 - gpx.jahr = 2000 + _y; - //if (option_verbose) printf("%05u ", dat2); - //printf("(%02d-%02d-%02d) ", gpx.tag, gpx.monat, gpx.jahr%100); // 2020: +20 ? - printf("(%04d-%02d-%02d) ", gpx.jahr, gpx.monat, gpx.tag); // 2020: +20 ? + _y = dat2 % 10; + gpx.jahr = est_year_ims100(_y, gpx.ref_yr); + printf("(%04d-%02d-%02d) ", gpx.jahr, gpx.monat, gpx.tag); lat1 = bits2val(subframe_bits+HEADLEN+46*0+17, 16); lat2 = bits2val(subframe_bits+HEADLEN+46*1 , 16); From 5a2925d2cf463d7161ee4eb230e6c9c99054c56a Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Sat, 6 Jan 2024 10:57:26 +1030 Subject: [PATCH 06/42] Bump testing version --- auto_rx/autorx/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index 18db9eed..77f423d3 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -12,7 +12,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.7.2-beta4" +__version__ = "1.7.3-beta4" # Global Variables From 6c887ce7d5bc97db23064ae66438248853a36000 Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Wed, 24 Jan 2024 16:57:53 +1030 Subject: [PATCH 07/42] Fix imperial units on historical page --- auto_rx/autorx/__init__.py | 2 +- auto_rx/autorx/templates/historical.html | 24 ++++++++++++++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index 77f423d3..919acd34 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -12,7 +12,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.7.3-beta4" +__version__ = "1.7.3-beta5" # Global Variables diff --git a/auto_rx/autorx/templates/historical.html b/auto_rx/autorx/templates/historical.html index 95d7b577..b3198beb 100644 --- a/auto_rx/autorx/templates/historical.html +++ b/auto_rx/autorx/templates/historical.html @@ -122,17 +122,29 @@ return cell.getValue().toFixed(3); }}, {title:"Count", field:"lines", width:72, resizable:false, headerTooltip:"Received Lines of Telemetry"}, // 75 - {title:"Last H", field:"min_height", width:75, resizable:false, headerTooltip:"Last Observed Height (m)", + {title:"Last H", field:"min_height", width:75, resizable:false, headerTooltip:"Last Observed Height", formatter:function(cell, formatterParams, onRendered){ - return cell.getValue() + " m"; //return the contents of the cell; + if (getCookie('imperial') == 'true') { + return cell.getValue()*3.28084.toFixed(0) + " ft"; + } else { + return cell.getValue() + " m"; + } }}, - {title:"Last R", field:"last_range", width:75, resizable:false, headerTooltip:"Last Observed Range (km)", + {title:"Last R", field:"last_range", width:75, resizable:false, headerTooltip:"Last Observed Range", formatter:function(cell, formatterParams, onRendered){ - return cell.getValue() + " km"; //return the contents of the cell; + if (getCookie('imperial') == 'true') { + return cell.getValue()*0.621371.toFixed(0) + " mi"; + } else { + return cell.getValue() + " km"; + } }}, - {title:"Max R", field:"max_range", width:73, resizable:false, headerTooltip:"Maximum Observed Range (km))", + {title:"Max R", field:"max_range", width:73, resizable:false, headerTooltip:"Maximum Observed Range", formatter:function(cell, formatterParams, onRendered){ - return cell.getValue() + " km"; //return the contents of the cell; + if (getCookie('imperial') == 'true') { + return cell.getValue()*0.621371.toFixed(0) + " mi"; + } else { + return cell.getValue() + " km"; + } }}, {formatter:"rowSelection", titleFormatter:"rowSelection", align:"center", width:40, headerSort:false, titleFormatter: function(cell, formatterParams, onRendered){}} ], From 0e6af12b397959f706999790c6e58f143916b993 Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Thu, 25 Jan 2024 14:10:11 +1030 Subject: [PATCH 08/42] Some fixes to statistics generation --- auto_rx/autorx/log_files.py | 7 +++++-- auto_rx/autorx/stats.py | 18 +++++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/auto_rx/autorx/log_files.py b/auto_rx/autorx/log_files.py index bb916abc..d40a5ed8 100644 --- a/auto_rx/autorx/log_files.py +++ b/auto_rx/autorx/log_files.py @@ -200,14 +200,17 @@ def log_quick_look(filename): return _output -def list_log_files(quicklook=False): +def list_log_files(quicklook=False, custom_log_dir=None): """ Look for all sonde log files within the logging directory """ # Output list, which will contain one object per log file, ordered by time _output = [] # Search for file matching the expected log file name - _log_mask = os.path.join(autorx.logging_path, "*_sonde.log") + if custom_log_dir: + _log_mask = os.path.join(custom_log_dir, "*_sonde.log") + else: + _log_mask = os.path.join(autorx.logging_path, "*_sonde.log") _log_files = glob.glob(_log_mask) # Sort alphanumerically, which will result in the entries being date ordered diff --git a/auto_rx/autorx/stats.py b/auto_rx/autorx/stats.py index 81bc7ccb..8e4508d4 100644 --- a/auto_rx/autorx/stats.py +++ b/auto_rx/autorx/stats.py @@ -74,10 +74,10 @@ def radio_horizon_plot(log_files, min_range_km=10, max_range_km=1000, save_figur plt.grid() -def normalised_snr(log_files, min_range_km=10, max_range_km=1000, maxsnr=False, meansnr=True, normalise=True): +def normalised_snr(log_files, min_range_km=10, max_range_km=1000, maxsnr=False, meansnr=True, normalise=True, norm_range=50): """ Read in ALL log files and store snr data into a set of bins, normalised to 50km range. """ - _norm_range = 50 # km + _norm_range = norm_range # km _snr_count = 0 @@ -198,6 +198,12 @@ def normalised_snr(log_files, min_range_km=10, max_range_km=1000, maxsnr=False, default=False, help="Generate Normalised SNR Map (Maximum SNR)" ) + parser.add_argument( + "--normrange", + type=float, + default=50, + help="Normalistion Range (km, default=50)" + ) parser.add_argument( "-v", "--verbose", help="Enable debug output.", action="store_true" @@ -213,6 +219,8 @@ def normalised_snr(log_files, min_range_km=10, max_range_km=1000, maxsnr=False, format="%(asctime)s %(levelname)s:%(message)s", level=_log_level ) + autorx.logging_path = args.log + # Read in the config and make it available to other functions _temp_cfg = read_auto_rx_config(args.config, no_sdr_test=True) autorx.config.global_config = _temp_cfg @@ -220,7 +228,7 @@ def normalised_snr(log_files, min_range_km=10, max_range_km=1000, maxsnr=False, # Read in the log files. logging.info("Quick-Looking Log Files") - log_list = list_log_files(quicklook=True) + log_list = list_log_files(quicklook=True, custom_log_dir=args.log) logging.info(f"Loaded in {len(log_list)} log files.") @@ -228,13 +236,13 @@ def normalised_snr(log_files, min_range_km=10, max_range_km=1000, maxsnr=False, radio_horizon_plot(log_list) if args.snrmap: - normalised_snr(log_list) + normalised_snr(log_list, norm_range=args.normrange) if args.snrmapmax: normalised_snr(log_list, meansnr=False, maxsnr=True, normalise=False) if args.snrmapmaxnorm: - normalised_snr(log_list, meansnr=False, maxsnr=True, normalise=True) + normalised_snr(log_list, meansnr=False, maxsnr=True, normalise=True, norm_range=args.normrange) plt.show() From 96228c41df71cd527f40a7a768f1af134003996f Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Sat, 27 Jan 2024 10:50:03 +1030 Subject: [PATCH 09/42] Write rtl_power log output to user-defined log directory --- auto_rx/autorx/__init__.py | 2 +- auto_rx/autorx/sdr_wrappers.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index 919acd34..dc6384cd 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -12,7 +12,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.7.3-beta5" +__version__ = "1.7.3-beta6" # Global Variables diff --git a/auto_rx/autorx/sdr_wrappers.py b/auto_rx/autorx/sdr_wrappers.py index e874b939..145f186e 100644 --- a/auto_rx/autorx/sdr_wrappers.py +++ b/auto_rx/autorx/sdr_wrappers.py @@ -5,6 +5,7 @@ # Copyright (C) 2022 Mark Jessop # Released under GNU GPL v3 or later # +import autorx import logging import os.path import platform @@ -564,7 +565,7 @@ def get_power_spectrum( # Use rtl_power to obtain power spectral density data # Create filename to output to. - _log_filename = f"log_power_{rtl_device_idx}.csv" + _log_filename = os.path.join(autorx.logging_path,f"log_power_{rtl_device_idx}.csv") # If the output log file exists, remove it. if os.path.exists(_log_filename): @@ -638,7 +639,7 @@ def get_power_spectrum( # Use a spyserver to obtain power spectral density data # Create filename to output to. - _log_filename = f"log_power_spyserver.csv" + _log_filename = os.path.join(autorx.logging_path,f"log_power_spyserver.csv") # If the output log file exists, remove it. if os.path.exists(_log_filename): From 49d213ea5e7a1103dfeee7a45da47ca9ec299cb2 Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Sat, 27 Jan 2024 10:51:26 +1030 Subject: [PATCH 10/42] Add spaces --- auto_rx/autorx/sdr_wrappers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/auto_rx/autorx/sdr_wrappers.py b/auto_rx/autorx/sdr_wrappers.py index 145f186e..6883a116 100644 --- a/auto_rx/autorx/sdr_wrappers.py +++ b/auto_rx/autorx/sdr_wrappers.py @@ -565,7 +565,7 @@ def get_power_spectrum( # Use rtl_power to obtain power spectral density data # Create filename to output to. - _log_filename = os.path.join(autorx.logging_path,f"log_power_{rtl_device_idx}.csv") + _log_filename = os.path.join(autorx.logging_path, f"log_power_{rtl_device_idx}.csv") # If the output log file exists, remove it. if os.path.exists(_log_filename): @@ -639,7 +639,7 @@ def get_power_spectrum( # Use a spyserver to obtain power spectral density data # Create filename to output to. - _log_filename = os.path.join(autorx.logging_path,f"log_power_spyserver.csv") + _log_filename = os.path.join(autorx.logging_path, f"log_power_spyserver.csv") # If the output log file exists, remove it. if os.path.exists(_log_filename): From 7226c31ca1362c7c67f9f538875fc541da8e91a1 Mon Sep 17 00:00:00 2001 From: Clayton Smith Date: Sat, 27 Jan 2024 22:59:39 -0500 Subject: [PATCH 11/42] Combine export_selected and export_all --- auto_rx/autorx/web.py | 36 +++--------------------------------- 1 file changed, 3 insertions(+), 33 deletions(-) diff --git a/auto_rx/autorx/web.py b/auto_rx/autorx/web.py index 67be9dab..f1e984e4 100644 --- a/auto_rx/autorx/web.py +++ b/auto_rx/autorx/web.py @@ -335,15 +335,16 @@ def flask_get_log_by_serial_detail(): return json.dumps(read_log_by_serial(_serial, skewt_decimation=_decim)) +@app.route("/export_all_log_files") @app.route("/export_log_files/") -def flask_export_selected_log_files(serialb64): +def flask_export_log_files(serialb64=None): """ Zip and download a set of log files. The list of log files is provided in the URL as a base64-encoded JSON list. """ try: - _serial_list = json.loads(base64.b64decode(serialb64)) + _serial_list = json.loads(base64.b64decode(serialb64)) if serialb64 else None _zip = zip_log_files(_serial_list) @@ -368,37 +369,6 @@ def flask_export_selected_log_files(serialb64): logging.error("Web - Error handling Zip request:" + str(e)) abort(400) - -@app.route("/export_all_log_files") -def flask_export_all_log_files(): - """ - Zip and download all log files. This may take some time. - """ - - try: - _zip = zip_log_files() - - _ts = datetime.datetime.strftime(datetime.datetime.utcnow(), "%Y%m%d-%H%M%SZ") - - response = make_response( - flask.send_file( - _zip, - mimetype="application/zip", - as_attachment=True, - download_name=f"autorx_logfiles_{autorx.config.global_config['habitat_uploader_callsign']}_{_ts}.zip", - ) - ) - - # Add header asking client not to cache the download - response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate" - response.headers["Pragma"] = "no-cache" - - return response - - except Exception as e: - logging.error("Web - Error handling Zip request:" + str(e)) - abort(400) - # # Control Endpoints. # From 52135dbe8dfdc2489f0551be5dcce5fdfc5efcfa Mon Sep 17 00:00:00 2001 From: Clayton Smith Date: Sun, 28 Jan 2024 13:04:58 -0500 Subject: [PATCH 12/42] Fix code style --- auto_rx/utils/log_to_kml.py | 104 +++++++++++++++++------------------- 1 file changed, 48 insertions(+), 56 deletions(-) diff --git a/auto_rx/utils/log_to_kml.py b/auto_rx/utils/log_to_kml.py index d5003796..565d6bdd 100644 --- a/auto_rx/utils/log_to_kml.py +++ b/auto_rx/utils/log_to_kml.py @@ -20,13 +20,14 @@ from dateutil.parser import parse from shapely.geometry import Point, LineString + def read_telemetry_csv(filename, - datetime_field = 0, - latitude_field = 3, - longitude_field = 4, - altitude_field = 5, - delimiter=','): - ''' + datetime_field=0, + latitude_field=3, + longitude_field=4, + altitude_field=5, + delimiter=','): + ''' Read in a radiosonde_auto_rx generated telemetry CSV file. Fields to use can be set as arguments to this function. These have output like the following: @@ -47,7 +48,7 @@ def read_telemetry_csv(filename, output = [] - f = open(filename,'r') + f = open(filename, 'r') for line in f: try: @@ -90,12 +91,13 @@ def flight_burst_position(flight_path): ns = '{http://www.opengis.net/kml/2.2}' + def new_placemark(lat, lon, alt, - placemark_id="Placemark ID", - name="Placemark Name", - absolute = False, - icon = "http://maps.google.com/mapfiles/kml/shapes/placemark_circle.png", - scale = 1.0): + placemark_id="Placemark ID", + name="Placemark Name", + absolute=False, + icon="http://maps.google.com/mapfiles/kml/shapes/placemark_circle.png", + scale=1.0): """ Generate a generic placemark object """ if absolute: @@ -104,8 +106,8 @@ def new_placemark(lat, lon, alt, _alt_mode = 'clampToGround' flight_icon_style = fastkml.styles.IconStyle( - ns=ns, - icon_href=icon, + ns=ns, + icon_href=icon, scale=scale) flight_style = fastkml.styles.Style( @@ -113,7 +115,7 @@ def new_placemark(lat, lon, alt, styles=[flight_icon_style]) flight_placemark = fastkml.kml.Placemark( - ns=ns, + ns=ns, id=placemark_id, name=name, description="", @@ -127,16 +129,15 @@ def new_placemark(lat, lon, alt, return flight_placemark - def flight_path_to_geometry(flight_path, - placemark_id="Flight Path ID", - name="Flight Path Name", - track_color="aaffffff", - poly_color="20000000", - track_width=2.0, - absolute = True, - extrude = True, - tessellate = True): + placemark_id="Flight Path ID", + name="Flight Path Name", + track_color="aaffffff", + poly_color="20000000", + track_width=2.0, + absolute=True, + extrude=True, + tessellate=True): ''' Produce a fastkml geometry object from a flight path array ''' # Handle selection of absolute altitude mode @@ -149,7 +150,7 @@ def flight_path_to_geometry(flight_path, track_points = [] for _point in flight_path: # Flight path array is in lat,lon,alt order, needs to be in lon,lat,alt - track_points.append([_point[2],_point[1],_point[3]]) + track_points.append([_point[2], _point[1], _point[3]]) _flight_geom = LineString(track_points) @@ -186,8 +187,8 @@ def flight_path_to_geometry(flight_path, def write_kml(geom_objects, - filename="output.kml", - comment=""): + filename="output.kml", + comment=""): """ Write out flight path geometry objects to a kml file. """ kml_root = fastkml.kml.KML() @@ -201,7 +202,7 @@ def write_kml(geom_objects, for _flight in geom_objects: kml_doc.append(_flight) - with open(filename,'w') as kml_file: + with open(filename, 'w') as kml_file: kml_file.write(kml_doc.to_string()) kml_file.close() @@ -214,7 +215,7 @@ def convert_single_file(filename, absolute=True, tessellate=True, last_only=Fals # Extract the flight's serial number and launch time from the first line in the file. _first_line = _flight_data[0][4] - _flight_serial = _first_line.split(',')[1] # Serial number is the second field in the line. + _flight_serial = _first_line.split(',')[1] # Serial number is the second field in the line. _launch_time = _flight_data[0][0].strftime("%Y%m%d-%H%M%SZ") # Generate a comment line to use in the folder and placemark descriptions _track_comment = "%s %s" % (_launch_time, _flight_serial) @@ -225,11 +226,13 @@ def convert_single_file(filename, absolute=True, tessellate=True, last_only=Fals _landing_pos = _flight_data[-1] # Generate the placemark & flight track. - _flight_geom = flight_path_to_geometry(_flight_data, name=_track_comment, absolute=absolute, tessellate=tessellate, extrude=tessellate) - _landing_geom = new_placemark(_landing_pos[1], _landing_pos[2], _landing_pos[3], name=_landing_comment, absolute=absolute) + _flight_geom = flight_path_to_geometry(_flight_data, name=_track_comment, absolute=absolute, + tessellate=tessellate, extrude=tessellate) + _landing_geom = new_placemark(_landing_pos[1], _landing_pos[2], _landing_pos[3], + name=_landing_comment, absolute=absolute) _folder = fastkml.kml.Folder(ns, _flight_serial, _track_comment, 'Radiosonde Flight Path') - if last_only == False: + if not last_only: _folder.append(_flight_geom) _folder.append(_landing_geom) @@ -238,12 +241,17 @@ def convert_single_file(filename, absolute=True, tessellate=True, last_only=Fals if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("-i", "--input", type=str, default="../log/*_sonde.log", - help="Path to log file. May include wildcards, though the path must be wrapped in quotes. Default=../log/*_sonde.log") - parser.add_argument("-o", "--output", type=str, default="sondes.kml", help="KML output file name. Default=sondes.kml") - parser.add_argument('--clamp', action="store_false", default=True, help="Clamp tracks to ground instead of showing absolute altitudes.") - parser.add_argument('--noextrude', action="store_false", default=True, help="Disable Extrusions for absolute flight paths.") - parser.add_argument('--lastonly', action="store_true", default=False, help="Only plot last-seen sonde positions, not the flight paths.") + parser.add_argument("-i", "--input", type=str, default="../log/*_sonde.log", + help="Path to log file. May include wildcards, though the path " + "must be wrapped in quotes. Default=../log/*_sonde.log") + parser.add_argument("-o", "--output", type=str, default="sondes.kml", + help="KML output file name. Default=sondes.kml") + parser.add_argument('--clamp', action="store_false", default=True, + help="Clamp tracks to ground instead of showing absolute altitudes.") + parser.add_argument('--noextrude', action="store_false", default=True, + help="Disable Extrusions for absolute flight paths.") + parser.add_argument('--lastonly', action="store_true", default=False, + help="Only plot last-seen sonde positions, not the flight paths.") args = parser.parse_args() _file_list = glob.glob(args.input) @@ -253,27 +261,11 @@ def convert_single_file(filename, absolute=True, tessellate=True, last_only=Fals for _file in _file_list: print("Processing: %s" % _file) try: - _placemarks.append(convert_single_file(_file, absolute=args.clamp, tessellate=args.noextrude, last_only=args.lastonly)) + _placemarks.append(convert_single_file(_file, absolute=args.clamp, + tessellate=args.noextrude, last_only=args.lastonly)) except: print("Failed to process: %s" % _file) write_kml(_placemarks, filename=args.output) print("Output saved to: %s" % args.output) - - - - - - - - - - - - - - - - - From 54ba77ed2b6f4380ba2a3e49c42aeaaf4f1f1d9f Mon Sep 17 00:00:00 2001 From: Clayton Smith Date: Sun, 28 Jan 2024 13:06:42 -0500 Subject: [PATCH 13/42] Remove unused imports --- auto_rx/utils/log_to_kml.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/auto_rx/utils/log_to_kml.py b/auto_rx/utils/log_to_kml.py index 565d6bdd..6c38d030 100644 --- a/auto_rx/utils/log_to_kml.py +++ b/auto_rx/utils/log_to_kml.py @@ -9,13 +9,9 @@ # sudo pip install fastkml shapely # -import sys -import time -import datetime import traceback import argparse import glob -import os import fastkml from dateutil.parser import parse from shapely.geometry import Point, LineString From 7d739ab7bba66f6977e1ced6a849b3f5ef90d178 Mon Sep 17 00:00:00 2001 From: Clayton Smith Date: Mon, 29 Jan 2024 17:05:04 -0500 Subject: [PATCH 14/42] Replace fastkml & shapely with ElementTree --- auto_rx/utils/log_to_kml.py | 166 +++++++++++++++--------------------- 1 file changed, 71 insertions(+), 95 deletions(-) diff --git a/auto_rx/utils/log_to_kml.py b/auto_rx/utils/log_to_kml.py index 6c38d030..f8b71497 100644 --- a/auto_rx/utils/log_to_kml.py +++ b/auto_rx/utils/log_to_kml.py @@ -5,16 +5,12 @@ # # 2018-02 Mark Jessop # -# Note: This utility requires the fastkml and shapely libraries, which can be installed using: -# sudo pip install fastkml shapely -# import traceback import argparse import glob -import fastkml +import xml.etree.ElementTree as ET from dateutil.parser import parse -from shapely.geometry import Point, LineString def read_telemetry_csv(filename, @@ -89,108 +85,86 @@ def flight_burst_position(flight_path): def new_placemark(lat, lon, alt, - placemark_id="Placemark ID", name="Placemark Name", absolute=False, icon="http://maps.google.com/mapfiles/kml/shapes/placemark_circle.png", scale=1.0): """ Generate a generic placemark object """ - if absolute: - _alt_mode = 'absolute' - else: - _alt_mode = 'clampToGround' - - flight_icon_style = fastkml.styles.IconStyle( - ns=ns, - icon_href=icon, - scale=scale) + placemark = ET.Element("Placemark") - flight_style = fastkml.styles.Style( - ns=ns, - styles=[flight_icon_style]) + pm_name = ET.SubElement(placemark, "name") + pm_name.text = name - flight_placemark = fastkml.kml.Placemark( - ns=ns, - id=placemark_id, - name=name, - description="", - styles=[flight_style]) + style = ET.SubElement(placemark, "Style") + icon_style = ET.SubElement(style, "IconStyle") + icon_scale = ET.SubElement(icon_style, "scale") + icon_scale.text = str(scale) + pm_icon = ET.SubElement(icon_style, "Icon") + href = ET.SubElement(pm_icon, "href") + href.text = icon - flight_placemark.geometry = fastkml.geometry.Geometry( - ns=ns, - geometry=Point(lon, lat, alt), - altitude_mode=_alt_mode) + point = ET.SubElement(placemark, "Point") + if absolute: + altitude_mode = ET.SubElement(point, "altitudeMode") + altitude_mode.text = "absolute" + coordinates = ET.SubElement(point, "coordinates") + coordinates.text = f"{lon:.6f},{lat:.6f},{alt:.6f}" - return flight_placemark + return placemark def flight_path_to_geometry(flight_path, - placemark_id="Flight Path ID", name="Flight Path Name", track_color="aaffffff", poly_color="20000000", track_width=2.0, absolute=True, - extrude=True, - tessellate=True): - ''' Produce a fastkml geometry object from a flight path array ''' - - # Handle selection of absolute altitude mode + extrude=True): + ''' Produce a placemark object from a flight path array ''' + + placemark = ET.Element("Placemark") + + pm_name = ET.SubElement(placemark, "name") + pm_name.text = name + + style = ET.SubElement(placemark, "Style") + line_style = ET.SubElement(style, "LineStyle") + color = ET.SubElement(line_style, "color") + color.text = track_color + width = ET.SubElement(line_style, "width") + width.text = str(track_width) + poly_style = ET.SubElement(style, "PolyStyle") + color = ET.SubElement(poly_style, "color") + color.text = poly_color + fill = ET.SubElement(poly_style, "fill") + fill.text = "1" + outline = ET.SubElement(poly_style, "outline") + outline.text = "1" + + line_string = ET.SubElement(placemark, "LineString") if absolute: - _alt_mode = 'absolute' + if extrude: + ls_extrude = ET.SubElement(line_string, "extrude") + ls_extrude.text = "1" + altitude_mode = ET.SubElement(line_string, "altitudeMode") + altitude_mode.text = "absolute" else: - _alt_mode = 'clampToGround' - - # Convert the flight path array [time, lat, lon, alt, comment] into a LineString object. - track_points = [] - for _point in flight_path: - # Flight path array is in lat,lon,alt order, needs to be in lon,lat,alt - track_points.append([_point[2], _point[1], _point[3]]) - - _flight_geom = LineString(track_points) + ls_tessellate = ET.SubElement(line_string, "tessellate") + ls_tessellate.text = "1" + coordinates = ET.SubElement(line_string, "coordinates") + coordinates.text = " ".join(f"{lon:.6f},{lat:.6f},{alt:.6f}" for _, lat, lon, alt, _ in flight_path) - # Define the Line and Polygon styles, which are used for the flight path, and the extrusions (if enabled) - flight_track_line_style = fastkml.styles.LineStyle( - ns=ns, - color=track_color, - width=track_width) - - flight_extrusion_style = fastkml.styles.PolyStyle( - ns=ns, - color=poly_color) - - flight_track_style = fastkml.styles.Style( - ns=ns, - styles=[flight_track_line_style, flight_extrusion_style]) - - # Generate the Placemark which will contain the track data. - flight_line = fastkml.kml.Placemark( - ns=ns, - id=placemark_id, - name=name, - styles=[flight_track_style]) - - # Add the track data to the Placemark - flight_line.geometry = fastkml.geometry.Geometry( - ns=ns, - geometry=_flight_geom, - altitude_mode=_alt_mode, - extrude=extrude, - tessellate=tessellate) - - return flight_line + return placemark def write_kml(geom_objects, - filename="output.kml", + kml_file, comment=""): """ Write out flight path geometry objects to a kml file. """ - kml_root = fastkml.kml.KML() - kml_doc = fastkml.kml.Document( - ns=ns, - name=comment) + kml_root = ET.Element("kml", {"xmlns": "http://www.opengis.net/kml/2.2"}) + kml_doc = ET.SubElement(kml_root, "Document") if type(geom_objects) is not list: geom_objects = [geom_objects] @@ -198,13 +172,12 @@ def write_kml(geom_objects, for _flight in geom_objects: kml_doc.append(_flight) - with open(filename, 'w') as kml_file: - kml_file.write(kml_doc.to_string()) - kml_file.close() + tree = ET.ElementTree(kml_root) + tree.write(kml_file, encoding="UTF-8", xml_declaration=True) -def convert_single_file(filename, absolute=True, tessellate=True, last_only=False): - ''' Convert a single sonde log file to a fastkml KML Folder object ''' +def convert_single_file(filename, absolute=True, extrude=True, last_only=False): + ''' Convert a single sonde log file to a KML Folder object ''' # Read file. _flight_data = read_telemetry_csv(filename) @@ -221,16 +194,18 @@ def convert_single_file(filename, absolute=True, tessellate=True, last_only=Fals _burst_pos = flight_burst_position(_flight_data) _landing_pos = _flight_data[-1] - # Generate the placemark & flight track. - _flight_geom = flight_path_to_geometry(_flight_data, name=_track_comment, absolute=absolute, - tessellate=tessellate, extrude=tessellate) - _landing_geom = new_placemark(_landing_pos[1], _landing_pos[2], _landing_pos[3], - name=_landing_comment, absolute=absolute) + _folder = ET.Element("Folder") + _name = ET.SubElement(_folder, "name") + _name.text = _track_comment + _description = ET.SubElement(_folder, "description") + _description.text = "Radiosonde Flight Path" - _folder = fastkml.kml.Folder(ns, _flight_serial, _track_comment, 'Radiosonde Flight Path') + # Generate the placemark & flight track. if not last_only: - _folder.append(_flight_geom) - _folder.append(_landing_geom) + _folder.append(flight_path_to_geometry(_flight_data, name=_track_comment, + absolute=absolute, extrude=extrude)) + _folder.append(new_placemark(_landing_pos[1], _landing_pos[2], _landing_pos[3], + name=_landing_comment, absolute=absolute)) return _folder @@ -258,10 +233,11 @@ def convert_single_file(filename, absolute=True, tessellate=True, last_only=Fals print("Processing: %s" % _file) try: _placemarks.append(convert_single_file(_file, absolute=args.clamp, - tessellate=args.noextrude, last_only=args.lastonly)) + extrude=args.noextrude, last_only=args.lastonly)) except: print("Failed to process: %s" % _file) - write_kml(_placemarks, filename=args.output) + with open(args.output, "wb") as kml_file: + write_kml(_placemarks, kml_file) print("Output saved to: %s" % args.output) From 6446070401c8d2cdd78e7c135884987b475d66b0 Mon Sep 17 00:00:00 2001 From: Clayton Smith Date: Mon, 29 Jan 2024 17:10:11 -0500 Subject: [PATCH 15/42] Add "Generate KML" button to the "Historical" page --- auto_rx/autorx/templates/historical.html | 38 +++++++++++++++ auto_rx/autorx/web.py | 60 ++++++++++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/auto_rx/autorx/templates/historical.html b/auto_rx/autorx/templates/historical.html index b3198beb..decd27d5 100644 --- a/auto_rx/autorx/templates/historical.html +++ b/auto_rx/autorx/templates/historical.html @@ -181,6 +181,7 @@ $("#showsonde-skew").prop('disabled', true); $("#hidesonde-skew").prop('disabled', true); $("#download-logs").prop('disabled', true); + $("#generate-kml").prop('disabled', true); } async function enableMenu () { @@ -194,6 +195,7 @@ $("#showsonde-skew").prop('disabled', false); $("#hidesonde-skew").prop('disabled', false); $("#download-logs").prop('disabled', false); + $("#generate-kml").prop('disabled', false); } if ((window.innerWidth/window.innerHeight) < 1) { @@ -975,6 +977,41 @@ downloadLogs(); }); + function generateKML() { + // Generate a KML file from a set of log files. + selectedrows = table.getSelectedData(); + if (selectedrows.length > 0) { + // Create the list of log files. + _serial_list = []; + for (let i = 0; i < selectedrows.length; i++){ + _serial_list.push(selectedrows[i]['serial']); + } + + if(_serial_list.length>50){ + if (confirm("Warning - downloading lots of log may take some time. Are you sure?")) { + // Just continue on. + } else { + return; + } + } + + if(_serial_list.length == table.getData().length){ + // Request all log files + window.open("generate_kml" , '_blank'); + }else { + // Just request the selected ones. + // Convert the list to JSON, and then to base64 + b64 = btoa(JSON.stringify(_serial_list)); + // Make the request in a new tab + window.open("generate_kml/"+b64 , '_blank'); + } + } + } + + $("#generate-kml").click(function(){ + generateKML(); + }); + // List of available map layers. var Mapnik = L.tileLayer.provider("OpenStreetMap.Mapnik", {edgeBufferTiles: 2}); var DarkMatter = L.tileLayer.provider("CartoDB.DarkMatter", {edgeBufferTiles: 2}); @@ -1472,6 +1509,7 @@

Sonde List

+

diff --git a/auto_rx/autorx/web.py b/auto_rx/autorx/web.py index f1e984e4..13aacb43 100644 --- a/auto_rx/autorx/web.py +++ b/auto_rx/autorx/web.py @@ -8,8 +8,11 @@ import base64 import copy import datetime +import glob +import io import json import logging +import os import random import requests import time @@ -22,6 +25,7 @@ from autorx.utils import check_autorx_versions from autorx.log_files import list_log_files, read_log_by_serial, zip_log_files from autorx.decode import SondeDecoder +from utils.log_to_kml import convert_single_file, write_kml from queue import Queue from threading import Thread import flask @@ -369,6 +373,62 @@ def flask_export_log_files(serialb64=None): logging.error("Web - Error handling Zip request:" + str(e)) abort(400) + +@app.route("/generate_kml") +@app.route("/generate_kml/") +def flask_generate_kml(serialb64=None): + """ + Generate a KML file from a set of log files. + The list of log files is provided in the URL as a base64-encoded JSON list. + """ + + try: + if serialb64: + _serial_list = json.loads(base64.b64decode(serialb64)) + _log_files = [] + for _serial in _serial_list: + _log_mask = os.path.join(autorx.logging_path, f"*_*{_serial}_*_sonde.log") + _matching_files = glob.glob(_log_mask) + + if len(_matching_files) >= 1: + _log_files.append(_matching_files[0]) + else: + _log_mask = os.path.join(autorx.logging_path, "*_sonde.log") + _log_files = glob.glob(_log_mask) + + _placemarks = [] + + for _file in _log_files: + try: + _placemarks.append(convert_single_file(_file, absolute=True, extrude=True, last_only=False)) + except: + pass + + _kml_file = io.BytesIO() + write_kml(_placemarks, _kml_file) + _kml_file.seek(0) + + _ts = datetime.datetime.strftime(datetime.datetime.utcnow(), "%Y%m%d-%H%M%SZ") + + response = make_response( + flask.send_file( + _kml_file, + mimetype="application/vnd.google-earth.kml+xml", + as_attachment=True, + download_name=f"autorx_logfiles_{autorx.config.global_config['habitat_uploader_callsign']}_{_ts}.kml", + ) + ) + + # Add header asking client not to cache the download + response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate" + response.headers["Pragma"] = "no-cache" + + return response + + except Exception as e: + logging.error("Web - Error handling KML request:" + str(e)) + abort(400) + # # Control Endpoints. # From 241376236b2e2917a389db05a15b41d52dd99e1f Mon Sep 17 00:00:00 2001 From: Clayton Smith Date: Mon, 29 Jan 2024 19:24:17 -0500 Subject: [PATCH 16/42] Remove unused code --- auto_rx/utils/log_to_kml.py | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/auto_rx/utils/log_to_kml.py b/auto_rx/utils/log_to_kml.py index f8b71497..35e96655 100644 --- a/auto_rx/utils/log_to_kml.py +++ b/auto_rx/utils/log_to_kml.py @@ -67,23 +67,6 @@ def read_telemetry_csv(filename, return output -def flight_burst_position(flight_path): - ''' Search through flight data for the burst position and return it. ''' - - # Read through array and hunt for max altitude point. - current_alt = 0.0 - current_index = 0 - for i in range(len(flight_path)): - if flight_path[i][3] > current_alt: - current_alt = flight_path[i][3] - current_index = i - - return flight_path[current_index] - - -ns = '{http://www.opengis.net/kml/2.2}' - - def new_placemark(lat, lon, alt, name="Placemark Name", absolute=False, @@ -190,8 +173,7 @@ def convert_single_file(filename, absolute=True, extrude=True, last_only=False): _track_comment = "%s %s" % (_launch_time, _flight_serial) _landing_comment = "%s Last Position" % (_flight_serial) - # Grab burst and last-seen positions - _burst_pos = flight_burst_position(_flight_data) + # Grab last-seen position _landing_pos = _flight_data[-1] _folder = ET.Element("Folder") From 65b1da7476e7f72c34a75493345a1a39f2d93ee3 Mon Sep 17 00:00:00 2001 From: Clayton Smith Date: Mon, 29 Jan 2024 20:51:52 -0500 Subject: [PATCH 17/42] Use autorx.log_files to read logs --- auto_rx/utils/log_to_kml.py | 75 ++++++------------------------------- 1 file changed, 12 insertions(+), 63 deletions(-) diff --git a/auto_rx/utils/log_to_kml.py b/auto_rx/utils/log_to_kml.py index 35e96655..bff0995f 100644 --- a/auto_rx/utils/log_to_kml.py +++ b/auto_rx/utils/log_to_kml.py @@ -6,65 +6,16 @@ # 2018-02 Mark Jessop # -import traceback import argparse import glob +import sys import xml.etree.ElementTree as ET from dateutil.parser import parse +from os.path import dirname, abspath - -def read_telemetry_csv(filename, - datetime_field=0, - latitude_field=3, - longitude_field=4, - altitude_field=5, - delimiter=','): - ''' - Read in a radiosonde_auto_rx generated telemetry CSV file. - Fields to use can be set as arguments to this function. - These have output like the following: - 2017-12-27T23:21:59.560,M2913374,982,-34.95143,138.52471,719.9,-273.0,RS92,401.520 - ,,,,,,,, - - Note that the datetime field must be parsable by dateutil.parsers.parse. - - If any fields are missing, or invalid, this function will return None. - - The output data structure is in the form: - [ - [datetime (as a datetime object), latitude, longitude, altitude, raw_line], - [datetime (as a datetime object), latitude, longitude, altitude, raw_line], - ... - ] - ''' - - output = [] - - f = open(filename, 'r') - - for line in f: - try: - # Split line by comma delimiters. - _fields = line.split(delimiter) - - if _fields[0] == 'timestamp': - # First line in file - header line. - continue - - # Attempt to parse fields. - _datetime = parse(_fields[datetime_field]) - _latitude = float(_fields[latitude_field]) - _longitude = float(_fields[longitude_field]) - _altitude = float(_fields[altitude_field]) - - output.append([_datetime, _latitude, _longitude, _altitude, line]) - except: - traceback.print_exc() - return None - - f.close() - - return output +parent_dir = dirname(dirname(abspath(__file__))) +sys.path.append(parent_dir) +from autorx.log_files import read_log_file def new_placemark(lat, lon, alt, @@ -136,7 +87,7 @@ def flight_path_to_geometry(flight_path, ls_tessellate = ET.SubElement(line_string, "tessellate") ls_tessellate.text = "1" coordinates = ET.SubElement(line_string, "coordinates") - coordinates.text = " ".join(f"{lon:.6f},{lat:.6f},{alt:.6f}" for _, lat, lon, alt, _ in flight_path) + coordinates.text = " ".join(f"{lon:.6f},{lat:.6f},{alt:.6f}" for lat, lon, alt in flight_path) return placemark @@ -163,18 +114,16 @@ def convert_single_file(filename, absolute=True, extrude=True, last_only=False): ''' Convert a single sonde log file to a KML Folder object ''' # Read file. - _flight_data = read_telemetry_csv(filename) + _flight_data = read_log_file(filename) - # Extract the flight's serial number and launch time from the first line in the file. - _first_line = _flight_data[0][4] - _flight_serial = _first_line.split(',')[1] # Serial number is the second field in the line. - _launch_time = _flight_data[0][0].strftime("%Y%m%d-%H%M%SZ") + _flight_serial = _flight_data["serial"] + _launch_time = parse(_flight_data["first_time"]).strftime("%Y%m%d-%H%M%SZ") # Generate a comment line to use in the folder and placemark descriptions _track_comment = "%s %s" % (_launch_time, _flight_serial) _landing_comment = "%s Last Position" % (_flight_serial) # Grab last-seen position - _landing_pos = _flight_data[-1] + _landing_pos = _flight_data["path"][-1] _folder = ET.Element("Folder") _name = ET.SubElement(_folder, "name") @@ -184,9 +133,9 @@ def convert_single_file(filename, absolute=True, extrude=True, last_only=False): # Generate the placemark & flight track. if not last_only: - _folder.append(flight_path_to_geometry(_flight_data, name=_track_comment, + _folder.append(flight_path_to_geometry(_flight_data["path"], name=_track_comment, absolute=absolute, extrude=extrude)) - _folder.append(new_placemark(_landing_pos[1], _landing_pos[2], _landing_pos[3], + _folder.append(new_placemark(_landing_pos[0], _landing_pos[1], _landing_pos[2], name=_landing_comment, absolute=absolute)) return _folder From e227eea6b7d3dfd396ff811298e423c10235ab7f Mon Sep 17 00:00:00 2001 From: Clayton Smith Date: Mon, 29 Jan 2024 21:17:25 -0500 Subject: [PATCH 18/42] Move KML generation into autorx.log_files --- auto_rx/autorx/log_files.py | 124 ++++++++++++++++++++++++++++++++ auto_rx/autorx/web.py | 13 +--- auto_rx/utils/log_to_kml.py | 140 +----------------------------------- 3 files changed, 129 insertions(+), 148 deletions(-) diff --git a/auto_rx/autorx/log_files.py b/auto_rx/autorx/log_files.py index d40a5ed8..ea1551cb 100644 --- a/auto_rx/autorx/log_files.py +++ b/auto_rx/autorx/log_files.py @@ -14,6 +14,7 @@ import os.path import time import zipfile +import xml.etree.ElementTree as ET import numpy as np @@ -521,6 +522,129 @@ def zip_log_files(serial_list=None): return data +def _coordinates_to_kml_placemark(lat, lon, alt, + name="Placemark Name", + absolute=False, + icon="http://maps.google.com/mapfiles/kml/shapes/placemark_circle.png", + scale=1.0): + """ Generate a generic placemark object """ + + placemark = ET.Element("Placemark") + + pm_name = ET.SubElement(placemark, "name") + pm_name.text = name + + style = ET.SubElement(placemark, "Style") + icon_style = ET.SubElement(style, "IconStyle") + icon_scale = ET.SubElement(icon_style, "scale") + icon_scale.text = str(scale) + pm_icon = ET.SubElement(icon_style, "Icon") + href = ET.SubElement(pm_icon, "href") + href.text = icon + + point = ET.SubElement(placemark, "Point") + if absolute: + altitude_mode = ET.SubElement(point, "altitudeMode") + altitude_mode.text = "absolute" + coordinates = ET.SubElement(point, "coordinates") + coordinates.text = f"{lon:.6f},{lat:.6f},{alt:.6f}" + + return placemark + + +def _flight_path_to_kml_placemark(flight_path, + name="Flight Path Name", + track_color="aaffffff", + poly_color="20000000", + track_width=2.0, + absolute=True, + extrude=True): + ''' Produce a placemark object from a flight path array ''' + + placemark = ET.Element("Placemark") + + pm_name = ET.SubElement(placemark, "name") + pm_name.text = name + + style = ET.SubElement(placemark, "Style") + line_style = ET.SubElement(style, "LineStyle") + color = ET.SubElement(line_style, "color") + color.text = track_color + width = ET.SubElement(line_style, "width") + width.text = str(track_width) + poly_style = ET.SubElement(style, "PolyStyle") + color = ET.SubElement(poly_style, "color") + color.text = poly_color + fill = ET.SubElement(poly_style, "fill") + fill.text = "1" + outline = ET.SubElement(poly_style, "outline") + outline.text = "1" + + line_string = ET.SubElement(placemark, "LineString") + if absolute: + if extrude: + ls_extrude = ET.SubElement(line_string, "extrude") + ls_extrude.text = "1" + altitude_mode = ET.SubElement(line_string, "altitudeMode") + altitude_mode.text = "absolute" + else: + ls_tessellate = ET.SubElement(line_string, "tessellate") + ls_tessellate.text = "1" + coordinates = ET.SubElement(line_string, "coordinates") + coordinates.text = " ".join(f"{lon:.6f},{lat:.6f},{alt:.6f}" for lat, lon, alt in flight_path) + + return placemark + + +def _log_file_to_kml_folder(filename, absolute=True, extrude=True, last_only=False): + ''' Convert a single sonde log file to a KML Folder object ''' + + # Read file. + _flight_data = read_log_file(filename) + + _flight_serial = _flight_data["serial"] + _launch_time = parse(_flight_data["first_time"]).strftime("%Y%m%d-%H%M%SZ") + # Generate a comment line to use in the folder and placemark descriptions + _track_comment = "%s %s" % (_launch_time, _flight_serial) + _landing_comment = "%s Last Position" % (_flight_serial) + + # Grab last-seen position + _landing_pos = _flight_data["path"][-1] + + _folder = ET.Element("Folder") + _name = ET.SubElement(_folder, "name") + _name.text = _track_comment + _description = ET.SubElement(_folder, "description") + _description.text = "Radiosonde Flight Path" + + # Generate the placemark & flight track. + if not last_only: + _folder.append(_flight_path_to_kml_placemark(_flight_data["path"], name=_track_comment, + absolute=absolute, extrude=extrude)) + _folder.append(_coordinates_to_kml_placemark(_landing_pos[0], _landing_pos[1], _landing_pos[2], + name=_landing_comment, absolute=absolute)) + + return _folder + + +def log_files_to_kml(file_list, kml_file, absolute=True, extrude=True, last_only=False): + """ Convert a collection of log files to a KML file """ + + kml_root = ET.Element("kml", {"xmlns": "http://www.opengis.net/kml/2.2"}) + kml_doc = ET.SubElement(kml_root, "Document") + + for file in file_list: + print("Processing: %s" % file) + try: + kml_doc.append(_log_file_to_kml_folder(file, absolute=absolute, + extrude=extrude, last_only=last_only)) + except: + print("Failed to process: %s" % file) + + tree = ET.ElementTree(kml_root) + tree.write(kml_file, encoding="UTF-8", xml_declaration=True) + + if __name__ == "__main__": import sys import json diff --git a/auto_rx/autorx/web.py b/auto_rx/autorx/web.py index 13aacb43..e4f96c11 100644 --- a/auto_rx/autorx/web.py +++ b/auto_rx/autorx/web.py @@ -23,9 +23,8 @@ import autorx.scan from autorx.geometry import GenericTrack from autorx.utils import check_autorx_versions -from autorx.log_files import list_log_files, read_log_by_serial, zip_log_files +from autorx.log_files import list_log_files, read_log_by_serial, zip_log_files, log_files_to_kml from autorx.decode import SondeDecoder -from utils.log_to_kml import convert_single_file, write_kml from queue import Queue from threading import Thread import flask @@ -396,16 +395,8 @@ def flask_generate_kml(serialb64=None): _log_mask = os.path.join(autorx.logging_path, "*_sonde.log") _log_files = glob.glob(_log_mask) - _placemarks = [] - - for _file in _log_files: - try: - _placemarks.append(convert_single_file(_file, absolute=True, extrude=True, last_only=False)) - except: - pass - _kml_file = io.BytesIO() - write_kml(_placemarks, _kml_file) + log_files_to_kml(_log_files, _kml_file) _kml_file.seek(0) _ts = datetime.datetime.strftime(datetime.datetime.utcnow(), "%Y%m%d-%H%M%SZ") diff --git a/auto_rx/utils/log_to_kml.py b/auto_rx/utils/log_to_kml.py index bff0995f..e6729b2e 100644 --- a/auto_rx/utils/log_to_kml.py +++ b/auto_rx/utils/log_to_kml.py @@ -9,136 +9,11 @@ import argparse import glob import sys -import xml.etree.ElementTree as ET -from dateutil.parser import parse from os.path import dirname, abspath parent_dir = dirname(dirname(abspath(__file__))) sys.path.append(parent_dir) -from autorx.log_files import read_log_file - - -def new_placemark(lat, lon, alt, - name="Placemark Name", - absolute=False, - icon="http://maps.google.com/mapfiles/kml/shapes/placemark_circle.png", - scale=1.0): - """ Generate a generic placemark object """ - - placemark = ET.Element("Placemark") - - pm_name = ET.SubElement(placemark, "name") - pm_name.text = name - - style = ET.SubElement(placemark, "Style") - icon_style = ET.SubElement(style, "IconStyle") - icon_scale = ET.SubElement(icon_style, "scale") - icon_scale.text = str(scale) - pm_icon = ET.SubElement(icon_style, "Icon") - href = ET.SubElement(pm_icon, "href") - href.text = icon - - point = ET.SubElement(placemark, "Point") - if absolute: - altitude_mode = ET.SubElement(point, "altitudeMode") - altitude_mode.text = "absolute" - coordinates = ET.SubElement(point, "coordinates") - coordinates.text = f"{lon:.6f},{lat:.6f},{alt:.6f}" - - return placemark - - -def flight_path_to_geometry(flight_path, - name="Flight Path Name", - track_color="aaffffff", - poly_color="20000000", - track_width=2.0, - absolute=True, - extrude=True): - ''' Produce a placemark object from a flight path array ''' - - placemark = ET.Element("Placemark") - - pm_name = ET.SubElement(placemark, "name") - pm_name.text = name - - style = ET.SubElement(placemark, "Style") - line_style = ET.SubElement(style, "LineStyle") - color = ET.SubElement(line_style, "color") - color.text = track_color - width = ET.SubElement(line_style, "width") - width.text = str(track_width) - poly_style = ET.SubElement(style, "PolyStyle") - color = ET.SubElement(poly_style, "color") - color.text = poly_color - fill = ET.SubElement(poly_style, "fill") - fill.text = "1" - outline = ET.SubElement(poly_style, "outline") - outline.text = "1" - - line_string = ET.SubElement(placemark, "LineString") - if absolute: - if extrude: - ls_extrude = ET.SubElement(line_string, "extrude") - ls_extrude.text = "1" - altitude_mode = ET.SubElement(line_string, "altitudeMode") - altitude_mode.text = "absolute" - else: - ls_tessellate = ET.SubElement(line_string, "tessellate") - ls_tessellate.text = "1" - coordinates = ET.SubElement(line_string, "coordinates") - coordinates.text = " ".join(f"{lon:.6f},{lat:.6f},{alt:.6f}" for lat, lon, alt in flight_path) - - return placemark - - -def write_kml(geom_objects, - kml_file, - comment=""): - """ Write out flight path geometry objects to a kml file. """ - - kml_root = ET.Element("kml", {"xmlns": "http://www.opengis.net/kml/2.2"}) - kml_doc = ET.SubElement(kml_root, "Document") - - if type(geom_objects) is not list: - geom_objects = [geom_objects] - - for _flight in geom_objects: - kml_doc.append(_flight) - - tree = ET.ElementTree(kml_root) - tree.write(kml_file, encoding="UTF-8", xml_declaration=True) - - -def convert_single_file(filename, absolute=True, extrude=True, last_only=False): - ''' Convert a single sonde log file to a KML Folder object ''' - - # Read file. - _flight_data = read_log_file(filename) - - _flight_serial = _flight_data["serial"] - _launch_time = parse(_flight_data["first_time"]).strftime("%Y%m%d-%H%M%SZ") - # Generate a comment line to use in the folder and placemark descriptions - _track_comment = "%s %s" % (_launch_time, _flight_serial) - _landing_comment = "%s Last Position" % (_flight_serial) - - # Grab last-seen position - _landing_pos = _flight_data["path"][-1] - - _folder = ET.Element("Folder") - _name = ET.SubElement(_folder, "name") - _name.text = _track_comment - _description = ET.SubElement(_folder, "description") - _description.text = "Radiosonde Flight Path" - - # Generate the placemark & flight track. - if not last_only: - _folder.append(flight_path_to_geometry(_flight_data["path"], name=_track_comment, - absolute=absolute, extrude=extrude)) - _folder.append(new_placemark(_landing_pos[0], _landing_pos[1], _landing_pos[2], - name=_landing_comment, absolute=absolute)) - - return _folder +from autorx.log_files import log_files_to_kml if __name__ == "__main__": @@ -158,17 +33,8 @@ def convert_single_file(filename, absolute=True, extrude=True, last_only=False): _file_list = glob.glob(args.input) - _placemarks = [] - - for _file in _file_list: - print("Processing: %s" % _file) - try: - _placemarks.append(convert_single_file(_file, absolute=args.clamp, - extrude=args.noextrude, last_only=args.lastonly)) - except: - print("Failed to process: %s" % _file) - with open(args.output, "wb") as kml_file: - write_kml(_placemarks, kml_file) + log_files_to_kml(_file_list, kml_file, absolute=args.clamp, + extrude=args.noextrude, last_only=args.lastonly) print("Output saved to: %s" % args.output) From 209a78ab90c00a8c3b78c3ea0ab469afb06ffdec Mon Sep 17 00:00:00 2001 From: Clayton Smith Date: Mon, 29 Jan 2024 22:30:02 -0500 Subject: [PATCH 19/42] Replace print statements with logging --- auto_rx/autorx/log_files.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/auto_rx/autorx/log_files.py b/auto_rx/autorx/log_files.py index ea1551cb..477e3a51 100644 --- a/auto_rx/autorx/log_files.py +++ b/auto_rx/autorx/log_files.py @@ -634,12 +634,12 @@ def log_files_to_kml(file_list, kml_file, absolute=True, extrude=True, last_only kml_doc = ET.SubElement(kml_root, "Document") for file in file_list: - print("Processing: %s" % file) + logging.debug(f"Converting {file} to KML") try: kml_doc.append(_log_file_to_kml_folder(file, absolute=absolute, extrude=extrude, last_only=last_only)) - except: - print("Failed to process: %s" % file) + except Exception: + logging.exception(f"Failed to convert {file} to KML") tree = ET.ElementTree(kml_root) tree.write(kml_file, encoding="UTF-8", xml_declaration=True) From 58a183dffad534858550cd6fe390677a17d15466 Mon Sep 17 00:00:00 2001 From: Clayton Smith Date: Tue, 30 Jan 2024 20:12:56 -0500 Subject: [PATCH 20/42] Generate live KML using ElementTree --- auto_rx/autorx/config.py | 2 +- auto_rx/autorx/log_files.py | 63 +++++++------- auto_rx/autorx/web.py | 161 ++++++++++++++++++++---------------- auto_rx/requirements.txt | 1 - 4 files changed, 119 insertions(+), 108 deletions(-) diff --git a/auto_rx/autorx/config.py b/auto_rx/autorx/config.py index bfc45a89..21153b4b 100644 --- a/auto_rx/autorx/config.py +++ b/auto_rx/autorx/config.py @@ -564,7 +564,7 @@ def read_auto_rx_config(filename, no_sdr_test=False): logging.warning( "Config - Did not find kml_refresh_rate setting, using default (10 seconds)." ) - auto_rx_config["kml_refresh_rate"] = 11 + auto_rx_config["kml_refresh_rate"] = 10 # New Sondehub db Settings try: diff --git a/auto_rx/autorx/log_files.py b/auto_rx/autorx/log_files.py index 477e3a51..d784488d 100644 --- a/auto_rx/autorx/log_files.py +++ b/auto_rx/autorx/log_files.py @@ -522,17 +522,20 @@ def zip_log_files(serial_list=None): return data -def _coordinates_to_kml_placemark(lat, lon, alt, - name="Placemark Name", - absolute=False, - icon="http://maps.google.com/mapfiles/kml/shapes/placemark_circle.png", - scale=1.0): +def coordinates_to_kml_placemark(lat, lon, alt, + name="Placemark Name", + description="Placemark Description", + absolute=False, + icon="https://maps.google.com/mapfiles/kml/shapes/placemark_circle.png", + scale=1.0): """ Generate a generic placemark object """ placemark = ET.Element("Placemark") pm_name = ET.SubElement(placemark, "name") pm_name.text = name + pm_desc = ET.SubElement(placemark, "description") + pm_desc.text = description style = ET.SubElement(placemark, "Style") icon_style = ET.SubElement(style, "IconStyle") @@ -552,13 +555,13 @@ def _coordinates_to_kml_placemark(lat, lon, alt, return placemark -def _flight_path_to_kml_placemark(flight_path, - name="Flight Path Name", - track_color="aaffffff", - poly_color="20000000", - track_width=2.0, - absolute=True, - extrude=True): +def path_to_kml_placemark(flight_path, + name="Flight Path Name", + track_color="ff03bafc", + poly_color="8003bafc", + track_width=2.0, + absolute=True, + extrude=True): ''' Produce a placemark object from a flight path array ''' placemark = ET.Element("Placemark") @@ -572,13 +575,14 @@ def _flight_path_to_kml_placemark(flight_path, color.text = track_color width = ET.SubElement(line_style, "width") width.text = str(track_width) - poly_style = ET.SubElement(style, "PolyStyle") - color = ET.SubElement(poly_style, "color") - color.text = poly_color - fill = ET.SubElement(poly_style, "fill") - fill.text = "1" - outline = ET.SubElement(poly_style, "outline") - outline.text = "1" + if extrude: + poly_style = ET.SubElement(style, "PolyStyle") + color = ET.SubElement(poly_style, "color") + color.text = poly_color + fill = ET.SubElement(poly_style, "fill") + fill.text = "1" + outline = ET.SubElement(poly_style, "outline") + outline.text = "1" line_string = ET.SubElement(placemark, "LineString") if absolute: @@ -603,26 +607,19 @@ def _log_file_to_kml_folder(filename, absolute=True, extrude=True, last_only=Fal _flight_data = read_log_file(filename) _flight_serial = _flight_data["serial"] - _launch_time = parse(_flight_data["first_time"]).strftime("%Y%m%d-%H%M%SZ") - # Generate a comment line to use in the folder and placemark descriptions - _track_comment = "%s %s" % (_launch_time, _flight_serial) - _landing_comment = "%s Last Position" % (_flight_serial) - - # Grab last-seen position + _landing_time = _flight_data["last_time"] _landing_pos = _flight_data["path"][-1] _folder = ET.Element("Folder") _name = ET.SubElement(_folder, "name") - _name.text = _track_comment - _description = ET.SubElement(_folder, "description") - _description.text = "Radiosonde Flight Path" + _name.text = _flight_serial # Generate the placemark & flight track. + _folder.append(coordinates_to_kml_placemark(_landing_pos[0], _landing_pos[1], _landing_pos[2], + name=_flight_serial, description=_landing_time, absolute=absolute)) if not last_only: - _folder.append(_flight_path_to_kml_placemark(_flight_data["path"], name=_track_comment, - absolute=absolute, extrude=extrude)) - _folder.append(_coordinates_to_kml_placemark(_landing_pos[0], _landing_pos[1], _landing_pos[2], - name=_landing_comment, absolute=absolute)) + _folder.append(path_to_kml_placemark(_flight_data["path"], name="Track", + absolute=absolute, extrude=extrude)) return _folder @@ -630,7 +627,7 @@ def _log_file_to_kml_folder(filename, absolute=True, extrude=True, last_only=Fal def log_files_to_kml(file_list, kml_file, absolute=True, extrude=True, last_only=False): """ Convert a collection of log files to a KML file """ - kml_root = ET.Element("kml", {"xmlns": "http://www.opengis.net/kml/2.2"}) + kml_root = ET.Element("kml", xmlns="http://www.opengis.net/kml/2.2") kml_doc = ET.SubElement(kml_root, "Document") for file in file_list: diff --git a/auto_rx/autorx/web.py b/auto_rx/autorx/web.py index e4f96c11..b0266890 100644 --- a/auto_rx/autorx/web.py +++ b/auto_rx/autorx/web.py @@ -18,12 +18,20 @@ import time import traceback import sys +import xml.etree.ElementTree as ET import autorx import autorx.config import autorx.scan from autorx.geometry import GenericTrack from autorx.utils import check_autorx_versions -from autorx.log_files import list_log_files, read_log_by_serial, zip_log_files, log_files_to_kml +from autorx.log_files import ( + list_log_files, + read_log_by_serial, + zip_log_files, + log_files_to_kml, + coordinates_to_kml_placemark, + path_to_kml_placemark +) from autorx.decode import SondeDecoder from queue import Queue from threading import Thread @@ -31,15 +39,6 @@ from flask import request, abort, make_response, send_file from flask_socketio import SocketIO from werkzeug.middleware.proxy_fix import ProxyFix -import re - -try: - from simplekml import Kml, AltitudeMode -except ImportError: - print( - "Could not import simplekml! Try running: sudo pip3 install -r requirements.txt" - ) - sys.exit(1) # Inhibit Flask warning message about running a development server... (we know!) @@ -149,47 +148,60 @@ def flask_get_task_list(): def flask_get_kml(): """ Return KML with autorefresh """ - _config = autorx.config.global_config - kml = Kml() - netlink = kml.newnetworklink(name="Radiosonde Auto-RX Live Telemetry") - netlink.open = 1 - netlink.link.href = flask.request.url_root + "rs_feed.kml" - try: - netlink.link.refreshinterval = _config["kml_refresh_rate"] - except KeyError: - netlink.link.refreshinterval = 10 - netlink.link.refreshmode = "onInterval" - return kml.kml(), 200, {"content-type": "application/vnd.google-earth.kml+xml"} + kml_root = ET.Element("kml", xmlns="http://www.opengis.net/kml/2.2") + kml_doc = ET.SubElement(kml_root, "Document") + + network_link = ET.SubElement(kml_doc, "NetworkLink") + + name = ET.SubElement(network_link, "name") + name.text = "Radiosonde Auto-RX Live Telemetry" + + open = ET.SubElement(network_link, "open") + open.text = "1" + + link = ET.SubElement(network_link, "Link") + + href = ET.SubElement(link, "href") + href.text = flask.request.url_root + "rs_feed.kml" + + refresh_mode = ET.SubElement(link, "refreshMode") + refresh_mode.text = "onInterval" + + refresh_interval = ET.SubElement(link, "refreshInterval") + refresh_interval.text = str(autorx.config.global_config["kml_refresh_rate"]) + + kml_string = ET.tostring(kml_root, encoding="UTF-8", xml_declaration=True) + return kml_string, 200, {"content-type": "application/vnd.google-earth.kml+xml"} @app.route("/rs_feed.kml") def flask_get_kml_feed(): """ Return KML with RS telemetry """ - kml = Kml() - kml.resetidcounter() - kml.document.name = "Track" - kml.document.open = 1 + kml_root = ET.Element("kml", xmlns="http://www.opengis.net/kml/2.2") + kml_doc = ET.SubElement(kml_root, "Document") + + name = ET.SubElement(kml_doc, "name") + name.text = "Track" + open = ET.SubElement(kml_doc, "open") + open.text = "1" + # Station Placemark - pnt = kml.newpoint( - name="Ground Station", - altitudemode=AltitudeMode.absolute, + kml_doc.append(coordinates_to_kml_placemark( + autorx.config.global_config["station_lat"], + autorx.config.global_config["station_lon"], + autorx.config.global_config["station_alt"], + name=autorx.config.global_config["habitat_uploader_callsign"], description="AutoRX Ground Station", - ) - pnt.open = 1 - pnt.iconstyle.icon.href = flask.request.url_root + "static/img/antenna-green.png" - pnt.coords = [ - ( - autorx.config.global_config["station_lon"], - autorx.config.global_config["station_lat"], - autorx.config.global_config["station_alt"], - ) - ] + absolute=True, + icon=flask.request.url_root + "static/img/antenna-green.png" + )) + for rs_id in flask_telemetry_store: try: coordinates = [] for tp in flask_telemetry_store[rs_id]["track"].track_history: - coordinates.append((tp[2], tp[1], tp[3])) + coordinates.append((tp[1], tp[2], tp[3])) rs_data = """\ {type}/{subtype} @@ -208,56 +220,59 @@ def flask_get_kml_feed(): icon = flask.request.url_root + "static/img/parachute-green.png" # Add folder - fol = kml.newfolder(name=rs_id) + folder = ET.SubElement(kml_doc, "Folder", id=f"folder_{rs_id}") + name = ET.SubElement(folder, "name") + name.text = rs_id + open = ET.SubElement(folder, "open") + open.text = "1" + # HAB Placemark - pnt = fol.newpoint( + folder.append(coordinates_to_kml_placemark( + flask_telemetry_store[rs_id]["latest_telem"]["lat"], + flask_telemetry_store[rs_id]["latest_telem"]["lon"], + flask_telemetry_store[rs_id]["latest_telem"]["alt"], name=rs_id, - altitudemode=AltitudeMode.absolute, - description=rs_data.format( - **flask_telemetry_store[rs_id]["latest_telem"] - ), - ) - pnt.iconstyle.icon.href = icon - pnt.coords = [ - ( - flask_telemetry_store[rs_id]["latest_telem"]["lon"], - flask_telemetry_store[rs_id]["latest_telem"]["lat"], - flask_telemetry_store[rs_id]["latest_telem"]["alt"], - ) - ] - linestring = fol.newlinestring(name="Track") - linestring.coords = coordinates - linestring.altitudemode = AltitudeMode.absolute - linestring.extrude = 1 - linestring.stylemap.normalstyle.linestyle.color = "ff03bafc" - linestring.stylemap.highlightstyle.linestyle.color = "ff03bafc" - linestring.stylemap.normalstyle.polystyle.color = "AA03bafc" - linestring.stylemap.highlightstyle.polystyle.color = "CC03bafc" - # Add LOS line - linestring = fol.newlinestring(name="LOS") - linestring.altitudemode = AltitudeMode.absolute - linestring.coords = [ + description=rs_data.format(**flask_telemetry_store[rs_id]["latest_telem"]), + absolute=True, + icon=icon + )) + + # Track + folder.append(path_to_kml_placemark( + coordinates, + name="Track", + absolute=True, + extrude=True + )) + + # LOS line + coordinates = [ ( - autorx.config.global_config["station_lon"], autorx.config.global_config["station_lat"], + autorx.config.global_config["station_lon"], autorx.config.global_config["station_alt"], ), ( - flask_telemetry_store[rs_id]["latest_telem"]["lon"], flask_telemetry_store[rs_id]["latest_telem"]["lat"], + flask_telemetry_store[rs_id]["latest_telem"]["lon"], flask_telemetry_store[rs_id]["latest_telem"]["alt"], ), ] + folder.append(path_to_kml_placemark( + coordinates, + name="LOS", + track_color="ffffffff", + absolute=True, + extrude=False + )) + except Exception as e: logging.error( "KML - Could not parse data from RS %s - %s" % (rs_id, str(e)) ) - return ( - re.sub("", "", kml.kml()), - 200, - {"content-type": "application/vnd.google-earth.kml+xml"}, - ) + kml_string = ET.tostring(kml_root, encoding="UTF-8", xml_declaration=True) + return kml_string, 200, {"content-type": "application/vnd.google-earth.kml+xml"} @app.route("/get_config") diff --git a/auto_rx/requirements.txt b/auto_rx/requirements.txt index 3f1bb376..347a1055 100644 --- a/auto_rx/requirements.txt +++ b/auto_rx/requirements.txt @@ -5,5 +5,4 @@ flask-socketio numpy requests semver -simplekml simple-websocket From 5e2c50ce20e153d7cf52d0ec5ab030ad1b89e3d9 Mon Sep 17 00:00:00 2001 From: Clayton Smith Date: Tue, 30 Jan 2024 22:19:50 -0500 Subject: [PATCH 21/42] Show recent sondes at the top of the list --- auto_rx/autorx/web.py | 1 + auto_rx/utils/log_to_kml.py | 1 + 2 files changed, 2 insertions(+) diff --git a/auto_rx/autorx/web.py b/auto_rx/autorx/web.py index b0266890..341edb5b 100644 --- a/auto_rx/autorx/web.py +++ b/auto_rx/autorx/web.py @@ -411,6 +411,7 @@ def flask_generate_kml(serialb64=None): _log_files = glob.glob(_log_mask) _kml_file = io.BytesIO() + _log_files.sort(reverse=True) log_files_to_kml(_log_files, _kml_file) _kml_file.seek(0) diff --git a/auto_rx/utils/log_to_kml.py b/auto_rx/utils/log_to_kml.py index e6729b2e..d7537c7f 100644 --- a/auto_rx/utils/log_to_kml.py +++ b/auto_rx/utils/log_to_kml.py @@ -32,6 +32,7 @@ args = parser.parse_args() _file_list = glob.glob(args.input) + _file_list.sort(reverse=True) with open(args.output, "wb") as kml_file: log_files_to_kml(_file_list, kml_file, absolute=args.clamp, From 765947c5500cfae152c5243b5e60330ef4ff1831 Mon Sep 17 00:00:00 2001 From: Clayton Smith Date: Wed, 31 Jan 2024 00:26:21 -0500 Subject: [PATCH 22/42] Remove unused Habitat code --- auto_rx/auto_rx.py | 25 - auto_rx/autorx/config.py | 35 +- auto_rx/autorx/decode.py | 7 +- auto_rx/autorx/habitat.py | 875 ---------------------------- auto_rx/autorx/templates/index.html | 11 - auto_rx/requirements.txt | 1 - 6 files changed, 5 insertions(+), 949 deletions(-) delete mode 100644 auto_rx/autorx/habitat.py diff --git a/auto_rx/auto_rx.py b/auto_rx/auto_rx.py index 99481433..977783be 100644 --- a/auto_rx/auto_rx.py +++ b/auto_rx/auto_rx.py @@ -38,7 +38,6 @@ from autorx.decode import SondeDecoder, VALID_SONDE_TYPES, DRIFTY_SONDE_TYPES from autorx.logger import TelemetryLogger from autorx.email_notification import EmailNotification -from autorx.habitat import HabitatUploader from autorx.aprs import APRSUploader from autorx.ozimux import OziUploader from autorx.sondehub import SondehubUploader @@ -933,30 +932,6 @@ def main(): exporter_objects.append(_email_notification) exporter_functions.append(_email_notification.add) - # Habitat Uploader - DEPRECATED - Sondehub DB now in use (>1.5.0) - # if config["habitat_enabled"]: - - # if config["habitat_upload_listener_position"] is False: - # _habitat_station_position = None - # else: - # _habitat_station_position = ( - # config["station_lat"], - # config["station_lon"], - # config["station_alt"], - # ) - - # _habitat = HabitatUploader( - # user_callsign=config["habitat_uploader_callsign"], - # user_antenna=config["habitat_uploader_antenna"], - # station_position=_habitat_station_position, - # synchronous_upload_time=config["habitat_upload_rate"], - # callsign_validity_threshold=config["payload_id_valid"], - # url=config["habitat_url"], - # ) - - # exporter_objects.append(_habitat) - # exporter_functions.append(_habitat.add) - # APRS Uploader if config["aprs_enabled"]: diff --git a/auto_rx/autorx/config.py b/auto_rx/autorx/config.py index bfc45a89..e47a8f4f 100644 --- a/auto_rx/autorx/config.py +++ b/auto_rx/autorx/config.py @@ -27,11 +27,10 @@ # Web interface credentials web_password = "none" -# Fixed minimum update rates for APRS & Habitat -# These are set to avoid congestion on the APRS-IS network, and on the Habitat server -# Please respect other users of these networks and leave these settings as they are. +# Fixed minimum update rate for APRS +# This is set to avoid congestion on the APRS-IS network +# Please respect other users of the network and leave this setting as it is. MINIMUM_APRS_UPDATE_RATE = 30 -MINIMUM_HABITAT_UPDATE_RATE = 30 def read_auto_rx_config(filename, no_sdr_test=False): @@ -98,12 +97,9 @@ def read_auto_rx_config(filename, no_sdr_test=False): "radius_temporary_block": False, # "sonde_time_threshold": 3, # Commented out to ensure warning message is shown. # Habitat Settings - "habitat_enabled": False, - "habitat_upload_rate": 30, "habitat_uploader_callsign": "SONDE_AUTO_RX", "habitat_uploader_antenna": "1/4-wave", "habitat_upload_listener_position": False, - "habitat_payload_callsign": "", # APRS Settings "aprs_enabled": False, "aprs_upload_rate": 30, @@ -166,12 +162,6 @@ def read_auto_rx_config(filename, no_sdr_test=False): "save_system_log": False, "enable_debug_logging": False, "save_cal_data": False, - # URL for the Habitat DB Server. - # As of July 2018 we send via sondehub.org, which will allow us to eventually transition away - # from using the habhub.org tracker, and leave it for use by High-Altitude Balloon Hobbyists. - # For now, sondehub.org just acts as a proxy to habhub.org. - # This setting is not exposed to users as it's only used for unit/int testing - "habitat_url": "https://habitat.sondehub.org/", # New Sondehub DB Settings "sondehub_enabled": True, "sondehub_upload_rate": 30, @@ -298,12 +288,6 @@ def read_auto_rx_config(filename, no_sdr_test=False): auto_rx_config["max_altitude"] = config.getint("filtering", "max_altitude") auto_rx_config["max_radius_km"] = config.getint("filtering", "max_radius_km") - # Habitat Settings - # Deprecated from v1.5.0 - # auto_rx_config["habitat_enabled"] = config.getboolean( - # "habitat", "habitat_enabled" - # ) - # auto_rx_config["habitat_upload_rate"] = config.getint("habitat", "upload_rate") auto_rx_config["habitat_uploader_callsign"] = config.get( "habitat", "uploader_callsign" ) @@ -314,19 +298,6 @@ def read_auto_rx_config(filename, no_sdr_test=False): "habitat", "uploader_antenna" ).strip() - # try: # Use the default configuration if not found - # auto_rx_config["habitat_url"] = config.get("habitat", "url") - # except: - # pass - - # Deprecated from v1.5.0 - # if auto_rx_config["habitat_upload_rate"] < MINIMUM_HABITAT_UPDATE_RATE: - # logging.warning( - # "Config - Habitat Update Rate clipped to minimum of %d seconds. Please be respectful of other users of Habitat." - # % MINIMUM_HABITAT_UPDATE_RATE - # ) - # auto_rx_config["habitat_upload_rate"] = MINIMUM_HABITAT_UPDATE_RATE - # APRS Settings auto_rx_config["aprs_enabled"] = config.getboolean("aprs", "aprs_enabled") auto_rx_config["aprs_upload_rate"] = config.getint("aprs", "upload_rate") diff --git a/auto_rx/autorx/decode.py b/auto_rx/autorx/decode.py index ed4e0b71..88969e94 100644 --- a/auto_rx/autorx/decode.py +++ b/auto_rx/autorx/decode.py @@ -1871,7 +1871,6 @@ def running(self): if __name__ == "__main__": # Test script. from .logger import TelemetryLogger - from .habitat import HabitatUploader logging.basicConfig( format="%(asctime)s %(levelname)s:%(message)s", level=logging.DEBUG @@ -1883,7 +1882,6 @@ def running(self): urllib3_log.setLevel(logging.CRITICAL) _log = TelemetryLogger(log_directory="./testlog/") - _habitat = HabitatUploader(user_callsign="VK5QI_AUTO_RX_DEV", inhibit=False) try: _decoder = SondeDecoder( @@ -1891,14 +1889,14 @@ def running(self): sonde_type="RS41", timeout=50, rtl_device_idx="00000002", - exporter=[_habitat.add, _log.add], + exporter=[_log.add], ) # _decoder2 = SondeDecoder(sonde_freq = 405.5*1e6, # sonde_type = "RS41", # timeout = 50, # rtl_device_idx="00000001", - # exporter=[_habitat.add, _log.add]) + # exporter=[_log.add]) while True: time.sleep(5) @@ -1911,5 +1909,4 @@ def running(self): traceback.print_exc() pass - _habitat.close() _log.close() diff --git a/auto_rx/autorx/habitat.py b/auto_rx/autorx/habitat.py deleted file mode 100644 index f76c4299..00000000 --- a/auto_rx/autorx/habitat.py +++ /dev/null @@ -1,875 +0,0 @@ -#!/usr/bin/env python -# -# radiosonde_auto_rx - Habitat Exporter -# -# Copyright (C) 2018 Mark Jessop -# Released under GNU GPL v3 or later -# -import crcmod -import datetime -import logging -import random -import requests -import time -import traceback -import json -from base64 import b64encode -from hashlib import sha256 -from queue import Queue -from threading import Thread, Lock -from . import __version__ as auto_rx_version - -# These get replaced out after init -url_habitat_uuids = "" -url_habitat_db = "" -habitat_url = "" - -# CRC16 function -def crc16_ccitt(data): - """ - Calculate the CRC16 CCITT checksum of *data*. - (CRC16 CCITT: start 0xFFFF, poly 0x1021) - - Args: - data (str): String to be CRC'd. The string will be encoded to ASCII prior to CRCing. - - Return: - str: Resultant checksum as two bytes of hexadecimal. - - """ - crc16 = crcmod.predefined.mkCrcFun("crc-ccitt-false") - # Encode to ASCII. - _data_ascii = data.encode("ascii") - return hex(crc16(_data_ascii))[2:].upper().zfill(4) - - -def sonde_telemetry_to_sentence(telemetry, payload_callsign=None, comment=None): - """ Convert a telemetry data dictionary into a UKHAS-compliant telemetry sentence. - - Args: - telemetry (dict): A sonde telemetry dictionary. Refer to the description in the autorx.decode.SondeDecoder docs. - payload_callsign (str): If supplied, override the callsign field with this string. - comment (str): Optional data to add to the comment field of the output sentence. - - Returns: - str: UKHAS-compliant telemetry sentence for uploading to Habitat - - - """ - # We only want HH:MM:SS for uploading to habitat. - _short_time = telemetry["datetime_dt"].strftime("%H:%M:%S") - - if payload_callsign is None: - # If we haven't been supplied a callsign, we generate one based on the serial number. - _callsign = "RS_" + telemetry["id"] - else: - _callsign = payload_callsign - - _sentence = "$$%s,%d,%s,%.5f,%.5f,%d,%.1f,%.1f,%.1f" % ( - _callsign, - telemetry["frame"], - _short_time, - telemetry["lat"], - telemetry["lon"], - int(telemetry["alt"]), # Round to the nearest metre. - telemetry["vel_h"], - telemetry["temp"], - telemetry["humidity"], - ) - - if "f_centre" in telemetry: - # We have an estimate of the sonde's centre frequency from the modem, use this in place of - # the RX frequency. - # Round to 1 kHz - _freq = round(telemetry["f_centre"] / 1000.0) - # Convert to MHz. - _freq = "%.3f MHz" % (_freq / 1e3) - else: - # Otherwise, use the normal frequency. - _freq = telemetry["freq"] - - # Add in a comment field, containing the sonde type, serial number, and frequency. - _sentence += ",%s %s %s" % (telemetry["type"], telemetry["id"], _freq) - - # Add in pressure data, if valid (not -1) - if telemetry["pressure"] > 0.0: - _sentence += " %.1fhPa" % telemetry["pressure"] - - # Check for Burst/Kill timer data, and add in. - if "bt" in telemetry: - if (telemetry["bt"] != -1) and (telemetry["bt"] != 65535): - _sentence += " BT %s" % time.strftime( - "%H:%M:%S", time.gmtime(telemetry["bt"]) - ) - - # Add in battery voltage, if the field is valid (e.g. not -1) - if telemetry["batt"] > 0.0: - _sentence += " %.1fV" % telemetry["batt"] - - # Add on any custom comment data if provided. - if comment != None: - comment = comment.replace(",", "_") - _sentence += " " + comment - - _checksum = crc16_ccitt(_sentence[2:]) - _output = _sentence + "*" + _checksum + "\n" - return _output - - -# -# Functions for uploading a listener position to Habitat. -# Derived from https://raw.githubusercontent.com/rossengeorgiev/hab-tools/master/spot2habitat_chase.py -# -callsign_init = False - -uuids = [] - - -def check_callsign(callsign, timeout=10): - """ - Check if a payload document exists for a given callsign. - - This is done in a bit of a hack-ish way at the moment. We just check to see if there have - been any reported packets for the payload callsign on the tracker. - This should really be replaced with the correct call into the habitat tracker. - - Args: - callsign (str): Payload callsign to search for. - timeout (int): Timeout for the search, in seconds. Defaults to 10 seconds. - - Returns: - bool: True if callsign has been observed within the last 6 hour, False otherwise. - """ - - _url_check_callsign = "http://legacy-snus.habhub.org/tracker/datanew.php?mode=6hours&type=positions&format=json&max_positions=10&position_id=0&vehicle=%s" - - logging.debug("Habitat - Checking if %s has been observed recently..." % callsign) - # Perform the request - _r = requests.get(_url_check_callsign % callsign, timeout=timeout) - - try: - # Read the response in as JSON - _r_json = _r.json() - - # Read out the list of positions for the requested callsign - _positions = _r_json["positions"]["position"] - - # If there is at least one position returned, we assume there is a valid payload document. - if len(_positions) > 0: - logging.info( - "Habitat - Callsign %s already present in Habitat DB, not creating new payload doc." - % callsign - ) - return True - else: - # Otherwise, we don't, and go create one. - return False - - except Exception as e: - # Handle errors with JSON parsing. - logging.error( - "Habitat - Unable to request payload positions from legacy-snus.habhub.org - %s" - % str(e) - ) - return False - - -# Keep an internal cache for which payload docs we've created so we don't spam couchdb with updates -payload_config_cache = {} - - -def ISOStringNow(): - return "%sZ" % datetime.datetime.utcnow().isoformat() - - -def initPayloadDoc( - serial, description="Meteorology Radiosonde", frequency=401.5, timeout=20 -): - """Creates a payload in Habitat for the radiosonde before uploading""" - global url_habitat_db - - payload_data = { - "type": "payload_configuration", - "name": serial, - "time_created": ISOStringNow(), - "metadata": {"description": description}, - "transmissions": [ - { - "frequency": frequency, - "modulation": "RTTY", - "mode": "USB", - "encoding": "ASCII-8", - "parity": "none", - "stop": 2, - "shift": 350, - "baud": 50, - "description": "DUMMY ENTRY, DATA IS VIA radiosonde_auto_rx", - } - ], - "sentences": [ - { - "protocol": "UKHAS", - "callsign": serial, - "checksum": "crc16-ccitt", - "fields": [ - {"name": "sentence_id", "sensor": "base.ascii_int"}, - {"name": "time", "sensor": "stdtelem.time"}, - { - "name": "latitude", - "sensor": "stdtelem.coordinate", - "format": "dd.dddd", - }, - { - "name": "longitude", - "sensor": "stdtelem.coordinate", - "format": "dd.dddd", - }, - {"name": "altitude", "sensor": "base.ascii_int"}, - {"name": "speed", "sensor": "base.ascii_float"}, - {"name": "temperature_external", "sensor": "base.ascii_float"}, - {"name": "humidity", "sensor": "base.ascii_float"}, - {"name": "comment", "sensor": "base.string"}, - ], - "filters": { - "post": [ - {"filter": "common.invalid_location_zero", "type": "normal"} - ] - }, - "description": "radiosonde_auto_rx to Habitat Bridge", - } - ], - } - - # Perform the POST request to the Habitat DB. - try: - _r = requests.post(url_habitat_db, json=payload_data, timeout=timeout) - - if _r.json()["ok"] is True: - logging.info("Habitat - Created a payload document for %s" % serial) - return True - else: - logging.error( - "Habitat - Failed to create a payload document for %s" % serial - ) - return False - - except Exception as e: - logging.error( - "Habitat - Failed to create a payload document for %s - %s" - % (serial, str(e)) - ) - return False - - -def postListenerData(doc, timeout=10): - global uuids, url_habitat_db - # do we have at least one uuid, if not go get more - if len(uuids) < 1: - fetchUuids() - - # Attempt to add UUID and time data to document. - try: - doc["_id"] = uuids.pop() - except IndexError: - logging.error("Habitat - Unable to post listener data - no UUIDs available.") - return False - - doc["time_uploaded"] = ISOStringNow() - - try: - _r = requests.post(url_habitat_db, json=doc, timeout=timeout) - return True - except Exception as e: - logging.error("Habitat - Could not post listener data - %s" % str(e)) - return False - - -def fetchUuids(timeout=10): - global uuids, url_habitat_uuids - - _retries = 5 - - while _retries > 0: - try: - _r = requests.get(url_habitat_uuids % 10, timeout=timeout) - uuids.extend(_r.json()["uuids"]) - # logging.debug("Habitat - Got UUIDs") - return - except Exception as e: - logging.error( - "Habitat - Unable to fetch UUIDs, retrying in 10 seconds - %s" % str(e) - ) - time.sleep(10) - _retries = _retries - 1 - continue - - logging.error("Habitat - Gave up trying to get UUIDs.") - return - - -def initListenerCallsign(callsign, version="", antenna=""): - doc = { - "type": "listener_information", - "time_created": ISOStringNow(), - "data": { - "callsign": callsign, - "antenna": antenna, - "radio": "radiosonde_auto_rx %s" % version, - }, - } - - resp = postListenerData(doc) - - if resp is True: - # logging.debug("Habitat - Listener Callsign Initialized.") - return True - else: - logging.error("Habitat - Unable to initialize callsign.") - return False - - -def uploadListenerPosition(callsign, lat, lon, version="", antenna=""): - """ Initializer Listener Callsign, and upload Listener Position """ - - # Attempt to initialize the listeners callsign - resp = initListenerCallsign(callsign, version=version, antenna=antenna) - # If this fails, it means we can't contact the Habitat server, - # so there is no point continuing. - if resp is False: - return False - - doc = { - "type": "listener_telemetry", - "time_created": ISOStringNow(), - "data": { - "callsign": callsign, - "chase": False, - "latitude": lat, - "longitude": lon, - "altitude": 0, - "speed": 0, - }, - } - - # post position to habitat - resp = postListenerData(doc) - if resp is True: - logging.info("Habitat - Station position uploaded.") - return True - else: - logging.error("Habitat - Unable to upload station position.") - return False - - -# -# Habitat Uploader Class -# - - -class HabitatUploader(object): - """ - Queued Habitat Telemetry Uploader class - This performs uploads to the Habitat servers, and also handles generation of flight documents. - - Incoming telemetry packets are fed into queue, which is checked regularly. - If a new callsign is sighted, a payload document is created in the Habitat DB. - The telemetry data is then converted into a UKHAS-compatible format, before being added to queue to be - uploaded as network speed permits. - - If an upload attempt times out, the packet is discarded. - If the queue fills up (probably indicating no network connection, and a fast packet downlink rate), - it is immediately emptied, to avoid upload of out-of-date packets. - - Note that this uploader object is intended to handle telemetry from multiple sondes - """ - - # We require the following fields to be present in the incoming telemetry dictionary data - REQUIRED_FIELDS = [ - "frame", - "id", - "datetime", - "lat", - "lon", - "alt", - "temp", - "type", - "freq", - "freq_float", - "datetime_dt", - ] - - def __init__( - self, - user_callsign="N0CALL", - station_position=(0.0, 0.0, 0.0), - user_antenna="", - synchronous_upload_time=30, - callsign_validity_threshold=2, - upload_queue_size=16, - upload_timeout=10, - upload_retries=5, - upload_retry_interval=0.25, - user_position_update_rate=6, - inhibit=False, - url="http://habitat.sondehub.org/", - ): - """ Initialise a Habitat Uploader object. - - Args: - user_callsign (str): Callsign of the uploader. - station_position (tuple): Optional - a tuple consisting of (lat, lon, alt), which if populated, - is used to plot the listener's position on the Habitat map, both when this class is initialised, and - when a new sonde ID is observed. - - synchronous_upload_time (int): Upload the most recent telemetry when time.time()%synchronous_upload_time == 0 - This is done in an attempt to get multiple stations uploading the same telemetry sentence simultaneously, - and also acts as decimation on the number of sentences uploaded to Habitat. - callsign_validity_threshold (int): Only upload telemetry data if the callsign has been observed more than N times. Default = 5 - - upload_queue_size (int): Maximum umber of sentences to keep in the upload queue. If the queue is filled, - it will be emptied (discarding the queue contents). - upload_timeout (int): Timeout (Seconds) when performing uploads to Habitat. Default: 10 seconds. - upload_retries (int): Retry an upload up to this many times. Default: 5 - upload_retry_interval (int): Time interval between upload retries. Default: 0.25 seconds. - - user_position_update_rate (int): Time interval between automatic station position updates, hours. - Set to 6 hours by default, updating any more often than this is not really useful. - - inhibit (bool): Inhibit all uploads. Mainly intended for debugging. - - """ - - self.user_callsign = user_callsign - self.station_position = station_position - self.user_antenna = user_antenna - self.upload_timeout = upload_timeout - self.upload_retries = upload_retries - self.upload_retry_interval = upload_retry_interval - self.upload_queue_size = upload_queue_size - self.synchronous_upload_time = synchronous_upload_time - self.callsign_validity_threshold = callsign_validity_threshold - self.inhibit = inhibit - self.user_position_update_rate = user_position_update_rate - - # set the habitat upload url - global url_habitat_uuids, url_habitat_db, habitat_url - url_habitat_uuids = url + "_uuids?count=%d" - url_habitat_db = url + "habitat/" - habitat_url = url - - # Our two Queues - one to hold sentences to be upload, the other to temporarily hold - # input telemetry dictionaries before they are converted and processed. - self.habitat_upload_queue = Queue(upload_queue_size) - self.input_queue = Queue() - - # Dictionary where we store sorted telemetry data for upload when required. - # Elements will be named after payload IDs, and will contain: - # 'count' (int): Number of times this callsign has been observed. Uploads will only occur when - # this number rises above callsign_validity_threshold. - # 'data' (Queue): A queue of telemetry sentences to be uploaded. When the upload timer fires, - # this queue will be dumped, and the most recent telemetry uploaded. - # 'habitat_document' (bool): Indicates if a habitat document has been created for this payload ID. - # 'listener_updated' (bool): Indicates if the listener position has been updated for the start of this ID's flight. - self.observed_payloads = {} - - # Record of when we last uploaded a user station position to Habitat. - self.last_user_position_upload = 0 - - # Lock for dealing with telemetry uploads. - self.upload_lock = Lock() - - # Start the uploader thread. - self.upload_thread_running = True - self.upload_thread = Thread(target=self.habitat_upload_thread) - self.upload_thread.start() - - # Start the input queue processing thread. - self.input_processing_running = True - self.input_thread = Thread(target=self.process_queue) - self.input_thread.start() - - self.timer_thread_running = True - self.timer_thread = Thread(target=self.upload_timer) - self.timer_thread.start() - - def user_position_upload(self): - """ Upload the the station position to Habitat. """ - if self.station_position == None: - # Upload is successful, just flag it as OK and move on. - self.last_user_position_upload = time.time() - return False - - if (self.station_position[0] != 0.0) or (self.station_position[1] != 0.0): - _success = uploadListenerPosition( - self.user_callsign, - self.station_position[0], - self.station_position[1], - version=auto_rx_version, - antenna=self.user_antenna, - ) - self.last_user_position_upload = time.time() - return _success - else: - # No position set, just flag the update as successful. - self.last_user_position_upload = time.time() - return False - - def habitat_upload(self, sentence): - """ Upload a UKHAS-standard telemetry sentence to Habitat - - Args: - sentence (str): The UKHAS-standard telemetry sentence to upload. - """ - - if self.inhibit: - self.log_info("Upload inhibited.") - return - - # Generate payload to be uploaded - _sentence_b64 = b64encode( - sentence.encode("ascii") - ) # Encode to ASCII to be able to perform B64 encoding... - _date = datetime.datetime.utcnow().isoformat("T") + "Z" - _user_call = self.user_callsign - - _data = { - "type": "payload_telemetry", - "data": { - "_raw": _sentence_b64.decode( - "ascii" - ) # ... but decode back to a string to enable JSON serialisation. - }, - "receivers": { - _user_call: {"time_created": _date, "time_uploaded": _date,}, - }, - } - - # The URL to upload to. - _url = ( - habitat_url - + "habitat/_design/payload_telemetry/_update/add_listener/%s" - % sha256(_sentence_b64).hexdigest() - ) - - # Delay for a random amount of time between 0 and upload_retry_interval*2 seconds. - time.sleep(random.random() * self.upload_retry_interval * 2.0) - - _retries = 0 - - # When uploading, we have three possible outcomes: - # - Can't connect. No point immediately re-trying in this situation. - # - The packet is uploaded successfuly (201 / 403) - # - There is a upload conflict on the Habitat DB end (409). We can retry and it might work. - while _retries < self.upload_retries: - # Run the request. - try: - headers = {"User-Agent": "autorx-" + auto_rx_version} - _req = requests.put( - _url, - data=json.dumps(_data), - timeout=(self.upload_timeout, 6.1), - headers=headers, - ) - except Exception as e: - self.log_error("Upload Failed: %s" % str(e)) - return - - if _req.status_code == 201 or _req.status_code == 403: - # 201 = Success, 403 = Success, sentence has already seen by others. - self.log_info( - "Uploaded sentence to Habitat successfully: %s" % sentence.strip() - ) - _upload_success = True - break - elif _req.status_code == 409: - # 409 = Upload conflict (server busy). Sleep for a moment, then retry. - self.log_debug("Upload conflict.. retrying.") - time.sleep(random.random() * self.upload_retry_interval) - _retries += 1 - else: - self.log_error( - "Error uploading to Habitat. Status Code: %d %s." - % (_req.status_code, _req.text) - ) - break - - if _retries == self.upload_retries: - self.log_error( - "Upload conflict not resolved with %d retries." % self.upload_retries - ) - - return - - def habitat_upload_thread(self): - """ Handle uploading of packets to Habitat """ - - self.log_debug("Started Habitat Uploader Thread.") - - while self.upload_thread_running: - - if self.habitat_upload_queue.qsize() > 0: - # If the queue is completely full, jump to the most recent telemetry sentence. - if self.habitat_upload_queue.qsize() == self.upload_queue_size: - while not self.habitat_upload_queue.empty(): - try: - sentence = self.habitat_upload_queue.get_nowait() - except: - pass - - self.log_warning( - "Upload queue was full when reading from queue, now flushed - possible connectivity issue." - ) - else: - # Otherwise, get the first item in the queue. - sentence = self.habitat_upload_queue.get() - - # Attempt to upload it. - if sentence: - self.habitat_upload(sentence) - - else: - # Wait for a short time before checking the queue again. - time.sleep(0.1) - - self.log_debug("Stopped Habitat Uploader Thread.") - - def handle_telem_dict(self, telem, immediate=False): - # Try and convert it to a UKHAS sentence - try: - _sentence = sonde_telemetry_to_sentence(telem) - except Exception as e: - self.log_error("Error converting telemetry to sentence - %s" % str(e)) - return - - _callsign = "RS_" + telem["id"] - - # Wait for the upload_lock to be available, to ensure we don't end up with - # race conditions resulting in multiple payload docs being created. - self.upload_lock.acquire() - - # Habitat Payload document creation has been disabled as of 2020-03-20. - # We now use a common payload document for all radiosonde telemetry. - # - # # Create a habitat document if one does not already exist: - # if not self.observed_payloads[telem['id']]['habitat_document']: - # # Check if there has already been telemetry from this ID observed on Habhub - # _document_exists = check_callsign(_callsign) - # # If so, we don't need to create a new document - # if _document_exists: - # self.observed_payloads[telem['id']]['habitat_document'] = True - # else: - # # Otherwise, we attempt to create a new document. - # if self.inhibit: - # # If we have an upload inhibit, don't create a payload doc. - # _created = True - # else: - # _created = initPayloadDoc(_callsign, description="Meteorology Radiosonde", frequency=telem['freq_float']) - - # if _created: - # self.observed_payloads[telem['id']]['habitat_document'] = True - # else: - # self.log_error("Error creating payload document!") - # self.upload_lock.release() - # return - - if immediate: - self.log_info( - "Performing immediate upload for first telemetry sentence of %s." - % telem["id"] - ) - self.habitat_upload(_sentence) - - else: - # Attept to add it to the habitat uploader queue. - try: - if self.habitat_upload_queue.qsize() == self.upload_queue_size: - # Flush queue. - while not self.habitat_upload_queue.empty(): - try: - self.habitat_upload_queue.get_nowait() - except: - pass - - self.log_error( - "Upload queue was full when adding to queue, now flushed - possible connectivity issue." - ) - - self.habitat_upload_queue.put_nowait(_sentence) - self.log_debug( - "Upload queue size: %d" % self.habitat_upload_queue.qsize() - ) - except Exception as e: - self.log_error( - "Error adding sentence to queue, queue likely full. %s" % str(e) - ) - self.log_error("Queue Size: %d" % self.habitat_upload_queue.qsize()) - - self.upload_lock.release() - - def upload_timer(self): - """ Add packets to the habitat upload queue if it is time for us to upload. """ - - while self.timer_thread_running: - if int(time.time()) % self.synchronous_upload_time == 0: - # Time to upload! - for _id in self.observed_payloads.keys(): - # If no data, continue... - if self.observed_payloads[_id]["data"].empty(): - continue - else: - # Otherwise, dump the queue and keep the latest telemetry. - while not self.observed_payloads[_id]["data"].empty(): - _telem = self.observed_payloads[_id]["data"].get() - - self.handle_telem_dict(_telem) - - # Sleep a second so we don't hit the synchronous upload time again. - time.sleep(1) - else: - # Not yet time to upload, wait for a bit. - time.sleep(0.1) - - def process_queue(self): - """ Process packets from the input queue. - - This thread handles packets from the input queue (provided by the decoders) - Packets are sorted by ID, and a dictionary entry is created. - - """ - - while self.input_processing_running: - # Process everything in the queue. - while self.input_queue.qsize() > 0: - # Grab latest telem dictionary. - _telem = self.input_queue.get_nowait() - - _id = _telem["id"] - - if _id not in self.observed_payloads: - # We haven't seen this ID before, so create a new dictionary entry for it. - self.observed_payloads[_id] = { - "count": 1, - "data": Queue(), - "habitat_document": False, - "first_uploaded": False, - } - self.log_debug( - "New Payload %s. Not observed enough to allow upload." % _id - ) - # However, we don't yet add anything to the queue for this payload... - else: - # We have seen this payload before! - # Increment the 'seen' counter. - self.observed_payloads[_id]["count"] += 1 - - # If we have seen this particular ID enough times, add the data to the ID's queue. - if ( - self.observed_payloads[_id]["count"] - >= self.callsign_validity_threshold - ): - - # If this is the first time we have observed this payload, immediately upload the first position we got. - if self.observed_payloads[_id]["first_uploaded"] == False: - # Because receiving balloon telemetry appears to be a competition, immediately upload the - # first valid position received. - self.handle_telem_dict(_telem, immediate=True) - - self.observed_payloads[_id]["first_uploaded"] = True - - else: - # Otherwise, add the telemetry to the upload queue - self.observed_payloads[_id]["data"].put(_telem) - - else: - self.log_debug( - "Payload ID %s not observed enough to allow upload." % _id - ) - - # If we haven't uploaded our station position recently, re-upload it. - if ( - time.time() - self.last_user_position_upload - ) > self.user_position_update_rate * 3600: - self.user_position_upload() - - time.sleep(0.1) - - def add(self, telemetry): - """ Add a dictionary of telemetry to the input queue. - - Args: - telemetry (dict): Telemetry dictionary to add to the input queue. - - """ - - # Discard any telemetry which is indicated to be encrypted. - if "encrypted" in telemetry: - if telemetry["encrypted"] == True: - return - - # Check the telemetry dictionary contains the required fields. - for _field in self.REQUIRED_FIELDS: - if _field not in telemetry: - self.log_error("JSON object missing required field %s" % _field) - return - - # Add it to the queue if we are running. - if self.input_processing_running: - self.input_queue.put(telemetry) - else: - self.log_error("Processing not running, discarding.") - - def update_station_position(self, lat, lon, alt): - """ Update the internal station position record. Used when determining the station position by GPSD """ - self.station_position = (lat, lon, alt) - - def close(self): - """ Shutdown uploader and processing threads. """ - self.log_debug("Waiting for threads to close...") - self.input_processing_running = False - self.timer_thread_running = False - self.upload_thread_running = False - - # Wait for all threads to close. - if self.upload_thread is not None: - self.upload_thread.join(60) - if self.upload_thread.is_alive(): - self.log_error("habitat upload thread failed to join") - - - if self.timer_thread is not None: - self.timer_thread.join(60) - if self.timer_thread.is_alive(): - self.log_error("habitat timer thread failed to join") - - if self.input_thread is not None: - self.input_thread.join(60) - if self.input_thread.is_alive(): - self.log_error("habitat input thread failed to join") - - def log_debug(self, line): - """ Helper function to log a debug message with a descriptive heading. - Args: - line (str): Message to be logged. - """ - logging.debug("Habitat - %s" % line) - - def log_info(self, line): - """ Helper function to log an informational message with a descriptive heading. - Args: - line (str): Message to be logged. - """ - logging.info("Habitat - %s" % line) - - def log_error(self, line): - """ Helper function to log an error message with a descriptive heading. - Args: - line (str): Message to be logged. - """ - logging.error("Habitat - %s" % line) - - def log_warning(self, line): - """ Helper function to log a warning message with a descriptive heading. - Args: - line (str): Message to be logged. - """ - logging.warning("Habitat - %s" % line) diff --git a/auto_rx/autorx/templates/index.html b/auto_rx/autorx/templates/index.html index 2e43c1ad..d8c574e3 100644 --- a/auto_rx/autorx/templates/index.html +++ b/auto_rx/autorx/templates/index.html @@ -689,17 +689,6 @@ sonde_id_data.vel_h = (sonde_id_data.vel_h*3.6).toFixed(1); - // Add a link to HabHub if we have habitat enabled. - // if (autorx_config.sondehub_enabled == true) { - // sonde_id_data.id = "" + sonde_id + ""; - // // These links are only going to work for Vaisala radiosondes since the APRS callsign is never passed through to the web interface, - // // and the APRS callsigns for everything other than RS41s and RS92s is different to the 'full' serials - // } else if (autorx_config.aprs_enabled == true && autorx_config.aprs_server == "radiosondy.info") { - // sonde_id_data.id = "" + sonde_id + ""; - // } else if (autorx_config.aprs_enabled == true) { - // sonde_id_data.id = "" + sonde_id + ""; - // } - sonde_id_data.realid = sonde_id; // Add SNR data, if it exists. diff --git a/auto_rx/requirements.txt b/auto_rx/requirements.txt index 3f1bb376..0aa0c24c 100644 --- a/auto_rx/requirements.txt +++ b/auto_rx/requirements.txt @@ -1,4 +1,3 @@ -crcmod python-dateutil flask flask-socketio From 6cf49620fd02d4db86800c33a38af0241aece934 Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Wed, 31 Jan 2024 16:53:34 +1030 Subject: [PATCH 23/42] Add sample count to fsk demod stats output, bump testing beta count --- auto_rx/autorx/__init__.py | 2 +- auto_rx/test/plot_fsk_demod_stats.py | 5 +---- utils/fsk_demod.c | 6 +++++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index dc6384cd..d7d6051a 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -12,7 +12,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.7.3-beta6" +__version__ = "1.7.3-beta7" # Global Variables diff --git a/auto_rx/test/plot_fsk_demod_stats.py b/auto_rx/test/plot_fsk_demod_stats.py index 17b5261f..33022966 100644 --- a/auto_rx/test/plot_fsk_demod_stats.py +++ b/auto_rx/test/plot_fsk_demod_stats.py @@ -48,10 +48,7 @@ _fest2.append(_data['f2_est']) _ppm.append(_data['ppm']) - if _time == []: - _time = [0] - else: - _time.append(_time[-1]+1.0/_sps) + _time.append(_data['samples']) _ebno_max = pd.Series(_ebno).rolling(10).max().dropna().tolist() diff --git a/utils/fsk_demod.c b/utils/fsk_demod.c index 3def541a..ecd57421 100644 --- a/utils/fsk_demod.c +++ b/utils/fsk_demod.c @@ -54,6 +54,7 @@ int main(int argc,char *argv[]){ struct FSK *fsk; struct MODEM_STATS stats; int Fs,Rs,M,P,stats_ctr,stats_loop; + long sample_count; float loop_time; int enable_stats = 0; FILE *fin,*fout; @@ -280,6 +281,7 @@ int main(int argc,char *argv[]){ for(i=0;ippm); + fprintf(stderr,"\"secs\": %ld, \"samples\": %ld, \"EbNodB\": %5.1f, \"ppm\": %4d,",seconds, sample_count, stats.snr_est, (int)fsk->ppm); float *f_est; if (fsk->freq_est_type) f_est = fsk->f2_est; From 2323ac5ef05bc20080b8c5fb125a0d7fb2894855 Mon Sep 17 00:00:00 2001 From: Clayton Smith Date: Wed, 31 Jan 2024 09:40:14 -0500 Subject: [PATCH 24/42] Log the shutdown timeout if set --- auto_rx/auto_rx.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/auto_rx/auto_rx.py b/auto_rx/auto_rx.py index 977783be..8ff6a0f7 100644 --- a/auto_rx/auto_rx.py +++ b/auto_rx/auto_rx.py @@ -765,9 +765,6 @@ def main(): ) args = parser.parse_args() - # Copy out timeout value, and convert to seconds, - _timeout = args.timeout * 60 - # Copy out RS92 ephemeris value, if provided. if args.ephemeris != "None": rs92_ephemeris = args.ephemeris @@ -826,6 +823,11 @@ def main(): logging.getLogger("engineio").setLevel(logging.ERROR) logging.getLogger("geventwebsocket").setLevel(logging.ERROR) + # Copy out timeout value, and convert to seconds. + if args.timeout > 0: + logging.info(f"Will shut down automatically after {args.timeout} minutes.") + _timeout = args.timeout * 60 + # Check all the RS utilities exist. logging.debug("Checking if required binaries exist") if not check_rs_utils(config): From ee54d3049e28cf7fd393d21ed263474dea244c17 Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Thu, 1 Feb 2024 17:02:22 +1030 Subject: [PATCH 25/42] Remove rm log_power line from autorx.sh --- auto_rx/auto_rx.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/auto_rx/auto_rx.sh b/auto_rx/auto_rx.sh index 6c901cd3..8b63b696 100755 --- a/auto_rx/auto_rx.sh +++ b/auto_rx/auto_rx.sh @@ -14,7 +14,4 @@ # change into appropriate directory cd $(dirname $0) -# Clean up old files -rm log_power*.csv - python3 auto_rx.py -t 180 \ No newline at end of file From 74e362bd8debe373679bc4432248dfeeb92fabe4 Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Sun, 11 Feb 2024 21:50:55 +1030 Subject: [PATCH 26/42] Rebase m20mod.c --- auto_rx/autorx/__init__.py | 2 +- demod/mod/m20mod.c | 195 +++++++++++++++---------------------- 2 files changed, 81 insertions(+), 116 deletions(-) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index d7d6051a..988053b8 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -12,7 +12,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.7.3-beta7" +__version__ = "1.7.3-beta8" # Global Variables diff --git a/demod/mod/m20mod.c b/demod/mod/m20mod.c index 9d496e8c..2802e043 100644 --- a/demod/mod/m20mod.c +++ b/demod/mod/m20mod.c @@ -83,7 +83,7 @@ static char rawheader[] = "10011001100110010100110010011001"; #define FRAME_LEN (100+1) // 0x64+1 #define BITFRAME_LEN (FRAME_LEN*BITS) -#define AUX_LEN 20 +#define AUX_LEN 64 #define BITAUX_LEN (AUX_LEN*BITS) @@ -264,6 +264,9 @@ frame[0x44..0x45]: frame check > done */ +#define COLOPT(tcol) ((gpx->option.col)?(tcol):("")) + + static int get_GPSweek(gpx_t *gpx) { int i; unsigned byte; @@ -721,6 +724,9 @@ static float get_P(gpx_t *gpx) { if (val > 0) { hPa = val/(float)(16*256); // 4096=0x1000 } + if (hPa > 2560.0f) { // val > 0xA00000 + hPa = -1.0f; + } return hPa; } @@ -767,96 +773,55 @@ static int print_pos(gpx_t *gpx, int bcOK, int csOK) { if ( !gpx->option.slt ) { - if (gpx->option.col) { - fprintf(stdout, col_TXT); - if (gpx->option.vbs >= 3) { - fprintf(stdout, "[%3d]", gpx->frame_bytes[pos_CNT]); - fprintf(stdout, " (W "col_GPSweek"%d"col_TXT") ", gpx->week); - } - fprintf(stdout, col_GPSTOW"%s"col_TXT" ", weekday[gpx->wday]); - fprintf(stdout, col_GPSdate"%04d-%02d-%02d"col_TXT" "col_GPSTOW"%02d:%02d:%06.3f"col_TXT" ", - gpx->jahr, gpx->monat, gpx->tag, gpx->std, gpx->min, gpx->sek); - fprintf(stdout, " lat: "col_GPSlat"%.5f"col_TXT" ", gpx->lat); - fprintf(stdout, " lon: "col_GPSlon"%.5f"col_TXT" ", gpx->lon); - fprintf(stdout, " alt: "col_GPSalt"%.2f"col_TXT" ", gpx->alt); - if (!err2) { - fprintf(stdout, " vH: "col_GPSvel"%4.1f"col_TXT" D: "col_GPSvel"%5.1f"col_TXT" vV: "col_GPSvel"%3.1f"col_TXT" ", gpx->vH, gpx->vD, gpx->vV); - } - if (gpx->option.vbs >= 1 && (bcOK || csOK)) { // SN - fprintf(stdout, " SN: "col_SN"%s"col_TXT, gpx->SN); - } - if (gpx->option.vbs >= 1) { - fprintf(stdout, " # "); - if (gpx->fwVer < 0x07) { - if (bcOK > 0) fprintf(stdout, " "col_CSok"(ok)"col_TXT); - else if (bcOK < 0) fprintf(stdout, " "col_CSoo"(oo)"col_TXT); - else fprintf(stdout, " "col_CSno"(no)"col_TXT); - } - if (csOK) fprintf(stdout, " "col_CSok"[OK]"col_TXT); - else fprintf(stdout, " "col_CSno"[NO]"col_TXT); - } - if (gpx->option.ptu && csOK) { - fprintf(stdout, " "); - if (gpx->T > -273.0f) fprintf(stdout, " T:%.1fC", gpx->T); - if (gpx->RH > -0.5f) fprintf(stdout, " RH=%.0f%%", gpx->RH); - if (gpx->option.vbs >= 2) { - if (gpx->TH > -273.0f) fprintf(stdout, " TH:%.1fC", gpx->TH); - } - if (gpx->P > 0.0f) { - if (gpx->P < 10.0f) fprintf(stdout, " P=%.3fhPa ", gpx->P); - else if (gpx->P < 100.0f) fprintf(stdout, " P=%.2fhPa ", gpx->P); - else fprintf(stdout, " P=%.1fhPa ", gpx->P); - } - } - if (gpx->option.vbs >= 3 && csOK) { - fprintf(stdout, " (bat:%.2fV)", gpx->batV); - } - fprintf(stdout, ANSI_COLOR_RESET""); + fprintf(stdout, "%s", COLOPT(col_TXT)); + if (gpx->option.vbs >= 3) { + fprintf(stdout, "[%3d]", gpx->frame_bytes[pos_CNT]); + fprintf(stdout, " (W %s%d%s) ", COLOPT(col_GPSweek), gpx->week, COLOPT(col_TXT)); } - else { - if (gpx->option.vbs >= 3) { - fprintf(stdout, "[%3d]", gpx->frame_bytes[pos_CNT]); - fprintf(stdout, " (W %d) ", gpx->week); - } - fprintf(stdout, "%s ", weekday[gpx->wday]); - fprintf(stdout, "%04d-%02d-%02d %02d:%02d:%06.3f ", - gpx->jahr, gpx->monat, gpx->tag, gpx->std, gpx->min, gpx->sek); - fprintf(stdout, " lat: %.5f ", gpx->lat); - fprintf(stdout, " lon: %.5f ", gpx->lon); - fprintf(stdout, " alt: %.2f ", gpx->alt); - if (!err2) { - fprintf(stdout, " vH: %4.1f D: %5.1f vV: %3.1f ", gpx->vH, gpx->vD, gpx->vV); - } - if (gpx->option.vbs >= 1 && (bcOK || csOK)) { // SN - fprintf(stdout, " SN: %s", gpx->SN); - } - if (gpx->option.vbs >= 1) { - fprintf(stdout, " # "); - if (gpx->fwVer < 0x07) { - //if (bcOK) fprintf(stdout, " (ok)"); else fprintf(stdout, " (no)"); - if (bcOK > 0) fprintf(stdout, " (ok)"); - else if (bcOK < 0) fprintf(stdout, " (oo)"); - else fprintf(stdout, " (no)"); - } - if (csOK) fprintf(stdout, " [OK]"); else fprintf(stdout, " [NO]"); + fprintf(stdout, "%s%s%s ", COLOPT(col_GPSTOW), weekday[gpx->wday], COLOPT(col_TXT)); + fprintf(stdout, "%s%04d-%02d-%02d%s %s%02d:%02d:%06.3f%s ", + COLOPT(col_GPSdate), gpx->jahr, gpx->monat, gpx->tag, COLOPT(col_TXT), + COLOPT(col_GPSTOW), gpx->std, gpx->min, gpx->sek, COLOPT(col_TXT)); + fprintf(stdout, " lat: %s%.5f%s ", COLOPT(col_GPSlat), gpx->lat, COLOPT(col_TXT)); + fprintf(stdout, " lon: %s%.5f%s ", COLOPT(col_GPSlon), gpx->lon, COLOPT(col_TXT)); + fprintf(stdout, " alt: %s%.2f%s ", COLOPT(col_GPSalt), gpx->alt, COLOPT(col_TXT)); + if (!err2) { + fprintf(stdout, " vH: %s%4.1f%s D: %s%5.1f%s vV: %s%3.1f%s ", + COLOPT(col_GPSvel), gpx->vH, COLOPT(col_TXT), + COLOPT(col_GPSvel), gpx->vD, COLOPT(col_TXT), + COLOPT(col_GPSvel), gpx->vV, COLOPT(col_TXT)); + } + if (gpx->option.vbs >= 1 && (bcOK || csOK)) { // SN + fprintf(stdout, " SN: %s%s%s", COLOPT(col_SN), gpx->SN, COLOPT(col_TXT)); + } + if (gpx->option.vbs >= 1) { + fprintf(stdout, " # "); + if (gpx->fwVer < 0x07) { + if (bcOK > 0) fprintf(stdout, " %s(ok)%s", COLOPT(col_CSok), COLOPT(col_TXT)); + else if (bcOK < 0) fprintf(stdout, " %s(oo)%s", COLOPT(col_CSoo), COLOPT(col_TXT)); + else fprintf(stdout, " %s(no)%s", COLOPT(col_CSno), COLOPT(col_TXT)); } - if (gpx->option.ptu && csOK) { - fprintf(stdout, " "); - if (gpx->T > -273.0f) fprintf(stdout, " T:%.1fC", gpx->T); - if (gpx->RH > -0.5f) fprintf(stdout, " RH=%.0f%%", gpx->RH); - if (gpx->option.vbs >= 2) { - if (gpx->TH > -273.0f) fprintf(stdout, " TH:%.1fC", gpx->TH); - } - if (gpx->P > 0.0f) { - if (gpx->P < 10.0f) fprintf(stdout, " P=%.3fhPa ", gpx->P); - else if (gpx->P < 100.0f) fprintf(stdout, " P=%.2fhPa ", gpx->P); - else fprintf(stdout, " P=%.1fhPa ", gpx->P); - } + if (csOK) fprintf(stdout, " %s[OK]%s", COLOPT(col_CSok), COLOPT(col_TXT)); + else fprintf(stdout, " %s[NO]%s", COLOPT(col_CSno), COLOPT(col_TXT)); + } + if (gpx->option.ptu && csOK) { + fprintf(stdout, " "); + if (gpx->T > -273.0f) fprintf(stdout, " T:%.1fC", gpx->T); + if (gpx->RH > -0.5f) fprintf(stdout, " RH=%.0f%%", gpx->RH); + if (gpx->option.vbs >= 2) { + if (gpx->TH > -273.0f) fprintf(stdout, " TH:%.1fC", gpx->TH); } - if (gpx->option.vbs >= 3 && csOK) { - fprintf(stdout, " (bat:%.2fV)", gpx->batV); + if (gpx->P > 0.0f) { + if (gpx->P < 10.0f) fprintf(stdout, " P=%.3fhPa ", gpx->P); + else if (gpx->P < 100.0f) fprintf(stdout, " P=%.2fhPa ", gpx->P); + else fprintf(stdout, " P=%.1fhPa ", gpx->P); } } + if (gpx->option.vbs >= 3 && csOK) { + fprintf(stdout, " (bat:%.2fV)", gpx->batV); + } + fprintf(stdout, "%s", COLOPT(ANSI_COLOR_RESET)); + fprintf(stdout, "\n"); } @@ -910,7 +875,7 @@ static int print_frame(gpx_t *gpx, int pos, int b2B) { ui8_t byte; int cs1, cs2; int bc1, bc2, bc; - int flen = stdFLEN; // stdFLEN=0x64, auxFLEN=0x76; M20:0x45 ? + int flen = stdFLEN; // M10:stdFLEN=0x64,auxFLEN=0x76; M20:stdFLEN=0x45,auxFLEN=0x6F ? int pos_fw = pos_stdFW; int pos_check = pos_stdCheck; @@ -954,46 +919,46 @@ static int print_frame(gpx_t *gpx, int pos, int b2B) { if (gpx->option.raw) { - if (gpx->option.col /* && gpx->frame_bytes[1] != 0x49 */) { - fprintf(stdout, col_FRTXT); + if (1 /*&& gpx->frame_bytes[1] != 0x49 */) { + fprintf(stdout, "%s", COLOPT(col_FRTXT)); for (i = 0; i < flen+1; i++) { byte = gpx->frame_bytes[i]; - if (i == 1) fprintf(stdout, col_Mtype); - if ((i >= pos_GPSTOW) && (i < pos_GPSTOW+3)) fprintf(stdout, col_GPSTOW); - if ((i >= pos_GPSlat) && (i < pos_GPSlat+4)) fprintf(stdout, col_GPSlat); - if ((i >= pos_GPSlon) && (i < pos_GPSlon+4)) fprintf(stdout, col_GPSlon); - if ((i >= pos_GPSalt) && (i < pos_GPSalt+3)) fprintf(stdout, col_GPSalt); - if ((i >= pos_GPSweek) && (i < pos_GPSweek+2)) fprintf(stdout, col_GPSweek); - if ((i >= pos_GPSvE) && (i < pos_GPSvE+2)) fprintf(stdout, col_GPSvel); - if ((i >= pos_GPSvN) && (i < pos_GPSvN+2)) fprintf(stdout, col_GPSvel); - if ((i >= pos_GPSvU) && (i < pos_GPSvU+2)) fprintf(stdout, col_GPSvel); - if ((i >= pos_SN) && (i < pos_SN+3)) fprintf(stdout, col_SN); - if (i == pos_CNT) fprintf(stdout, col_CNT); + if (i == 1) fprintf(stdout, "%s", COLOPT(col_Mtype)); + if ((i >= pos_GPSTOW) && (i < pos_GPSTOW+3)) fprintf(stdout, "%s", COLOPT(col_GPSTOW)); + if ((i >= pos_GPSlat) && (i < pos_GPSlat+4)) fprintf(stdout, "%s", COLOPT(col_GPSlat)); + if ((i >= pos_GPSlon) && (i < pos_GPSlon+4)) fprintf(stdout, "%s", COLOPT(col_GPSlon)); + if ((i >= pos_GPSalt) && (i < pos_GPSalt+3)) fprintf(stdout, "%s", COLOPT(col_GPSalt)); + if ((i >= pos_GPSweek) && (i < pos_GPSweek+2)) fprintf(stdout, "%s", COLOPT(col_GPSweek)); + if ((i >= pos_GPSvE) && (i < pos_GPSvE+2)) fprintf(stdout, "%s", COLOPT(col_GPSvel)); + if ((i >= pos_GPSvN) && (i < pos_GPSvN+2)) fprintf(stdout, "%s", COLOPT(col_GPSvel)); + if ((i >= pos_GPSvU) && (i < pos_GPSvU+2)) fprintf(stdout, "%s", COLOPT(col_GPSvel)); + if ((i >= pos_SN) && (i < pos_SN+3)) fprintf(stdout, "%s", COLOPT(col_SN)); + if (i == pos_CNT) fprintf(stdout, "%s", COLOPT(col_CNT)); if (gpx->fwVer < 0x07) { - if ((i >= pos_BlkChk) && (i < pos_BlkChk+2)) fprintf(stdout, col_Check); + if ((i >= pos_BlkChk) && (i < pos_BlkChk+2)) fprintf(stdout, "%s", COLOPT(col_Check)); } else { - if ((i >= pos_BlkChk+1) && (i < pos_BlkChk+2)) fprintf(stdout, col_Check); + if ((i >= pos_BlkChk+1) && (i < pos_BlkChk+2)) fprintf(stdout, "%s", COLOPT(col_Check)); } - if (i >= 0x02 && i <= 0x03) fprintf(stdout, col_ptuU); - if (i >= 0x04 && i <= 0x05) fprintf(stdout, col_ptuT); - if (i >= 0x06 && i <= 0x07) fprintf(stdout, col_ptuTH); - if (i == 0x16 && gpx->fwVer >= 0x07 || i >= 0x24 && i <= 0x25) fprintf(stdout, col_ptuP); + if (i >= 0x02 && i <= 0x03) fprintf(stdout, "%s", COLOPT(col_ptuU)); + if (i >= 0x04 && i <= 0x05) fprintf(stdout, "%s", COLOPT(col_ptuT)); + if (i >= 0x06 && i <= 0x07) fprintf(stdout, "%s", COLOPT(col_ptuTH)); + if (i == 0x16 && gpx->fwVer >= 0x07 || i >= 0x24 && i <= 0x25) fprintf(stdout, "%s", COLOPT(col_ptuP)); - if ((i >= pos_check) && (i < pos_check+2)) fprintf(stdout, col_Check); + if ((i >= pos_check) && (i < pos_check+2)) fprintf(stdout, "%s", COLOPT(col_Check)); fprintf(stdout, "%02x", byte); - fprintf(stdout, col_FRTXT); + fprintf(stdout, "%s", COLOPT(col_FRTXT)); } if (gpx->option.vbs) { - fprintf(stdout, " # "col_Check"%04x"col_FRTXT, cs2); + fprintf(stdout, " # %s%04x%s", COLOPT(col_Check), cs2, COLOPT(col_FRTXT)); if (gpx->fwVer < 0x07) { - if (bc > 0) fprintf(stdout, " "col_CSok"(ok)"col_TXT); - else if (bc < 0) fprintf(stdout, " "col_CSoo"(oo)"col_TXT); - else fprintf(stdout, " "col_CSno"(no)"col_TXT); + if (bc > 0) fprintf(stdout, " %s(ok)%s", COLOPT(col_CSok), COLOPT(col_TXT)); + else if (bc < 0) fprintf(stdout, " %s(oo)%s", COLOPT(col_CSoo), COLOPT(col_TXT)); + else fprintf(stdout, " %s(no)%s", COLOPT(col_CSno), COLOPT(col_TXT)); } - if (cs1 == cs2) fprintf(stdout, " "col_CSok"[OK]"col_TXT); - else fprintf(stdout, " "col_CSno"[NO]"col_TXT); + if (cs1 == cs2) fprintf(stdout, " %s[OK]%s", COLOPT(col_CSok), COLOPT(col_TXT)); + else fprintf(stdout, " %s[NO]%s", COLOPT(col_CSno), COLOPT(col_TXT)); } - fprintf(stdout, ANSI_COLOR_RESET"\n"); + fprintf(stdout, "%s\n", COLOPT(ANSI_COLOR_RESET)); } else { for (i = 0; i < flen+1; i++) { From ab398f9666a6d0f1141dc0bc0a47d65ad2b778af Mon Sep 17 00:00:00 2001 From: Clayton Smith Date: Wed, 27 Mar 2024 19:38:43 -0400 Subject: [PATCH 27/42] Replace deprecated utcnow() with now() --- auto_rx/auto_rx.py | 2 +- auto_rx/autorx/aprs.py | 18 +++++++++--------- auto_rx/autorx/decode.py | 4 ++-- auto_rx/autorx/email_notification.py | 8 ++++---- auto_rx/autorx/emulation.py | 2 +- auto_rx/autorx/gps.py | 2 +- auto_rx/autorx/logger.py | 4 ++-- auto_rx/autorx/scan.py | 2 +- auto_rx/autorx/sonde_specific.py | 2 +- auto_rx/autorx/sondehub.py | 2 +- auto_rx/autorx/static/js/scan_chart.js | 26 ++++++++++++++++---------- auto_rx/autorx/web.py | 6 +++--- auto_rx/utils/listener_nmea_crlf.py | 10 +++++----- auto_rx/utils/plot_sonde_log.py | 2 +- 14 files changed, 48 insertions(+), 42 deletions(-) diff --git a/auto_rx/auto_rx.py b/auto_rx/auto_rx.py index 8ff6a0f7..85f69d41 100644 --- a/auto_rx/auto_rx.py +++ b/auto_rx/auto_rx.py @@ -787,7 +787,7 @@ def main(): autorx.logging_path = logging_path # Configure logging - _log_suffix = datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S_system.log") + _log_suffix = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d-%H%M%S_system.log") _log_path = os.path.join(logging_path, _log_suffix) system_log_enabled = False diff --git a/auto_rx/autorx/aprs.py b/auto_rx/autorx/aprs.py index 747eeaa0..2af03532 100644 --- a/auto_rx/autorx/aprs.py +++ b/auto_rx/autorx/aprs.py @@ -214,7 +214,7 @@ def generate_station_object( _datum = "!w%s%s!" % (_lat_prec, _lon_prec) # Generate timestamp using current UTC time - _aprs_timestamp = datetime.datetime.utcnow().strftime("%H%M%S") + _aprs_timestamp = datetime.datetime.now(datetime.timezone.utc).strftime("%H%M%S") # Add version string to position comment, if requested. _aprs_comment = comment @@ -807,10 +807,10 @@ def log_warning(self, line): # ['frame', 'id', 'datetime', 'lat', 'lon', 'alt', 'temp', 'type', 'freq', 'freq_float', 'datetime_dt'] test_telem = [ # These types of DFM serial IDs are deprecated - # {'id':'DFM06-123456', 'frame':10, 'lat':-10.0, 'lon':10.0, 'alt':10000, 'temp':1.0, 'type':'DFM', 'freq':'401.520 MHz', 'freq_float':401.52, 'heading':0.0, 'vel_h':5.1, 'vel_v':-5.0, 'datetime_dt':datetime.datetime.utcnow()}, - # {'id':'DFM09-123456', 'frame':10, 'lat':-10.0, 'lon':10.0, 'alt':10000, 'temp':1.0, 'type':'DFM', 'freq':'401.520 MHz', 'freq_float':401.52, 'heading':0.0, 'vel_h':5.1, 'vel_v':-5.0, 'datetime_dt':datetime.datetime.utcnow()}, - # {'id':'DFM15-123456', 'frame':10, 'lat':-10.0, 'lon':10.0, 'alt':10000, 'temp':1.0, 'type':'DFM', 'freq':'401.520 MHz', 'freq_float':401.52, 'heading':0.0, 'vel_h':5.1, 'vel_v':-5.0, 'datetime_dt':datetime.datetime.utcnow()}, - # {'id':'DFM17-12345678', 'frame':10, 'lat':-10.0, 'lon':10.0, 'alt':10000, 'temp':1.0, 'type':'DFM', 'freq':'401.520 MHz', 'freq_float':401.52, 'heading':0.0, 'vel_h':5.1, 'vel_v':-5.0, 'datetime_dt':datetime.datetime.utcnow()}, + # {'id':'DFM06-123456', 'frame':10, 'lat':-10.0, 'lon':10.0, 'alt':10000, 'temp':1.0, 'type':'DFM', 'freq':'401.520 MHz', 'freq_float':401.52, 'heading':0.0, 'vel_h':5.1, 'vel_v':-5.0, 'datetime_dt':datetime.datetime.now(datetime.timezone.utc)}, + # {'id':'DFM09-123456', 'frame':10, 'lat':-10.0, 'lon':10.0, 'alt':10000, 'temp':1.0, 'type':'DFM', 'freq':'401.520 MHz', 'freq_float':401.52, 'heading':0.0, 'vel_h':5.1, 'vel_v':-5.0, 'datetime_dt':datetime.datetime.now(datetime.timezone.utc)}, + # {'id':'DFM15-123456', 'frame':10, 'lat':-10.0, 'lon':10.0, 'alt':10000, 'temp':1.0, 'type':'DFM', 'freq':'401.520 MHz', 'freq_float':401.52, 'heading':0.0, 'vel_h':5.1, 'vel_v':-5.0, 'datetime_dt':datetime.datetime.now(datetime.timezone.utc)}, + # {'id':'DFM17-12345678', 'frame':10, 'lat':-10.0, 'lon':10.0, 'alt':10000, 'temp':1.0, 'type':'DFM', 'freq':'401.520 MHz', 'freq_float':401.52, 'heading':0.0, 'vel_h':5.1, 'vel_v':-5.0, 'datetime_dt':datetime.datetime.now(datetime.timezone.utc)}, { "id": "DFM-19123456", "frame": 10, @@ -827,7 +827,7 @@ def log_warning(self, line): "heading": 0.0, "vel_h": 5.1, "vel_v": -5.0, - "datetime_dt": datetime.datetime.utcnow(), + "datetime_dt": datetime.datetime.now(datetime.timezone.utc), }, { "id": "DFM-123456", @@ -845,7 +845,7 @@ def log_warning(self, line): "heading": 0.0, "vel_h": 5.1, "vel_v": -5.0, - "datetime_dt": datetime.datetime.utcnow(), + "datetime_dt": datetime.datetime.now(datetime.timezone.utc), }, { "id": "N1234567", @@ -863,7 +863,7 @@ def log_warning(self, line): "heading": 0.0, "vel_h": 5.1, "vel_v": -5.0, - "datetime_dt": datetime.datetime.utcnow(), + "datetime_dt": datetime.datetime.now(datetime.timezone.utc), }, { "id": "M1234567", @@ -881,7 +881,7 @@ def log_warning(self, line): "heading": 0.0, "vel_h": 5.1, "vel_v": -5.0, - "datetime_dt": datetime.datetime.utcnow(), + "datetime_dt": datetime.datetime.now(datetime.timezone.utc), }, ] diff --git a/auto_rx/autorx/decode.py b/auto_rx/autorx/decode.py index 88969e94..85f859a0 100644 --- a/auto_rx/autorx/decode.py +++ b/auto_rx/autorx/decode.py @@ -220,7 +220,7 @@ def __init__( # Raw hex filename if self.save_raw_hex: - _outfilename = f"{datetime.datetime.utcnow().strftime('%Y%m%d-%H%M%S')}_{self.sonde_type}_{int(self.sonde_freq)}.raw" + _outfilename = f"{datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%d-%H%M%S')}_{self.sonde_type}_{int(self.sonde_freq)}.raw" _outfilename = os.path.join(autorx.logging_path, _outfilename) self.raw_file_option = "-r" else: @@ -1520,7 +1520,7 @@ def handle_decoder_line(self, data): ) # Overwrite the datetime field to make the email notifier happy - _telemetry['datetime_dt'] = datetime.datetime.utcnow() + _telemetry['datetime_dt'] = datetime.datetime.now(datetime.timezone.utc) _telemetry["freq"] = "%.3f MHz" % (self.sonde_freq / 1e6) # Send this to only the Email Notifier, if it exists. diff --git a/auto_rx/autorx/email_notification.py b/auto_rx/autorx/email_notification.py index a4c8883e..7cf34595 100644 --- a/auto_rx/autorx/email_notification.py +++ b/auto_rx/autorx/email_notification.py @@ -463,7 +463,7 @@ def log_error(self, line): "heading": 0.0, "vel_h": 5.1, "vel_v": -5.0, - "datetime_dt": datetime.datetime.utcnow(), + "datetime_dt": datetime.datetime.now(datetime.timezone.utc), } ) @@ -485,7 +485,7 @@ def log_error(self, line): "heading": 0.0, "vel_h": 5.1, "vel_v": -5.0, - "datetime_dt": datetime.datetime.utcnow(), + "datetime_dt": datetime.datetime.now(datetime.timezone.utc), "encrypted": True } ) @@ -506,7 +506,7 @@ def log_error(self, line): "heading": 0.0, "vel_h": 5.1, "vel_v": -5.0, - "datetime_dt": datetime.datetime.utcnow(), + "datetime_dt": datetime.datetime.now(datetime.timezone.utc), } print("Testing landing alert.") @@ -516,7 +516,7 @@ def log_error(self, line): _test["alt"] = _test["alt"] - 10.0 _test["lat"] = _test["lat"] + 0.001 _test["lon"] = _test["lon"] + 0.001 - _test["datetime_dt"] = datetime.datetime.utcnow() + _test["datetime_dt"] = datetime.datetime.now(datetime.timezone.utc) time.sleep(1) time.sleep(60) diff --git a/auto_rx/autorx/emulation.py b/auto_rx/autorx/emulation.py index 9618d45c..1b92eb97 100644 --- a/auto_rx/autorx/emulation.py +++ b/auto_rx/autorx/emulation.py @@ -124,7 +124,7 @@ def emulate_telemetry(filename, port=55673, speed=1.0): _fields = _line.split(",") _telemetry_datetime = parse(_fields[0]) - _current_datetime = datetime.datetime.utcnow() + _current_datetime = datetime.datetime.now(datetime.timezone.utc) for _line in _f: _fields = _line.split(",") diff --git a/auto_rx/autorx/gps.py b/auto_rx/autorx/gps.py index 0f030e70..79226d80 100644 --- a/auto_rx/autorx/gps.py +++ b/auto_rx/autorx/gps.py @@ -17,7 +17,7 @@ def get_ephemeris(destination="ephemeris.dat"): logging.debug("GPS Grabber - Connecting to ESA's FTP Server...") ftp = ftplib.FTP("gssc.esa.int", timeout=10) ftp.login("anonymous", "anonymous") - ftp.cwd("gnss/data/daily/%s/" % datetime.datetime.utcnow().strftime("%Y")) + ftp.cwd("gnss/data/daily/%s/" % datetime.datetime.now(datetime.timezone.utc).strftime("%Y")) # Ideally we would grab this data from: YYYY/brdc/brdcDDD0.YYn.Z # .. but the ESA brdc folder seems to be getting of date. The daily directories are OK though! # So instead, we use: YYYY/DDD/brdcDDD0.YYn.Z diff --git a/auto_rx/autorx/logger.py b/auto_rx/autorx/logger.py index f893b7d5..ed7633f8 100644 --- a/auto_rx/autorx/logger.py +++ b/auto_rx/autorx/logger.py @@ -223,7 +223,7 @@ def write_telemetry(self, telemetry): else: # Create a new log file. _log_suffix = "%s_%s_%s_%d_sonde.log" % ( - datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S"), + datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d-%H%M%S"), _id, _type, int(telemetry["freq_float"] * 1e3), # Convert frequency to kHz @@ -287,7 +287,7 @@ def write_rs41_subframe(self, telemetry): _type += "-XDATA" _subframe_log_suffix = "%s_%s_%s_%d_subframe.bin" % ( - datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S"), + datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d-%H%M%S"), _id, _type, int(telemetry["freq_float"] * 1e3), # Convert frequency to kHz diff --git a/auto_rx/autorx/scan.py b/auto_rx/autorx/scan.py index c1085dd5..fe040da5 100644 --- a/auto_rx/autorx/scan.py +++ b/auto_rx/autorx/scan.py @@ -969,7 +969,7 @@ def sonde_search(self, first_only=False): (_freq_decimate, _power_decimate) = peak_decimation(freq / 1e6, power, 10) scan_result["freq"] = list(_freq_decimate) scan_result["power"] = list(_power_decimate) - scan_result["timestamp"] = datetime.datetime.utcnow().isoformat() + scan_result["timestamp"] = datetime.datetime.now(datetime.timezone.utc).isoformat() scan_result["peak_freq"] = [] scan_result["peak_lvl"] = [] diff --git a/auto_rx/autorx/sonde_specific.py b/auto_rx/autorx/sonde_specific.py index 9ef531b8..d1b4d10c 100644 --- a/auto_rx/autorx/sonde_specific.py +++ b/auto_rx/autorx/sonde_specific.py @@ -16,7 +16,7 @@ def fix_datetime(datetime_str, local_dt_str=None): """ if local_dt_str is None: - _now = datetime.datetime.utcnow() + _now = datetime.datetime.now(datetime.timezone.utc) else: _now = parse(local_dt_str) diff --git a/auto_rx/autorx/sondehub.py b/auto_rx/autorx/sondehub.py index a917055f..296ce352 100644 --- a/auto_rx/autorx/sondehub.py +++ b/auto_rx/autorx/sondehub.py @@ -120,7 +120,7 @@ def reformat_data(self, telemetry): "uploader_callsign": self.user_callsign, "uploader_position": self.user_position, "uploader_antenna": self.user_antenna, - "time_received": datetime.datetime.utcnow().strftime( + "time_received": datetime.datetime.now(datetime.timezone.utc).strftime( "%Y-%m-%dT%H:%M:%S.%fZ" ), } diff --git a/auto_rx/autorx/static/js/scan_chart.js b/auto_rx/autorx/static/js/scan_chart.js index da649da7..9a53aa6e 100644 --- a/auto_rx/autorx/static/js/scan_chart.js +++ b/auto_rx/autorx/static/js/scan_chart.js @@ -96,14 +96,20 @@ function redraw_scan_chart(){ } // Show the latest scan time. - if (getCookie('UTC') == 'false') { - temp_date = scan_chart_latest_timestamp; - temp_date = temp_date.slice(0, -3); - temp_date += "Z"; - var date = new Date(temp_date); - var date_converted = date.toLocaleString(window.navigator.language,{hourCycle:'h23', year:"numeric", month:"2-digit", day:'2-digit', hour:'2-digit',minute:'2-digit', second:'2-digit'}); - $('#scan_results').html('Latest Scan: ' + date_converted); - } else { - $('#scan_results').html('Latest Scan: ' + (scan_chart_latest_timestamp.slice(0, -3) + 'Z').replace("T", " ").replace("Z", "").slice(0, -4) + ' UTC'); + var date = new Date(scan_chart_latest_timestamp); + var date_options = { + hourCycle: 'h23', + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + timeZoneName: 'short' + }; + if (getCookie('UTC') != 'false') { + date_options.timeZone = 'UTC'; } -} \ No newline at end of file + var date_converted = date.toLocaleString(window.navigator.language, date_options); + $('#scan_results').html('Latest Scan: ' + date_converted); +} diff --git a/auto_rx/autorx/web.py b/auto_rx/autorx/web.py index 341edb5b..70f76cce 100644 --- a/auto_rx/autorx/web.py +++ b/auto_rx/autorx/web.py @@ -366,7 +366,7 @@ def flask_export_log_files(serialb64=None): _zip = zip_log_files(_serial_list) - _ts = datetime.datetime.strftime(datetime.datetime.utcnow(), "%Y%m%d-%H%M%SZ") + _ts = datetime.datetime.strftime(datetime.datetime.now(datetime.timezone.utc), "%Y%m%d-%H%M%SZ") response = make_response( flask.send_file( @@ -415,7 +415,7 @@ def flask_generate_kml(serialb64=None): log_files_to_kml(_log_files, _kml_file) _kml_file.seek(0) - _ts = datetime.datetime.strftime(datetime.datetime.utcnow(), "%Y%m%d-%H%M%SZ") + _ts = datetime.datetime.strftime(datetime.datetime.now(datetime.timezone.utc), "%Y%m%d-%H%M%SZ") response = make_response( flask.send_file( @@ -667,7 +667,7 @@ def emit(self, record): # Convert log record into a dictionary log_data = { "level": record.levelname, - "timestamp": datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"), + "timestamp": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "msg": record.msg, } # Emit to all socket.io clients diff --git a/auto_rx/utils/listener_nmea_crlf.py b/auto_rx/utils/listener_nmea_crlf.py index 7284f6f1..29233877 100644 --- a/auto_rx/utils/listener_nmea_crlf.py +++ b/auto_rx/utils/listener_nmea_crlf.py @@ -12,7 +12,7 @@ import socket, json, sys, traceback from threading import Thread from dateutil.parser import parse -from datetime import datetime, timedelta +import datetime from io import StringIO import time @@ -26,7 +26,7 @@ def fix_datetime(datetime_str, local_dt_str = None): ''' if local_dt_str is None: - _now = datetime.utcnow() + _now = datetime.datetime.now(datetime.timezone.utc) else: _now = parse(local_dt_str) @@ -54,18 +54,18 @@ def fix_datetime(datetime_str, local_dt_str = None): # We are within the window, and need to adjust the day backwards or forwards based on the sonde time. if _telem_dt.hour == 23 and _now.hour == 0: # Assume system clock running slightly fast, and subtract a day from the telemetry date. - _telem_dt = _telem_dt - timedelta(days=1) + _telem_dt = _telem_dt - datetime.timedelta(days=1) elif _telem_dt.hour == 00 and _now.hour == 23: # System clock running slow. Add a day. - _telem_dt = _telem_dt + timedelta(days=1) + _telem_dt = _telem_dt + datetime.timedelta(days=1) return _telem_dt def udp_listener_nmea_callback(info): ''' Handle a Payload Summary Message from UDPListener ''' - dateRS = datetime.strptime(info['time'], '%H:%M:%S') + dateRS = datetime.datetime.strptime(info['time'], '%H:%M:%S') hms = dateRS.hour*10000.0+dateRS.minute*100.0+dateRS.second+dateRS.microsecond; dateNMEA = dateRS.year%100+dateRS.month*100+dateRS.day*10000 diff --git a/auto_rx/utils/plot_sonde_log.py b/auto_rx/utils/plot_sonde_log.py index c15bde2f..d829f7da 100644 --- a/auto_rx/utils/plot_sonde_log.py +++ b/auto_rx/utils/plot_sonde_log.py @@ -417,7 +417,7 @@ def process_directory(log_dir, output_dir, status_file, time_limit = 60): # Calculate the age of the last data point in minutes. - _data_age = (pytz.utc.localize(datetime.datetime.utcnow()) - parse(last_time)).total_seconds() / 60.0 + _data_age = (datetime.datetime.now(datetime.timezone.utc) - parse(last_time)).total_seconds() / 60.0 if burst or (_data_age > time_limit): # We consider this file to be finished. _log_status[_basename]['complete'] = True From f2201b280929f0523bc06c94fcef84edac6dfe44 Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Fri, 29 Mar 2024 10:42:01 +1030 Subject: [PATCH 28/42] Bump beta version --- auto_rx/autorx/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index 988053b8..811987dd 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -12,7 +12,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.7.3-beta8" +__version__ = "1.7.3-beta9" # Global Variables From a72bcfeca9abce79c35270a48b6880c8877374b4 Mon Sep 17 00:00:00 2001 From: Clayton Smith Date: Fri, 29 Mar 2024 12:00:30 -0400 Subject: [PATCH 29/42] Restore the previous time format in Scan Results --- auto_rx/autorx/static/js/scan_chart.js | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/auto_rx/autorx/static/js/scan_chart.js b/auto_rx/autorx/static/js/scan_chart.js index 9a53aa6e..88e8e941 100644 --- a/auto_rx/autorx/static/js/scan_chart.js +++ b/auto_rx/autorx/static/js/scan_chart.js @@ -96,20 +96,11 @@ function redraw_scan_chart(){ } // Show the latest scan time. - var date = new Date(scan_chart_latest_timestamp); - var date_options = { - hourCycle: 'h23', - year: 'numeric', - month: '2-digit', - day: '2-digit', - hour: '2-digit', - minute: '2-digit', - second: '2-digit', - timeZoneName: 'short' - }; - if (getCookie('UTC') != 'false') { - date_options.timeZone = 'UTC'; + if (getCookie('UTC') == 'false') { + var date = new Date(scan_chart_latest_timestamp); + var date_converted = date.toLocaleString(window.navigator.language,{hourCycle:'h23', year:"numeric", month:"2-digit", day:'2-digit', hour:'2-digit',minute:'2-digit', second:'2-digit'}); + } else { + var date_converted = scan_chart_latest_timestamp.slice(0, 19).replace("T", " ") + ' UTC' } - var date_converted = date.toLocaleString(window.navigator.language, date_options); $('#scan_results').html('Latest Scan: ' + date_converted); } From a9fbe7186418d8a96247b99b344c380af0a3c10a Mon Sep 17 00:00:00 2001 From: Clayton Smith Date: Fri, 29 Mar 2024 19:26:54 -0400 Subject: [PATCH 30/42] Don't update scan chart data until there's actual data --- auto_rx/autorx/templates/index.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/auto_rx/autorx/templates/index.html b/auto_rx/autorx/templates/index.html index d8c574e3..8b4b2be8 100644 --- a/auto_rx/autorx/templates/index.html +++ b/auto_rx/autorx/templates/index.html @@ -197,6 +197,10 @@ // There is Scan data ready for us! // Grab the latest set of data. $.getJSON("get_scan_data", function(data){ + if (data.freq.length == 0) { + return; + } + // Load the data into our data stores. scan_chart_spectra.columns[0] = ['x_spectra'].concat(data.freq); scan_chart_spectra.columns[1] = ['Spectra'].concat(data.power); From 3fdea498b011b7a72f492fad0554f985d94d18fa Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Sun, 19 May 2024 09:12:41 +0930 Subject: [PATCH 31/42] Rebase dfm demodulator for better DFM17 handling --- auto_rx/autorx/__init__.py | 2 +- auto_rx/station.cfg.example | 1 + auto_rx/station.cfg.example.network | 1 + demod/mod/dfm09mod.c | 18 ++++++++++-------- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index 811987dd..78da7c4d 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -12,7 +12,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.7.3-beta9" +__version__ = "1.7.3-beta10" # Global Variables diff --git a/auto_rx/station.cfg.example b/auto_rx/station.cfg.example index 1588252a..b5ba124c 100644 --- a/auto_rx/station.cfg.example +++ b/auto_rx/station.cfg.example @@ -467,6 +467,7 @@ save_cal_data = False ########################### [web] # Server Host - Can be set to :: to listen on IPv6 +# Leave this at 0.0.0.0 to have the web server listen on all interfaces. web_host = 0.0.0.0 # Server Port - Ports below 1024 can only be used if you run auto_rx as root (not recommended) web_port = 5000 diff --git a/auto_rx/station.cfg.example.network b/auto_rx/station.cfg.example.network index b94e0152..77a4f596 100644 --- a/auto_rx/station.cfg.example.network +++ b/auto_rx/station.cfg.example.network @@ -467,6 +467,7 @@ save_cal_data = False ########################### [web] # Server Host - Can be set to :: to listen on IPv6 +# Leave this at 0.0.0.0 to have the web server listen on all interfaces. web_host = 0.0.0.0 # Server Port - Ports below 1024 can only be used if you run auto_rx as root (not recommended) web_port = 5000 diff --git a/demod/mod/dfm09mod.c b/demod/mod/dfm09mod.c index ea5e52e9..8679fe82 100644 --- a/demod/mod/dfm09mod.c +++ b/demod/mod/dfm09mod.c @@ -704,6 +704,7 @@ static int conf_out(gpx_t *gpx, ui8_t *conf_bits, int ec) { ui32_t SN6, SN; ui8_t dfm6typ; ui8_t sn2_ch, sn_ch; + int dfm17_0xA = 0; conf_id = bits2val(conf_bits, 4); @@ -777,9 +778,9 @@ static int conf_out(gpx_t *gpx, ui8_t *conf_bits, int ec) { gpx->SN = SN; gpx->ptu_out = 0; - if (sn_ch == 0xA /*&& (sn2_ch & 0xF) == 0xC*/) gpx->ptu_out = sn_ch; // <+> DFM-09 + if (sn_ch == 0xA /*&& (sn2_ch & 0xF) == 0xC*/) gpx->ptu_out = sn_ch; // <+> DFM-09 (T+) / <-> DFM-17 (T-) if (sn_ch == 0xB /*&& (sn2_ch & 0xF) == 0xC*/) gpx->ptu_out = sn_ch; // <-> DFM-17 - if (sn_ch == 0xC) gpx->ptu_out = sn_ch; // <+> DFM-09P(?) , <-> DFM-17TU(?) + if (sn_ch == 0xC) gpx->ptu_out = sn_ch; // <+> DFM-09P / <-> DFM-17TU if (sn_ch == 0xD) gpx->ptu_out = sn_ch; // <-> DFM-17P(?) // PS-15 ? (sn2_ch & 0xF) == 0x0 : gpx->ptu_out = 0 // <-> PS-15 @@ -800,6 +801,7 @@ static int conf_out(gpx_t *gpx, ui8_t *conf_bits, int ec) { ret = (gpx->sonde_typ & 0xF); } + dfm17_0xA = (gpx->SN > 23050000 && gpx->option.inv); // detected Manchester type/polarity could depend on receiver/sdr if (conf_id >= 0 && conf_id <= 8 && ec == 0) { gpx->cfgchk24[conf_id] = 1; @@ -827,6 +829,7 @@ static int conf_out(gpx_t *gpx, ui8_t *conf_bits, int ec) { gpx->sensortyp = 'P'; // gpx->meas24[0] > 2e5 ? } if ( ((gpx->ptu_out == 0xB || gpx->ptu_out == 0xC) && gpx->sensortyp == 'T') || gpx->ptu_out >= 0xD) gpx->Rf = 332e3; // DFM-17 ? + if (gpx->ptu_out == 0xA && gpx->sensortyp == 'T' && dfm17_0xA) gpx->Rf = 332e3; // DFM-17 ? if (gpx->ptu_out == 6 && (gpx->sonde_typ & 0xF) == 8) { gpx->sensortyp = 'P'; @@ -846,7 +849,7 @@ static int conf_out(gpx_t *gpx, ui8_t *conf_bits, int ec) { val = bits2val(conf_bits+8, 4*4); gpx->status[1] = val/100.0; } - if (conf_id == 0x7+ofs && gpx->Rf > 300e3) { // DFM17 counter + if (conf_id == 0x7+ofs && gpx->Rf > 300e3) { // DFM17 counter (also DFM09?) val = bits2val(conf_bits+8, 4*4); gpx->status[2] = val/1.0; // sec counter } @@ -862,6 +865,7 @@ static int conf_out(gpx_t *gpx, ui8_t *conf_bits, int ec) { V/Ti Tf012 Rf 0xA DFM-09 5/6 0,3,4 'T+' 220k 0xC DFM-09P 7/8 1,5,6 'P+' 220k + 0xA DFM-17 5/6 0,3,4 'T-' 332k 0xB DFM-17 5/6 0,3,4 'T-' 332k 0xC DFM-17TU 5/6 0,3,4 'T-' 332k 0xD DFM-17P 7/8 1,5,6 'P-' 332k @@ -874,7 +878,8 @@ static int conf_out(gpx_t *gpx, ui8_t *conf_bits, int ec) { case 0x8: if (gpx->SN6) gpx->dfmtyp = DFM_types[DFM06P]; //gpx->sensortyp == 'P' else gpx->dfmtyp = DFM_types[PS15]; break; - case 0xA: gpx->dfmtyp = DFM_types[DFM09]; + case 0xA: if (dfm17_0xA) gpx->dfmtyp = DFM_types[DFM17]; + else gpx->dfmtyp = DFM_types[DFM09]; break; case 0xB: gpx->dfmtyp = DFM_types[DFM17]; break; @@ -1364,10 +1369,7 @@ int main(int argc, char **argv) { } else if ( (strcmp(*argv, "--ecc" ) == 0) ) { option_ecc = 1; } else if ( (strcmp(*argv, "--ecc2") == 0) ) { option_ecc = 2; } - else if ( (strcmp(*argv, "--ptu") == 0) ) { - option_ptu = 1; - //gpx.ptu_out = 1; // force ptu (non PS-15) - } + else if ( (strcmp(*argv, "--ptu") == 0) ) { option_ptu = 1; } //gpx.ptu_out = 1; // force ptu (non PS-15) else if ( (strcmp(*argv, "--spike") == 0) ) { spike = 1; } From 111f03f6e6eeff1bb815a183f0b0ea57e955446b Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Mon, 20 May 2024 19:47:15 +0930 Subject: [PATCH 32/42] Rebase both DFM and MP3H1 demods --- auto_rx/autorx/__init__.py | 2 +- demod/mod/dfm09mod.c | 3 +- demod/mod/mp3h1mod.c | 178 ++++++++++++++++++++++++++++++------- 3 files changed, 148 insertions(+), 35 deletions(-) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index 78da7c4d..1c75940f 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -12,7 +12,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.7.3-beta10" +__version__ = "1.7.3-beta11" # Global Variables diff --git a/demod/mod/dfm09mod.c b/demod/mod/dfm09mod.c index 8679fe82..e248b491 100644 --- a/demod/mod/dfm09mod.c +++ b/demod/mod/dfm09mod.c @@ -801,7 +801,8 @@ static int conf_out(gpx_t *gpx, ui8_t *conf_bits, int ec) { ret = (gpx->sonde_typ & 0xF); } - dfm17_0xA = (gpx->SN > 23050000 && gpx->option.inv); // detected Manchester type/polarity could depend on receiver/sdr + // 23038743, 2307.... + dfm17_0xA = (gpx->SN >= 23000000 && gpx->option.inv); // detected Manchester type/polarity could depend on receiver/sdr if (conf_id >= 0 && conf_id <= 8 && ec == 0) { gpx->cfgchk24[conf_id] = 1; diff --git a/demod/mod/mp3h1mod.c b/demod/mod/mp3h1mod.c index ccb553cf..5c7caf1c 100644 --- a/demod/mod/mp3h1mod.c +++ b/demod/mod/mp3h1mod.c @@ -63,12 +63,16 @@ typedef struct { } option_t; -#define BITFRAME_LEN ((51*16)/2) // ofs=8: 52..53: AA AA (1..5) or 00 00 (6) +#define CRCLEN_ECEF 45 // default +#define CRCLEN_LATLON 42 + +#define BITFRAME_LEN ((CRCLEN_ECEF+6)*8) //8=16/2 // ofs=8: 52..53: AA AA (1..5) or 00 00 (6) #define RAWBITFRAME_LEN (BITFRAME_LEN*2) #define FRAMESTART (HEADOFS+HEADLEN) #define FRAME_LEN (BITFRAME_LEN/8) + typedef struct { ui8_t subcnt1; ui8_t subcnt2; @@ -83,15 +87,19 @@ typedef struct { float calC; // C(ntc) float A_adcT; float B_adcT; float C_adcT; float A_adcH; float B_adcH; float C_adcH; + float Tadc; float RHadc; + float T; float RH; ui8_t frame[FRAME_LEN+16]; char frame_bits[BITFRAME_LEN+16]; ui32_t cfg[16]; ui32_t snC; ui32_t snD; - float T; float RH; ui8_t cfg_ntc; ui8_t cfg_T; ui8_t cfg_H; ui8_t crcOK; // + int crclen; + int bitfrm_len; + // int sec_day; int sec_day_prev; int gps_cnt; @@ -250,11 +258,22 @@ static i16_t i2(ui8_t *bytes) { // 16bit signed int #define pos_GPSecefZ (OFS+16) // 4 byte #define pos_GPSecefV (OFS+20) // 3*2 byte #define pos_GPSnSats (OFS+26) // 1 byte (num Sats ?) -#define pos_PTU1 (OFS+35) // 4 byte -#define pos_PTU2 (OFS+39) // 4 byte +#define pos_T16 (OFS+29) // 2 byte +#define pos_H16 (OFS+31) // 2 byte +#define pos_FFFF (OFS+33) // 2 byte +#define pos_ADCT (OFS+35) // 4 byte +#define pos_ADCH (OFS+39) // 4 byte #define pos_CNT2 (OFS+43) // 1 byte (0x01..0x10 ?) #define pos_CFG (OFS+44) // 2/4 byte -#define pos_CRC (OFS+48) // 2 byte +#define pos_CRC_ECEF (OFS+CRCLEN_ECEF+1) // 2 byte +#define pos_CRC_LATLON (OFS+CRCLEN_LATLON+1) // 2 byte + +#define pos_GPSlat (OFS+ 7) // 4 byte +#define pos_GPSlon (OFS+11) // 4 byte +#define pos_GPSalt (OFS+15) // 4 byte +#define pos_GPSvH (OFS+19) // 2 byte +#define pos_GPSvD (OFS+21) // 2 byte + // ----------------------------------------------------------------------------- @@ -280,10 +299,10 @@ static int crc16rev(gpx_t *gpx, int start, int len) { } return rem; } -static int check_CRC(gpx_t *gpx) { - ui32_t crclen = 45; +static int check_CRC(gpx_t *gpx, ui32_t crclen) { + //ui32_t crclen = 45; // 45/42 ui32_t crcdat = 0; - crcdat = u2(gpx->frame+pos_CRC); + crcdat = u2(gpx->frame+crclen+3); if ( crcdat != crc16rev(gpx, pos_CNT1, crclen) ) { return 1; // CRC NO } @@ -321,7 +340,7 @@ static void ecef2elli(double X[], double *lat, double *lon, double *alt) { *lon = lam*180/M_PI; } -static int get_GPSkoord(gpx_t *gpx) { +static int get_GPSkoord_ecef(gpx_t *gpx) { int k; int XYZ; // 32bit double X[3], lat, lon, alt; @@ -348,7 +367,7 @@ static int get_GPSkoord(gpx_t *gpx) { gpx->lat = lat; gpx->lon = lon; gpx->alt = alt; - if ((alt < -1000.0) || (alt > 80000.0)) return -3; // plausibility-check: altitude, if ecef=(0,0,0) + if (alt < -1000.0 || alt > 80000.0) return -3; // plausibility-check: altitude, if ecef=(0,0,0) // ECEF-Velocities @@ -374,6 +393,38 @@ static int get_GPSkoord(gpx_t *gpx) { return 0; } +static int get_GPSkoord_latlon(gpx_t *gpx) { + int XYZ; // 32bit + short vH, vV; // 16bit + unsigned short vD; + + + memcpy(&XYZ, gpx->frame+pos_GPSlat, 4); + gpx->lat = XYZ * 1e-6; + + memcpy(&XYZ, gpx->frame+pos_GPSlon, 4); + gpx->lon = XYZ * 1e-6; + + memcpy(&XYZ, gpx->frame+pos_GPSalt, 4); + gpx->alt = XYZ * 1e-2; + + if (gpx->alt < -1000.0 || gpx->alt > 80000.0) return -3; // plausibility-check: altitude + + vH = gpx->frame[pos_GPSvH] | (gpx->frame[pos_GPSvH+1] << 8); + vD = gpx->frame[pos_GPSvD] | (gpx->frame[pos_GPSvD+1] << 8); + + gpx->vH = vH / 100.0; + gpx->vD = vD / 100.0; + gpx->vV = 0; + + //TODO: Sats + // num Sats solution ? GLONASS + GPS ? + gpx->numSats = gpx->frame[pos_GPSnSats-3]; // ? + + + return 0; +} + static int reset_time(gpx_t *gpx) { gpx->gps_cnt = 0; @@ -428,7 +479,7 @@ static float f32(ui32_t w) { return f; } -static int get_ptu(gpx_t *gpx) { +static int get_ptu(gpx_t *gpx, int ofs) { // cf. МРЗ-3МК documentation float t = -273.15f; @@ -436,10 +487,10 @@ static int get_ptu(gpx_t *gpx) { float ADC_MAX = 32767.0; //32767=(1<<15)? 32767? - int ADCT = u4(gpx->frame+pos_PTU1); // u3? + int ADCT = u4(gpx->frame+pos_ADCT+ofs); // u3? float adc_t = ADCT/100.0; - int ADCH = u4(gpx->frame+pos_PTU2); // u3? + int ADCH = u4(gpx->frame+pos_ADCH+ofs); // u3? float adc_h = ADCH/100.0; @@ -454,15 +505,15 @@ static int get_ptu(gpx_t *gpx) { } } } - gpx->T = t; + gpx->Tadc = t; - if (gpx->T > -273.0f) + if (gpx->Tadc > -273.0f) { if (gpx->cfg_H == 0x7) { float poly2 = adc_h*adc_h * gpx->A_adcH + adc_h * gpx->B_adcH + gpx->A_adcH; float K = poly2/ADC_MAX; - rh = (K - 0.1515) / (0.00636*(1.05460 - 0.00216*gpx->T)); // if T = 273.15, set T=0 ? + rh = (K - 0.1515) / (0.00636*(1.05460 - 0.00216*gpx->Tadc)); // if T = 273.15, set T=0 ? if (rh < -10.0f || rh > 120.0f) rh = -1.0f; else { if (rh < 0.0f) rh = 0.0f; @@ -470,19 +521,23 @@ static int get_ptu(gpx_t *gpx) { } } } - gpx->RH = rh; + gpx->RHadc = rh; + + + gpx->T = i2(gpx->frame+pos_T16+ofs) / 100.0; + gpx->RH = i2(gpx->frame+pos_H16+ofs) / 100.0; return 0; } -static int get_cfg(gpx_t *gpx) { +static int get_cfg(gpx_t *gpx, int ofs) { gpx->subcnt1 = (gpx->frame[pos_CNT1] & 0xF); - gpx->subcnt2 = gpx->frame[pos_CNT2] ; // ? subcnt2 == subcnt1 + 1 ? + gpx->subcnt2 = gpx->frame[pos_CNT2+ofs] ; // ? subcnt2 == subcnt1 + 1 ? if (gpx->crcOK) { - ui32_t cfg32 = u4(gpx->frame+pos_CFG); + ui32_t cfg32 = u4(gpx->frame+pos_CFG+ofs); gpx->cfg[gpx->subcnt1] = cfg32; switch (gpx->subcnt1) { // or use subcnt2 ? @@ -576,13 +631,16 @@ static void print_gpx(gpx_t *gpx, int crcOK) { //printf(" :%6.1f: ", sample_count/(double)sample_rate); // + int ofs_ptucfg = (gpx->crclen == CRCLEN_ECEF) ? 0 : -3; + gpx->crcOK = crcOK; - get_cfg(gpx); + get_cfg(gpx, ofs_ptucfg); get_time(gpx); - get_GPSkoord(gpx); + if (ofs_ptucfg) get_GPSkoord_latlon(gpx); + else get_GPSkoord_ecef(gpx); - get_ptu(gpx); + get_ptu(gpx, ofs_ptucfg); if (gpx->sec_day != gpx->sec_day_prev || !gpx->option.unq) { @@ -593,14 +651,38 @@ static void print_gpx(gpx_t *gpx, int crcOK) { printf(" lat: %.5f ", gpx->lat); printf(" lon: %.5f ", gpx->lon); printf(" alt: %.2f ", gpx->alt); - printf(" vH: %4.1f D: %5.1f vV: %3.1f ", gpx->vH, gpx->vD, gpx->vV); + + printf(" vH: %4.1f D: %5.1f ", gpx->vH, gpx->vD); + if ( !ofs_ptucfg ) { + printf(" vV: %3.1f ", gpx->vV); + } + if (gpx->option.vbs > 1) printf(" sats: %d ", gpx->numSats); + if (gpx->option.vbs > 1 && ofs_ptucfg < 0) + { + static float alt0; + static int t0; + if (gpx->crcOK && gpx->sec_day > t0) { + if (t0 > 0 && gpx->sec_day < t0+10) { + printf(" (d_alt: %+4.1f) ", (gpx->alt - alt0)/(float)(gpx->sec_day - t0) ); + } + alt0 = gpx->alt; + t0 = gpx->sec_day; + } + } + if (gpx->option.ptu) { if (gpx->T > -273.0f || gpx->RH > -0.5f) printf(" "); - if (gpx->T > -273.0f) printf(" T=%.1fC", gpx->T); - if (gpx->RH > -0.5f) printf(" RH=%.0f%%", gpx->RH); + if (gpx->T > -273.0f) printf(" T=%.2fC", gpx->T); + if (gpx->RH > -0.5f) printf(" RH=%.2f%%", gpx->RH); if (gpx->T > -273.0f || gpx->RH > -0.5f) printf(" "); + if (gpx->option.vbs > 1) { + if (gpx->Tadc > -273.0f || gpx->RHadc > -0.5f) printf(" ("); + if (gpx->Tadc > -273.0f) printf(" T0=%.1fC", gpx->Tadc); + if (gpx->RHadc > -0.5f) printf(" RH0=%.0f%%", gpx->RHadc); + if (gpx->Tadc > -273.0f || gpx->RHadc > -0.5f) printf(" ) "); + } } if (gpx->option.col) { @@ -664,8 +746,14 @@ static void print_gpx(gpx_t *gpx, int crcOK) { char *ver_jsn = NULL; printf("{ \"type\": \"%s\"", "MRZ"); printf(", \"frame\": %lu, ", (unsigned long)gpx->gps_cnt); // sec_gps0+0.5 - printf("\"id\": \"MRZ-%d-%d\", \"datetime\": \"%04d-%02d-%02dT%02d:%02d:%02dZ\", \"lat\": %.5f, \"lon\": %.5f, \"alt\": %.5f, \"vel_h\": %.5f, \"heading\": %.5f, \"vel_v\": %.5f, \"sats\": %d", - gpx->snC, gpx->snD, gpx->yr, gpx->mth, gpx->day, gpx->hrs, gpx->min, gpx->sec, gpx->lat, gpx->lon, gpx->alt, gpx->vH, gpx->vD, gpx->vV, gpx->numSats); + printf("\"id\": \"MRZ-%d-%d\", \"datetime\": \"%04d-%02d-%02dT%02d:%02d:%02dZ\", \"lat\": %.5f, \"lon\": %.5f, \"alt\": %.5f", + gpx->snC, gpx->snD, gpx->yr, gpx->mth, gpx->day, gpx->hrs, gpx->min, gpx->sec, gpx->lat, gpx->lon, gpx->alt); + printf(", \"vel_h\": %.5f, \"heading\": %.5f", gpx->vH, gpx->vD); + if ( !ofs_ptucfg ) { + printf(", \"vel_v\": %.5f", gpx->vV); + } + printf(", \"sats\": %d", gpx->numSats); + if (gpx->option.ptu) { if (gpx->T > -273.0f) { fprintf(stdout, ", \"temp\": %.1f", gpx->T ); @@ -680,7 +768,7 @@ static void print_gpx(gpx_t *gpx, int crcOK) { // Reference time/position printf(", \"ref_datetime\": \"%s\"", "UTC" ); // {"GPS", "UTC"} GPS-UTC=leap_sec - printf(", \"ref_position\": \"%s\"", "GPS" ); // {"GPS", "MSL"} GPS=ellipsoid , MSL=geoid + printf(", \"ref_position\": \"%s\"", !ofs_ptucfg ? "GPS" : "MSL" ); // {"GPS", "MSL"} GPS=ellipsoid , MSL=geoid #ifdef VER_JSN_STR ver_jsn = VER_JSN_STR; @@ -717,7 +805,14 @@ static void print_frame(gpx_t *gpx, int pos, int b2B) { int frmlen = (pos-bits_ofs)/8; bits2bytes(gpx->frame_bits+bits_ofs, gpx->frame, frmlen); - crcOK = (check_CRC(gpx) == 0); + if (u2(gpx->frame+30) == 0xFFFF) gpx->crclen = CRCLEN_LATLON; + else gpx->crclen = CRCLEN_ECEF; + + crcOK = (check_CRC(gpx, gpx->crclen) == 0); + + if (crcOK) { + gpx->bitfrm_len = (gpx->crclen+6)*8; + } if (gpx->option.raw == 1) { //printf(" :%6.1f: ", sample_count/(double)sample_rate); @@ -741,7 +836,14 @@ static void print_frame(gpx_t *gpx, int pos, int b2B) { { int frmlen = pos; - crcOK = (check_CRC(gpx) == 0); + if (u2(gpx->frame+30) == 0xFFFF) gpx->crclen = CRCLEN_LATLON; + else gpx->crclen = CRCLEN_ECEF; + + crcOK = (check_CRC(gpx, gpx->crclen) == 0); + + if (crcOK) { + gpx->bitfrm_len = (gpx->crclen+6)*8; + } if (gpx->option.raw) { //printf(" :%6.1f: ", sample_count/(double)sample_rate); @@ -955,6 +1057,16 @@ int main(int argc, char **argv) { if (cfreq > 0) gpx.jsn_freq = (cfreq+500)/1000; + // init frame/type + if ((CRCLEN_ECEF+6)*8 > BITFRAME_LEN) { + if (fp) fclose(fp); + fprintf(stderr, "error: int frame\n"); + return -1; + } + gpx.crclen = CRCLEN_ECEF; + gpx.bitfrm_len = (gpx.crclen+6)*8; + + #ifdef EXT_FSK if (!option_softin) { option_softin = 1; @@ -1082,7 +1194,7 @@ int main(int argc, char **argv) { bitpos = 0; pos = FRAMESTART/2; - while ( pos < BITFRAME_LEN ) + while ( pos < gpx.bitfrm_len ) { if (option_softin) { float s1 = 0.0; @@ -1122,7 +1234,7 @@ int main(int argc, char **argv) { gpx.frame_bits[pos] = '\0'; print_frame(&gpx, pos, 1); - if (pos < BITFRAME_LEN) break; + if (pos < gpx.bitfrm_len) break; header_found = 0; } From 40b8cfa9c2fe63080090018fbee984be4d8b752d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Pinkava?= Date: Fri, 24 May 2024 08:31:52 +0200 Subject: [PATCH 33/42] auto_rx/build.sh: exit immediatelly on error This ensures script will not continue to build when error accures. Continuing the build after some part fails only brings more errors and makes the output harder to understand. --- auto_rx/build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/auto_rx/build.sh b/auto_rx/build.sh index 84e6ac65..817030f7 100755 --- a/auto_rx/build.sh +++ b/auto_rx/build.sh @@ -2,6 +2,8 @@ # # Auto Sonde Decoder build script. +set -e + # Get the auto-rx version. AUTO_RX_VERSION="\"$(python3 -m autorx.version 2>/dev/null || python -m autorx.version)\"" From 5809521ae19bba54af754276188141f9c08f4549 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Pinkava?= Date: Sat, 25 May 2024 06:38:06 +0200 Subject: [PATCH 34/42] delete lms6mod - replaced by lms6Xmod --- .gitignore | 2 - auto_rx/test/test_demod.py | 32 -- demod/mod/Makefile | 6 +- demod/mod/README.md | 2 +- demod/mod/lms6mod.c | 1058 ------------------------------------ 5 files changed, 3 insertions(+), 1097 deletions(-) delete mode 100644 demod/mod/lms6mod.c diff --git a/.gitignore b/.gitignore index 986e154b..9b4f4476 100644 --- a/.gitignore +++ b/.gitignore @@ -43,7 +43,6 @@ auto_rx/fsk_demod auto_rx/imet1rs_dft auto_rx/iq_dec auto_rx/lms6Xmod -auto_rx/lms6mod auto_rx/m10mod auto_rx/m20mod auto_rx/mk2a1680mod @@ -61,7 +60,6 @@ mk2a_lms1680 demod/dfm09ecc demod/mod/dfm09mod demod/mod/lms6Xmod -demod/mod/lms6mod demod/mod/meisei100mod demod/mod/rs41mod demod/mod/rs92mod diff --git a/auto_rx/test/test_demod.py b/auto_rx/test/test_demod.py index e51d16f5..b09998b3 100644 --- a/auto_rx/test/test_demod.py +++ b/auto_rx/test/test_demod.py @@ -612,38 +612,6 @@ } -# LMS6 - 400 MHz version -_fm_rate = 22000 -# Calculate the necessary conversions -_rtlfm_oversampling = 8.0 # Viproz's hacked rtl_fm oversamples by 8x. -_shift = -2.0*_fm_rate/_sample_fs # rtl_fm tunes 'up' by rate*2, so we need to shift the signal down by this amount. - -_resample = (_fm_rate*_rtlfm_oversampling)/_sample_fs - -if _resample != 1.0: - # We will need to resample. - _resample_command = "csdr convert_f_s16 | ./tsrc - - %.4f | csdr convert_s16_f |" % _resample - _shift = (-2.0*_fm_rate)/(_sample_fs*_resample) -else: - _resample_command = "" - -_demod_command = "| %s csdr shift_addition_cc %.5f 2>/dev/null | csdr convert_f_u8 |" % (_resample_command, _shift) -_demod_command += " ./rtl_fm_stdin -M fm -f 401000000 -F9 -s %d 2>/dev/null|" % (int(_fm_rate)) -_demod_command += " sox -t raw -r %d -e s -b 16 -c 1 - -r 48000 -b 8 -t wav - highpass 20 2>/dev/null |" % int(_fm_rate) - - -processing_type['lms6-400_rtlfm'] = { - 'demod': _demod_command, - # Decode using rs92ecc - 'decode': "../lms6mod 2>/dev/null", - #'decode': "../rs92ecc -vx -v --crc --ecc -r --vel 2>/dev/null", # For measuring No-ECC performance - # Count the number of telemetry lines. - "post_process" : " | wc -l", - #"post_process" : " | grep \"errors: 0\" | wc -l", - 'files' : "./generated/lms6-400*.bin" -} - - # # LMS6 - 1680 _fm_rate = 200000 _sample_fs = 480000 diff --git a/demod/mod/Makefile b/demod/mod/Makefile index 1cddb31c..3bdf5680 100644 --- a/demod/mod/Makefile +++ b/demod/mod/Makefile @@ -1,6 +1,6 @@ LDLIBS = -lm -PROGRAMS := rs41mod dfm09mod rs92mod lms6mod lms6Xmod meisei100mod m10mod m20mod imet54mod mp3h1mod mts01mod iq_dec +PROGRAMS := rs41mod dfm09mod rs92mod lms6Xmod meisei100mod m10mod m20mod imet54mod mp3h1mod mts01mod iq_dec all: $(PROGRAMS) @@ -10,8 +10,6 @@ dfm09mod: dfm09mod.o demod_mod.o rs92mod: rs92mod.o demod_mod.o bch_ecc_mod.o -lms6mod: lms6mod.o demod_mod.o bch_ecc_mod.o - lms6Xmod: lms6Xmod.o demod_mod.o bch_ecc_mod.o meisei100mod: meisei100mod.o demod_mod.o bch_ecc_mod.o @@ -35,4 +33,4 @@ iq_dec: CFLAGS += -Ofast iq_dec: iq_dec.o clean: - $(RM) $(PROGRAMS) $(PROGRAMS:=.o) demod_mod.o bch_ecc_mod.o \ No newline at end of file + $(RM) $(PROGRAMS) $(PROGRAMS:=.o) demod_mod.o bch_ecc_mod.o diff --git a/demod/mod/README.md b/demod/mod/README.md index 3cd0687e..c82d204b 100644 --- a/demod/mod/README.md +++ b/demod/mod/README.md @@ -6,7 +6,7 @@ alternative decoders using cross-correlation for better header-synchronization #### Files * `demod_mod.c`, `demod_mod.h`,
- `rs41mod.c`, `rs92mod.c`, `dfm09mod.c`, `m10mod.c`, `lms6mod.c`, `lms6Xmod.c`, `meisei100mod.c`,
+ `rs41mod.c`, `rs92mod.c`, `dfm09mod.c`, `m10mod.c`, `lms6Xmod.c`, `meisei100mod.c`,
`bch_ecc_mod.c`, `bch_ecc_mod.h` #### Compile diff --git a/demod/mod/lms6mod.c b/demod/mod/lms6mod.c deleted file mode 100644 index a0d11dfc..00000000 --- a/demod/mod/lms6mod.c +++ /dev/null @@ -1,1058 +0,0 @@ - -/* - * LMS6 - * (403 MHz) - * - * sync header: correlation/matched filter - * files: lms6mod.c demod_mod.c demod_mod.h bch_ecc_mod.c bch_ecc_mod.h - * compile, either (a) or (b): - * (a) - * gcc -c demod_mod.c - * gcc -DINCLUDESTATIC lms6mod.c demod_mod.o -lm -o lms6mod - * (b) - * gcc -c demod_mod.c - * gcc -c bch_ecc_mod.c - * gcc lms6mod.c demod_mod.o bch_ecc_mod.o -lm -o lms6mod - * - * usage: - * ./lms6mod --vit --ecc - * ( --vit recommended) - * author: zilog80 - */ - -#include -#include -#include -#include - -#ifdef CYGWIN - #include // cygwin: _setmode() - #include -#endif - - -//typedef unsigned char ui8_t; -//typedef unsigned short ui16_t; -//typedef unsigned int ui32_t; - -#include "demod_mod.h" - -//#define INCLUDESTATIC 1 -#ifdef INCLUDESTATIC - #include "bch_ecc_mod.c" -#else - #include "bch_ecc_mod.h" -#endif - - -typedef struct { - i8_t vbs; // verbose output - i8_t raw; // raw frames - i8_t crc; // CRC check output - i8_t ecc; // Reed-Solomon ECC - i8_t sat; // GPS sat data - i8_t ptu; // PTU: temperature - i8_t inv; - i8_t vit; - i8_t jsn; // JSON output (auto_rx) -} option_t; - - -/* -------------------------------------------------------------------------- */ - -#define BAUD_RATE 4800 - -#define BITS 8 -#define HEADOFS 0 //16 -#define HEADLEN ((4*16)-HEADOFS) - -#define SYNC_LEN 5 -#define FRM_LEN (223) -#define PAR_LEN (32) -#define FRMBUF_LEN (3*FRM_LEN) -#define BLOCKSTART (SYNC_LEN*BITS*2) -#define BLOCK_LEN (FRM_LEN+PAR_LEN+SYNC_LEN) // 255+5 = 260 -#define RAWBITBLOCK_LEN ((BLOCK_LEN+1)*BITS*2) // (+1 tail) - -#define FRAME_LEN (300) // 4800baud, 16bits/byte -#define BITFRAME_LEN (FRAME_LEN*BITS) -#define RAWBITFRAME_LEN (BITFRAME_LEN*2) -#define OVERLAP 64 -#define OFS 4 - - -static char rawheader[] = "0101011000001000""0001110010010111""0001101010100111""0011110100111110"; // (c0,inv(c1)) -// (00) 58 f3 3f b8 -// char header[] = "0000001101011101""0100100111000010""0100111111110010""0110100001101011"; // (c0,c1) -static ui8_t rs_sync[] = { 0x00, 0x58, 0xf3, 0x3f, 0xb8}; -// 0x58f33fb8 little-endian <-> 0x1ACFFC1D big-endian bytes - -// (00) 58 f3 3f b8 -static char blk_syncbits[] = "0000000000000000""0000001101011101""0100100111000010""0100111111110010""0110100001101011"; - -static ui8_t frm_sync[] = { 0x24, 0x54, 0x00, 0x00}; - - -#define L 7 // d_f=10 -static char polyA[] = "1001111"; // 0x4f: x^6+x^3+x^2+x+1 -static char polyB[] = "1101101"; // 0x6d: x^6+x^5+x^3+x^2+1 -/* -// d_f=6 -qA[] = "1110011"; // 0x73: x^6+x^5+x^4+x+1 -qB[] = "0011110"; // 0x1e: x^4+x^3+x^2+x -pA[] = "10010101"; // 0x95: x^7+x^4+x^2+1 = (x+1)(x^6+x^5+x^4+x+1) = (x+1)qA -pB[] = "00100010"; // 0x22: x^5+x = (x+1)(x^4+x^3+x^2+x)=x(x+1)^3 = (x+1)qB -polyA = qA + x*qB -polyB = qA + qB -*/ - -#define N (1 << L) -#define M (1 << (L-1)) - -typedef struct { - ui8_t bIn; - ui8_t codeIn; - ui8_t prevState; // 0..M=64 - int w; // > 255 : if (w>250): w=250 ? - //float sw; -} states_t; - -typedef struct { - char rawbits[RAWBITFRAME_LEN+OVERLAP*BITS*2 +8]; - states_t state[RAWBITFRAME_LEN+OVERLAP +8][M]; - states_t d[N]; -} VIT_t; - -typedef struct { - int frnr; - int sn; - int week; int gpstow; int gpssec; - int jahr; int monat; int tag; - int wday; - int std; int min; float sek; - double lat; double lon; double alt; - double vH; double vD; double vV; - double vE; double vN; double vU; - char blk_rawbits[RAWBITBLOCK_LEN+SYNC_LEN*BITS*2 +8]; - ui8_t frame[FRM_LEN]; // = { 0x24, 0x54, 0x00, 0x00}; // dataheader - int frm_pos; // ecc_blk <-> frm_blk - int sf; - option_t option; - RS_t RS; - VIT_t *vit; -} gpx_t; - - -/* ------------------------------------------------------------------------------------ */ -static int gpstow_start = -1; -static double time_elapsed_sec = 0.0; - -/* - * Convert GPS Week and Seconds to Modified Julian Day. - * - Adapted from sci.astro FAQ. - * - Ignores UTC leap seconds. - */ -// in : week, gpssec -// out: jahr, monat, tag -static void Gps2Date(gpx_t *gpx) { - long GpsDays, Mjd; - long _J, _C, _Y, _M; - - GpsDays = gpx->week * 7 + (gpx->gpssec / 86400); - Mjd = 44244 + GpsDays; - - _J = Mjd + 2468570; - _C = 4 * _J / 146097; - _J = _J - (146097 * _C + 3) / 4; - _Y = 4000 * (_J + 1) / 1461001; - _J = _J - 1461 * _Y / 4 + 31; - _M = 80 * _J / 2447; - gpx->tag = _J - 2447 * _M / 80; - _J = _M / 11; - gpx->monat = _M + 2 - (12 * _J); - gpx->jahr = 100 * (_C - 49) + _Y + _J; -} -/* ------------------------------------------------------------------------------------ */ - -// ------------------------------------------------------------------------ - -static ui8_t vit_code[N]; -static vitCodes_init = 0; - -static int vit_initCodes(gpx_t *gpx) { - int cA, cB; - int i, bits; - - VIT_t *pv = calloc(1, sizeof(VIT_t)); - if (pv == NULL) return -1; - gpx->vit = pv; - - if ( vitCodes_init == 0 ) { - for (bits = 0; bits < N; bits++) { - cA = 0; - cB = 0; - for (i = 0; i < L; i++) { - cA ^= (polyA[L-1-i]&1) & ((bits >> i)&1); - cB ^= (polyB[L-1-i]&1) & ((bits >> i)&1); - } - vit_code[bits] = (cA<<1) | cB; - } - vitCodes_init = 1; - } - - return 0; -} - -static int vit_dist(int c, char *rc) { - return (((c>>1)^rc[0])&1) + ((c^rc[1])&1); -} - -static int vit_start(VIT_t *vit, char *rc) { - int t, m, j, c, d; - - t = L-1; - m = M; - while ( t > 0 ) { // t=0..L-2: nextStatestate[t][j].prevState = j/2; - } - t--; - m /= 2; - } - - m = 2; - for (t = 1; t < L; t++) { - for (j = 0; j < m; j++) { - c = vit_code[j]; - vit->state[t][j].bIn = j % 2; - vit->state[t][j].codeIn = c; - d = vit_dist( c, rc+2*(t-1) ); - vit->state[t][j].w = vit->state[t-1][vit->state[t][j].prevState].w + d; - } - m *= 2; - } - - return t; -} - -static int vit_next(VIT_t *vit, int t, char *rc) { - int b, nstate; - int j, index; - - for (j = 0; j < M; j++) { - for (b = 0; b < 2; b++) { - nstate = j*2 + b; - vit->d[nstate].bIn = b; - vit->d[nstate].codeIn = vit_code[nstate]; - vit->d[nstate].prevState = j; - vit->d[nstate].w = vit->state[t][j].w + vit_dist( vit->d[nstate].codeIn, rc ); - } - } - - for (j = 0; j < M; j++) { - - if ( vit->d[j].w <= vit->d[j+M].w ) index = j; else index = j+M; - - vit->state[t+1][j] = vit->d[index]; - } - - return 0; -} - -static int vit_path(VIT_t *vit, int j, int t) { - int c; - - vit->rawbits[2*t] = '\0'; - while (t > 0) { - c = vit->state[t][j].codeIn; - vit->rawbits[2*t -2] = 0x30 + ((c>>1) & 1); - vit->rawbits[2*t -1] = 0x30 + (c & 1); - j = vit->state[t][j].prevState; - t--; - } - - return 0; -} - -static int viterbi(VIT_t *vit, char *rc) { - int t, tmax; - int j, j_min, w_min; - - vit_start(vit, rc); - - tmax = strlen(rc)/2; - - for (t = L-1; t < tmax; t++) - { - vit_next(vit, t, rc+2*t); - } - - w_min = -1; - for (j = 0; j < M; j++) { - if (w_min < 0) { - w_min = vit->state[tmax][j].w; - j_min = j; - } - if (vit->state[tmax][j].w < w_min) { - w_min = vit->state[tmax][j].w; - j_min = j; - } - } - vit_path(vit, j_min, tmax); - - return 0; -} - -// ------------------------------------------------------------------------ - -static int deconv(char* rawbits, char *bits) { - - int j, n, bitA, bitB; - char *p; - int len; - int errors = 0; - int m = L-1; - - len = strlen(rawbits); - for (j = 0; j < m; j++) bits[j] = '0'; - n = 0; - while ( 2*(m+n) < len ) { - p = rawbits+2*(m+n); - bitA = bitB = 0; - for (j = 0; j < m; j++) { - bitA ^= (bits[n+j]&1) & (polyA[j]&1); - bitB ^= (bits[n+j]&1) & (polyB[j]&1); - } - if ( (bitA^(p[0]&1))==(polyA[m]&1) && (bitB^(p[1]&1))==(polyB[m]&1) ) bits[n+m] = '1'; - else if ( (bitA^(p[0]&1))==0 && (bitB^(p[1]&1))==0 ) bits[n+m] = '0'; - else { - if ( (bitA^(p[0]&1))!=(polyA[m]&1) && (bitB^(p[1]&1))==(polyB[m]&1) ) bits[n+m] = 0x39; - else bits[n+m] = 0x38; - errors = n; - break; - } - n += 1; - } - bits[n+m] = '\0'; - - return errors; -} - -// ------------------------------------------------------------------------ - -static int crc16_0(ui8_t frame[], int len) { - int crc16poly = 0x1021; - int rem = 0x0, i, j; - int byte; - - for (i = 0; i < len; i++) { - byte = frame[i]; - rem = rem ^ (byte << 8); - for (j = 0; j < 8; j++) { - if (rem & 0x8000) { - rem = (rem << 1) ^ crc16poly; - } - else { - rem = (rem << 1); - } - rem &= 0xFFFF; - } - } - return rem; -} - -static int check_CRC(ui8_t frame[]) { - ui32_t crclen = 0, - crcdat = 0; - - crclen = 221; - crcdat = (frame[crclen]<<8) | frame[crclen+1]; - if ( crcdat != crc16_0(frame, crclen) ) { - return 1; // CRC NO - } - else return 0; // CRC OK -} - -// ------------------------------------------------------------------------ - -static int bits2bytes(char *bitstr, ui8_t *bytes) { - int i, bit, d, byteval; - int len = strlen(bitstr)/8; - int bitpos, bytepos; - - bitpos = 0; - bytepos = 0; - - while (bytepos < len) { - - byteval = 0; - d = 1; - for (i = 0; i < BITS; i++) { - bit=*(bitstr+bitpos+i); /* little endian */ - //bit=*(bitstr+bitpos+7-i); /* big endian */ - if ((bit == '1') || (bit == '9')) byteval += d; - else /*if ((bit == '0') || (bit == '8'))*/ byteval += 0; - d <<= 1; - } - bitpos += BITS; - bytes[bytepos++] = byteval & 0xFF; - } - - //while (bytepos < FRAME_LEN+OVERLAP) bytes[bytepos++] = 0; - - return bytepos; -} - -/* -------------------------------------------------------------------------- */ - - -#define pos_SondeSN (OFS+0x00) // ?4 byte 00 7A.... -#define pos_FrameNb (OFS+0x04) // 2 byte -//GPS Position -#define pos_GPSTOW (OFS+0x06) // 4 byte -#define pos_GPSlat (OFS+0x0E) // 4 byte -#define pos_GPSlon (OFS+0x12) // 4 byte -#define pos_GPSalt (OFS+0x16) // 4 byte -//GPS Velocity East-North-Up (ENU) -#define pos_GPSvO (OFS+0x1A) // 3 byte -#define pos_GPSvN (OFS+0x1D) // 3 byte -#define pos_GPSvV (OFS+0x20) // 3 byte - - -static int get_SondeSN(gpx_t *gpx) { - unsigned byte; - - byte = (gpx->frame[pos_SondeSN]<<24) | (gpx->frame[pos_SondeSN+1]<<16) - | (gpx->frame[pos_SondeSN+2]<<8) | gpx->frame[pos_SondeSN+3]; - gpx->sn = byte & 0xFFFFFF; - - return 0; -} - -static int get_FrameNb(gpx_t *gpx) { - int i; - unsigned byte; - ui8_t frnr_bytes[2]; - int frnr; - - for (i = 0; i < 2; i++) { - byte = gpx->frame[pos_FrameNb + i]; - frnr_bytes[i] = byte; - } - - frnr = (frnr_bytes[0] << 8) + frnr_bytes[1] ; - gpx->frnr = frnr; - - return 0; -} - - -//char weekday[7][3] = { "So", "Mo", "Di", "Mi", "Do", "Fr", "Sa"}; -static char weekday[7][4] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"}; - -static int get_GPStime(gpx_t *gpx, int crc_err) { - int i; - unsigned byte; - ui8_t gpstime_bytes[4]; - int gpstime = 0, // 32bit - day; - float ms; - - for (i = 0; i < 4; i++) { - byte = gpx->frame[pos_GPSTOW + i]; - gpstime_bytes[i] = byte; - } - gpstime = 0; - for (i = 0; i < 4; i++) { - gpstime |= gpstime_bytes[i] << (8*(3-i)); - } - - if (gpstow_start < 0 && !crc_err) { - gpstow_start = gpstime; // time elapsed since start-up? - if (gpx->week > 0 && gpstime/1000.0 < time_elapsed_sec) gpx->week += 1; - } - gpx->gpstow = gpstime; - - ms = gpstime % 1000; - gpstime /= 1000; - gpx->gpssec = gpstime; - - day = gpstime / (24 * 3600); - gpstime %= (24*3600); - - if ((day < 0) || (day > 6)) return -1; - - gpx->wday = day; - gpx->std = gpstime / 3600; - gpx->min = (gpstime % 3600) / 60; - gpx->sek = gpstime % 60 + ms/1000.0; - - return 0; -} - -static double B60B60 = (1<<30)/90.0; // 2^32/360 = 2^30/90 = 0xB60B60.711x - -static int get_GPSlat(gpx_t *gpx) { - int i; - unsigned byte; - ui8_t gpslat_bytes[4]; - int gpslat; - double lat; - - for (i = 0; i < 4; i++) { - byte = gpx->frame[pos_GPSlat + i]; - gpslat_bytes[i] = byte; - } - - gpslat = 0; - for (i = 0; i < 4; i++) { - gpslat |= gpslat_bytes[i] << (8*(3-i)); - } - lat = gpslat / B60B60; - gpx->lat = lat; - - return 0; -} - -static int get_GPSlon(gpx_t *gpx) { - int i; - unsigned byte; - ui8_t gpslon_bytes[4]; - int gpslon; - double lon; - - for (i = 0; i < 4; i++) { - byte = gpx->frame[pos_GPSlon + i]; - gpslon_bytes[i] = byte; - } - - gpslon = 0; - for (i = 0; i < 4; i++) { - gpslon |= gpslon_bytes[i] << (8*(3-i)); - } - lon = gpslon / B60B60; - gpx->lon = lon; - - return 0; -} - -static int get_GPSalt(gpx_t *gpx) { - int i; - unsigned byte; - ui8_t gpsheight_bytes[4]; - int gpsheight; - double height; - - for (i = 0; i < 4; i++) { - byte = gpx->frame[pos_GPSalt + i]; - gpsheight_bytes[i] = byte; - } - - gpsheight = 0; - for (i = 0; i < 4; i++) { - gpsheight |= gpsheight_bytes[i] << (8*(3-i)); - } - height = gpsheight / 1000.0; - gpx->alt = height; - - if (height < -200 || height > 60000) return -1; - return 0; -} - -static int get_GPSvel24(gpx_t *gpx) { - int i; - unsigned byte; - ui8_t gpsVel_bytes[3]; - int vel24; - double vx, vy, vz, dir; //, alpha; - - for (i = 0; i < 3; i++) { - byte = gpx->frame[pos_GPSvO + i]; - gpsVel_bytes[i] = byte; - } - vel24 = gpsVel_bytes[0] << 16 | gpsVel_bytes[1] << 8 | gpsVel_bytes[2]; - if (vel24 > (0x7FFFFF)) vel24 -= 0x1000000; - vx = vel24 / 1e3; // ost - - for (i = 0; i < 3; i++) { - byte = gpx->frame[pos_GPSvN + i]; - gpsVel_bytes[i] = byte; - } - vel24 = gpsVel_bytes[0] << 16 | gpsVel_bytes[1] << 8 | gpsVel_bytes[2]; - if (vel24 > (0x7FFFFF)) vel24 -= 0x1000000; - vy= vel24 / 1e3; // nord - - for (i = 0; i < 3; i++) { - byte = gpx->frame[pos_GPSvV + i]; - gpsVel_bytes[i] = byte; - } - vel24 = gpsVel_bytes[0] << 16 | gpsVel_bytes[1] << 8 | gpsVel_bytes[2]; - if (vel24 > (0x7FFFFF)) vel24 -= 0x1000000; - vz = vel24 / 1e3; // hoch - - gpx->vE = vx; - gpx->vN = vy; - gpx->vU = vz; - - - gpx->vH = sqrt(vx*vx+vy*vy); -/* - alpha = atan2(vy, vx)*180/M_PI; // ComplexPlane (von x-Achse nach links) - GeoMeteo (von y-Achse nach rechts) - dir = 90-alpha; // z=x+iy= -> i*conj(z)=y+ix=re(i(pi/2-t)), Achsen und Drehsinn vertauscht - if (dir < 0) dir += 360; // atan2(y,x)=atan(y/x)=pi/2-atan(x/y) , atan(1/t) = pi/2 - atan(t) - gpx->vD2 = dir; -*/ - dir = atan2(vx, vy) * 180 / M_PI; - if (dir < 0) dir += 360; - gpx->vD = dir; - - gpx->vV = vz; - - return 0; -} - - -// RS(255,223)-CCSDS -#define rs_N 255 -#define rs_K 223 -#define rs_R (rs_N-rs_K) // 32 - -static int lms6_ecc(gpx_t *gpx, ui8_t *cw) { - int errors; - ui8_t err_pos[rs_R], - err_val[rs_R]; - - errors = rs_decode(&gpx->RS, cw, err_pos, err_val); - - return errors; -} - -static void print_frame(gpx_t *gpx, int crc_err, int len) { - int err=0; - - if (gpx->frame[0] != 0) - { - //if ((gpx->frame[pos_SondeSN+1] & 0xF0) == 0x70) // ? beginnen alle SNs mit 0x7A.... bzw 80..... ? - if ( gpx->frame[pos_SondeSN+1] ) - { - get_SondeSN(gpx); - get_FrameNb(gpx); - printf(" (%7d) ", gpx->sn); - printf(" [%5d] ", gpx->frnr); - err = get_GPStime(gpx, crc_err); - if (!err) printf("%s ", weekday[gpx->wday]); - if (gpx->week > 0) { - if (gpx->gpstow < gpstow_start && !crc_err) { - gpx->week += 1; // week roll-over - gpstow_start = gpx->gpstow; - } - Gps2Date(gpx); - fprintf(stdout, "%04d-%02d-%02d ", gpx->jahr, gpx->monat, gpx->tag); - } - printf("%02d:%02d:%06.3f ", gpx->std, gpx->min, gpx->sek); // falls Rundung auf 60s: Ueberlauf - - get_GPSlat(gpx); - get_GPSlon(gpx); - err = get_GPSalt(gpx); - if (!err) { - printf(" lat: %.5f ", gpx->lat); - printf(" lon: %.5f ", gpx->lon); - printf(" alt: %.2fm ", gpx->alt); - get_GPSvel24(gpx); - //if (gpx->option.vbs == 2) printf(" (%.1f ,%.1f,%.1f) ", gpx->vE, gpx->vN, gpx->vU); - printf(" vH: %.1fm/s D: %.1f vV: %.1fm/s ", gpx->vH, gpx->vD, gpx->vV); - } - - if (crc_err==0) printf(" [OK]"); else printf(" [NO]"); - - printf("\n"); - - - if (gpx->option.jsn) { - // Print JSON output required by auto_rx. - if (crc_err==0) { // CRC-OK - // UTC oder GPS? - printf("{ \"frame\": %d, \"id\": \"LMS6-%d\", \"datetime\": \"", gpx->frnr, gpx->sn ); - //if (gpx->week > 0) printf("%04d-%02d-%02dT", gpx->jahr, gpx->monat, gpx->tag ); - printf("%02d:%02d:%06.3fZ\", \"lat\": %.5f, \"lon\": %.5f, \"alt\": %.5f, \"vel_h\": %.5f, \"heading\": %.5f, \"vel_v\": %.5f", - gpx->std, gpx->min, gpx->sek, gpx->lat, gpx->lon, gpx->alt, gpx->vH, gpx->vD, gpx->vV ); - printf(", \"gpstow\": %d", gpx->gpstow ); - printf(" }\n"); - printf("\n"); - } - } - - } - } -} - - -static void proc_frame(gpx_t *gpx, int len) { - int blk_pos = SYNC_LEN; - ui8_t block_bytes[BLOCK_LEN+8]; - ui8_t rs_cw[rs_N]; - char frame_bits[BITFRAME_LEN+OVERLAP*BITS +8]; // init L-1 bits mit 0 - char *rawbits = NULL; - int i, j; - int err = 0; - int errs = 0; - int crc_err = 0; - int flen, blen; - - - if ((len % 8) > 4) { - while (len % 8) gpx->blk_rawbits[len++] = '0'; - } - gpx->blk_rawbits[len] = '\0'; - - flen = len / (2*BITS); - - if (gpx->option.vit == 1) { - viterbi(gpx->vit, gpx->blk_rawbits); - rawbits = gpx->vit->rawbits; - } - else rawbits = gpx->blk_rawbits; - - err = deconv(rawbits, frame_bits); - - if (err) { for (i=err; i < RAWBITBLOCK_LEN/2; i++) frame_bits[i] = 0; } - - - blen = bits2bytes(frame_bits, block_bytes); - for (j = blen; j < BLOCK_LEN+8; j++) block_bytes[j] = 0; - - - if (gpx->option.ecc) { - for (j = 0; j < rs_N; j++) rs_cw[rs_N-1-j] = block_bytes[SYNC_LEN+j]; - errs = lms6_ecc(gpx, rs_cw); - for (j = 0; j < rs_N; j++) block_bytes[SYNC_LEN+j] = rs_cw[rs_N-1-j]; - } - - if (gpx->option.raw == 2) { - for (i = 0; i < flen; i++) printf("%02x ", block_bytes[i]); - if (gpx->option.ecc) printf("(%d)", errs); - printf("\n"); - } - else if (gpx->option.raw == 4 && gpx->option.ecc) { - for (i = 0; i < rs_N; i++) printf("%02x", block_bytes[SYNC_LEN+i]); - printf(" (%d)", errs); - printf("\n"); - } - else if (gpx->option.raw == 8) { - if (gpx->option.vit == 1) { - for (i = 0; i < len; i++) printf("%c", gpx->vit->rawbits[i]); printf("\n"); - } - else { - for (i = 0; i < len; i++) printf("%c", gpx->blk_rawbits[i]); printf("\n"); - } - } - - blk_pos = SYNC_LEN; - - while ( blk_pos-SYNC_LEN < FRM_LEN ) { - - if (gpx->sf == 0) { - while ( blk_pos-SYNC_LEN < FRM_LEN ) { - gpx->sf = 0; - for (j = 0; j < 4; j++) gpx->sf += (block_bytes[blk_pos+j] == frm_sync[j]); - if (gpx->sf == 4) { - gpx->frm_pos = 0; - break; - } - blk_pos++; - } - } - - if ( gpx->sf && gpx->frm_pos < FRM_LEN ) { - gpx->frame[gpx->frm_pos] = block_bytes[blk_pos]; - gpx->frm_pos++; - blk_pos++; - } - - if (gpx->frm_pos == FRM_LEN) { - - crc_err = check_CRC(gpx->frame); - - if (gpx->option.raw == 1) { - for (i = 0; i < FRM_LEN; i++) printf("%02x ", gpx->frame[i]); - if (crc_err==0) printf(" [OK]"); else printf(" [NO]"); - printf("\n"); - } - - if (gpx->option.raw == 0) print_frame(gpx, crc_err, len); - - gpx->frm_pos = 0; - gpx->sf = 0; - } - - } - -} - - -int main(int argc, char **argv) { - - int option_inv = 0; // invertiert Signal - int option_iq = 0; - int option_lp = 0; - int option_dc = 0; - int wavloaded = 0; - int sel_wavch = 0; // audio channel: left - int gpsweek = 0; - - FILE *fp = NULL; - char *fpname = NULL; - - int k; - - int bit, rbit; - int bitpos = 0; - int bitQ; - int pos; - //int headerlen = 0; - - int header_found = 0; - - float thres = 0.76; - float _mv = 0.0; - - int symlen = 1; - int bitofs = 1; // +1 .. +2 - int shift = 0; - - unsigned int bc = 0; - - pcm_t pcm = {0}; - dsp_t dsp = {0}; //memset(&dsp, 0, sizeof(dsp)); -/* - // gpx_t _gpx = {0}; gpx_t *gpx = &_gpx; // stack size ... - gpx_t *gpx = NULL; - gpx = calloc(1, sizeof(gpx_t)); - //memset(gpx, 0, sizeof(gpx_t)); -*/ - gpx_t _gpx = {0}; gpx_t *gpx = &_gpx; - - -#ifdef CYGWIN - _setmode(fileno(stdin), _O_BINARY); // _setmode(_fileno(stdin), _O_BINARY); -#endif - setbuf(stdout, NULL); - - - fpname = argv[0]; - ++argv; - while ((*argv) && (!wavloaded)) { - if ( (strcmp(*argv, "-h") == 0) || (strcmp(*argv, "--help") == 0) ) { - fprintf(stderr, "%s [options] audio.wav\n", fpname); - fprintf(stderr, " options:\n"); - fprintf(stderr, " -v, --verbose\n"); - fprintf(stderr, " -r, --raw\n"); - fprintf(stderr, " --vit (Viterbi)\n"); - fprintf(stderr, " --ecc (Reed-Solomon)\n"); - return 0; - } - else if ( (strcmp(*argv, "-v") == 0) || (strcmp(*argv, "--verbose") == 0) ) { - gpx->option.vbs = 1; - } - else if ( (strcmp(*argv, "-r") == 0) || (strcmp(*argv, "--raw") == 0) ) { - gpx->option.raw = 1; // bytes - rs_ecc_codewords - } - else if ( (strcmp(*argv, "-r0") == 0) || (strcmp(*argv, "--raw0") == 0) ) { - gpx->option.raw = 2; // bytes: sync + codewords - } - else if ( (strcmp(*argv, "-rc") == 0) || (strcmp(*argv, "--rawecc") == 0) ) { - gpx->option.raw = 4; // rs_ecc_codewords - } - else if ( (strcmp(*argv, "-R") == 0) || (strcmp(*argv, "--RAW") == 0) ) { - gpx->option.raw = 8; // rawbits - } - else if (strcmp(*argv, "--ecc" ) == 0) { gpx->option.ecc = 1; } // RS-ECC - else if (strcmp(*argv, "--vit" ) == 0) { gpx->option.vit = 1; } // viterbi - else if ( (strcmp(*argv, "--gpsweek") == 0) ) { - ++argv; - if (*argv) { - gpsweek = atoi(*argv); - if (gpsweek < 1024 || gpsweek > 3072) gpsweek = 0; - } - else return -1; - } - else if ( (strcmp(*argv, "-i") == 0) || (strcmp(*argv, "--invert") == 0) ) { - option_inv = 1; // nicht noetig - } - else if ( (strcmp(*argv, "--dc") == 0) ) { - option_dc = 1; - } - else if ( (strcmp(*argv, "--ch2") == 0) ) { sel_wavch = 1; } // right channel (default: 0=left) - else if ( (strcmp(*argv, "--ths") == 0) ) { - ++argv; - if (*argv) { - thres = atof(*argv); - } - else return -1; - } - else if ( (strcmp(*argv, "-d") == 0) ) { - ++argv; - if (*argv) { - shift = atoi(*argv); - if (shift > 4) shift = 4; - if (shift < -4) shift = -4; - } - else return -1; - } - else if (strcmp(*argv, "--iq0") == 0) { option_iq = 1; } // differential/FM-demod - else if (strcmp(*argv, "--iq2") == 0) { option_iq = 2; } - else if (strcmp(*argv, "--iq3") == 0) { option_iq = 3; } // iq2==iq3 - else if (strcmp(*argv, "--IQ") == 0) { // fq baseband -> IF (rotate from and decimate) - double fq = 0.0; // --IQ , -0.5 < fq < 0.5 - ++argv; - if (*argv) fq = atof(*argv); - else return -1; - if (fq < -0.5) fq = -0.5; - if (fq > 0.5) fq = 0.5; - dsp.xlt_fq = -fq; // S(t) -> S(t)*exp(-f*2pi*I*t) - option_iq = 5; - } - else if (strcmp(*argv, "--lp") == 0) { option_lp = 1; } // IQ lowpass - else if (strcmp(*argv, "--json") == 0) { - gpx->option.jsn = 1; - gpx->option.ecc = 1; - gpx->option.vit = 1; - } - else { - fp = fopen(*argv, "rb"); - if (fp == NULL) { - fprintf(stderr, "%s konnte nicht geoeffnet werden\n", *argv); - return -1; - } - wavloaded = 1; - } - ++argv; - } - if (!wavloaded) fp = stdin; - - - if (gpx->option.raw == 4) gpx->option.ecc = 1; - - // init gpx - memcpy(gpx->blk_rawbits, blk_syncbits, sizeof(blk_syncbits)); - memcpy(gpx->frame, frm_sync, sizeof(frm_sync)); - gpx->frm_pos = 0; // ecc_blk <-> frm_blk - gpx->sf = 0; - - gpx->option.inv = option_inv; // irrelevant - - gpx->week = gpsweek; - - if (option_iq) sel_wavch = 0; - - pcm.sel_ch = sel_wavch; - k = read_wav_header(&pcm, fp); - if ( k < 0 ) { - fclose(fp); - fprintf(stderr, "error: wav header\n"); - return -1; - } - - symlen = 1; - - // init dsp - // - dsp.fp = fp; - dsp.sr = pcm.sr; - dsp.bps = pcm.bps; - dsp.nch = pcm.nch; - dsp.ch = pcm.sel_ch; - dsp.br = (float)BAUD_RATE; - dsp.sps = (float)dsp.sr/dsp.br; - dsp.symlen = symlen; - dsp.symhd = 1; - dsp._spb = dsp.sps*symlen; - dsp.hdr = rawheader; - dsp.hdrlen = strlen(rawheader); - dsp.BT = 1.5; // bw/time (ISI) // 1.0..2.0 - dsp.h = 0.9; // 1.0 modulation index - dsp.lpIQ_bw = 8e3; - dsp.opt_iq = option_iq; - dsp.opt_lp = option_lp; - - if ( dsp.sps < 8 ) { - fprintf(stderr, "note: sample rate low (%.1f sps)\n", dsp.sps); - } - - //headerlen = dsp.hdrlen; - - k = init_buffers(&dsp); - if ( k < 0 ) { - fprintf(stderr, "error: init buffers\n"); - return -1; - }; - - - if (gpx->option.vit) { - k = vit_initCodes(gpx); - if (k < 0) return -1; - } - if (gpx->option.ecc) { - rs_init_RS255ccsds(&gpx->RS); // bch_ecc.c - } - - - bitofs += shift; - - - while ( 1 ) - { - - header_found = find_header(&dsp, thres, 3, bitofs, option_dc); - _mv = dsp.mv; - - if (header_found == EOF) break; - - // mv == correlation score - if (_mv*(0.5-gpx->option.inv) < 0) { - gpx->option.inv ^= 0x1; // LMS6: irrelevant - } - - if (header_found) { - - bitpos = 0; - pos = BLOCKSTART; - - if (_mv > 0) bc = 0; else bc = 1; - - while ( pos < RAWBITBLOCK_LEN ) { - - bitQ = read_slbit(&dsp, &rbit, 0, bitofs, bitpos, -1, 0); // symlen=1 - - if (bitQ == EOF) { break; } - - bit = rbit ^ (bc%2); // (c0,inv(c1)) - gpx->blk_rawbits[pos] = 0x30 + bit; - - bc++; - pos++; - bitpos += 1; - } - - gpx->blk_rawbits[pos] = '\0'; - - time_elapsed_sec = dsp.sample_in / (double)dsp.sr; - proc_frame(gpx, pos); - - if (pos < RAWBITBLOCK_LEN) break; - - pos = BLOCKSTART; - header_found = 0; - } - - } - - - free_buffers(&dsp); - if (gpx->vit) { free(gpx->vit); gpx->vit = NULL; } - - fclose(fp); - - return 0; -} - From bbb432274e3978d286f9d3d336c6349c52a5cc75 Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Fri, 31 May 2024 19:43:49 +0930 Subject: [PATCH 35/42] Add support for WxR-301 PN9 variant --- auto_rx/autorx/__init__.py | 2 +- auto_rx/autorx/config.py | 3 +- auto_rx/autorx/decode.py | 76 +++++++++++++++++++++- auto_rx/autorx/scan.py | 9 +++ auto_rx/autorx/sondehub.py | 10 +++ auto_rx/autorx/templates/index.html | 1 + auto_rx/autorx/utils.py | 4 ++ scan/dft_detect.c | 55 +++++++++------- weathex/weathex301d.c | 98 +++++++++++++++++------------ 9 files changed, 189 insertions(+), 69 deletions(-) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index 1c75940f..22d92de3 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -12,7 +12,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.7.3-beta11" +__version__ = "1.7.3-beta12" # Global Variables diff --git a/auto_rx/autorx/config.py b/auto_rx/autorx/config.py index e291747f..faf6d544 100644 --- a/auto_rx/autorx/config.py +++ b/auto_rx/autorx/config.py @@ -428,7 +428,8 @@ def read_auto_rx_config(filename, no_sdr_test=False): "MEISEI": True, "MTS01": False, # Until we test it "MRZ": False, # .... except for the MRZ, until we know it works. - "WXR301": True, # No fsk_demod chain for this yet. + "WXR301": True, + "WXRPN9": True, "UDP": False, } diff --git a/auto_rx/autorx/decode.py b/auto_rx/autorx/decode.py index 85f859a0..639b5b96 100644 --- a/auto_rx/autorx/decode.py +++ b/auto_rx/autorx/decode.py @@ -40,7 +40,8 @@ "MRZ", "MTS01", "UDP", - "WXR301" + "WXR301", + "WXRPN9" ] # Known 'Drifty' Radiosonde types @@ -120,7 +121,8 @@ class SondeDecoder(object): "MRZ", "MTS01", "UDP", - "WXR301" + "WXR301", + "WXRPN9" ] def __init__( @@ -762,7 +764,31 @@ def generate_decoder_command(self): # WXR301, via iq_dec as a FM Demod. decode_cmd += f"./iq_dec --FM --IFbw {_if_bw} --lpFM --wav --iq 0.0 - {_sample_rate} 16 2>/dev/null | ./weathex301d -b --json" + elif self.sonde_type == "WXRPN9": + # Weathex WxR-301D (PN9) + + _sample_rate = 96000 + _if_bw = 64 + decode_cmd = get_sdr_iq_cmd( + sdr_type = self.sdr_type, + frequency = self.sonde_freq, + sample_rate = _sample_rate, + sdr_hostname = self.sdr_hostname, + sdr_port = self.sdr_port, + ss_iq_path = self.ss_iq_path, + rtl_device_idx = self.rtl_device_idx, + ppm = self.ppm, + gain = self.gain, + bias = self.bias + ) + + # Add in tee command to save IQ to disk if debugging is enabled. + if self.save_decode_iq: + decode_cmd += f" tee {self.save_decode_iq_path} |" + + # WXR301, via iq_dec as a FM Demod. + decode_cmd += f"./iq_dec --FM --IFbw {_if_bw} --lpFM --wav --iq 0.0 - {_sample_rate} 16 2>/dev/null | ./weathex301d -b --json --pn9" elif self.sonde_type == "UDP": # UDP Input Mode. @@ -1313,6 +1339,50 @@ def generate_decoder_command_experimental(self): demod_stats = FSKDemodStats(averaging_time=5.0, peak_hold=True) self.rx_frequency = self.sonde_freq + elif self.sonde_type == "WXRPN9": + # Weathex WxR-301D Sonde, PN9 variant + + _baud_rate = 4800 + _sample_rate = 96000 + + # Limit FSK estimator window to roughly +/- 40 kHz + _lower = -40000 + _upper = 40000 + + demod_cmd = get_sdr_iq_cmd( + sdr_type = self.sdr_type, + frequency = self.sonde_freq, + sample_rate = _sample_rate, + sdr_hostname = self.sdr_hostname, + sdr_port = self.sdr_port, + ss_iq_path = self.ss_iq_path, + rtl_device_idx = self.rtl_device_idx, + ppm = self.ppm, + gain = self.gain, + bias = self.bias, + dc_block = True + ) + + # Add in tee command to save IQ to disk if debugging is enabled. + if self.save_decode_iq: + demod_cmd += f" tee {self.save_decode_iq_path} |" + + # Trying out using the mask estimator here to reduce issues with interference + demod_cmd += "./fsk_demod --cs16 -s -b %d -u %d --mask 60000 --stats=%d 2 %d %d - -" % ( + _lower, + _upper, + _stats_rate, + _sample_rate, + _baud_rate, + ) + + # Soft-decision decoding, inverted. + decode_cmd = f"./weathex301d --softin -i --json --pn9 2>/dev/null" + + # Weathex sondes transmit continuously - average over the last frame, and use a peak hold + demod_stats = FSKDemodStats(averaging_time=5.0, peak_hold=True) + self.rx_frequency = self.sonde_freq + else: return None @@ -1700,7 +1770,7 @@ def handle_decoder_line(self, data): # Weathex Specific Actions # Same datetime issues as with iMets, and LMS6 - if self.sonde_type == "WXR301": + if (self.sonde_type == "WXR301") or (self.sonde_type == "WXRPN9"): # Fix up the time. _telemetry["datetime_dt"] = fix_datetime(_telemetry["datetime"]) # Re-generate the datetime string. diff --git a/auto_rx/autorx/scan.py b/auto_rx/autorx/scan.py index fe040da5..d571f4bf 100644 --- a/auto_rx/autorx/scan.py +++ b/auto_rx/autorx/scan.py @@ -620,6 +620,15 @@ def detect_sonde( # to do no whitening on the signal. _offset_est = 0.0 + elif "WXRPN9" in _type: + logging.debug( + "Scanner (%s) - Detected a Weathex WxR-301D Sonde (PN9 Variant)! (Score: %.2f, Offset: %.1f Hz)" + % (_sdr_name, _score, _offset_est) + ) + _sonde_type = "WXRPN9" + # Clear out the offset estimate for WxR-301's as it's not accurate + # to do no whitening on the signal. + _offset_est = 0.0 else: _sonde_type = None diff --git a/auto_rx/autorx/sondehub.py b/auto_rx/autorx/sondehub.py index 296ce352..e5137dd3 100644 --- a/auto_rx/autorx/sondehub.py +++ b/auto_rx/autorx/sondehub.py @@ -233,6 +233,16 @@ def reformat_data(self, telemetry): _output["type"] = "WxR-301D" _output["serial"] = telemetry["id"].split("-")[1] + # Double check for the subtype being present, just in case... + if "subtype" in telemetry: + if telemetry["subtype"] == "WXR_PN9": + _output["type"] = "WxR-301D (PN9)" + + elif telemetry["type"] == "WXRPN9": + _output["manufacturer"] = "Weathex" + _output["type"] = "WxR-301D (PN9)" + _output["serial"] = telemetry["id"].split("-")[1] + else: self.log_error("Unknown Radiosonde Type %s" % telemetry["type"]) return None diff --git a/auto_rx/autorx/templates/index.html b/auto_rx/autorx/templates/index.html index 8b4b2be8..f03c0e00 100644 --- a/auto_rx/autorx/templates/index.html +++ b/auto_rx/autorx/templates/index.html @@ -1665,6 +1665,7 @@

Decoder Control

+
diff --git a/auto_rx/autorx/utils.py b/auto_rx/autorx/utils.py index 9c789f05..ac5f3c5e 100644 --- a/auto_rx/autorx/utils.py +++ b/auto_rx/autorx/utils.py @@ -212,6 +212,8 @@ def short_type_lookup(type_name): return "Meteosis MTS01" elif type_name == "WXR301": return "Weathex WxR-301D" + elif type_name == "WXRPN9": + return "Weathex WxR-301D (PN9 Variant)" else: return "Unknown" @@ -256,6 +258,8 @@ def short_short_type_lookup(type_name): return "MTS01" elif type_name == "WXR301": return "WXR301" + elif type_name == "WXRPN9": + return "WXR301(PN9)" else: return "Unknown" diff --git a/scan/dft_detect.c b/scan/dft_detect.c index 32ea4cd4..ea39222d 100644 --- a/scan/dft_detect.c +++ b/scan/dft_detect.c @@ -116,6 +116,10 @@ static char weathex_header[] = "10101010""10101010""10101010" // AA AA AA (preamble) "00101101""11010100"; //"10101010"; // 2D D4 55/AA +static char wxr2pn9_header[] = + "10101010""10101010""10101010" // AA AA AA (preamble) + "11000001""10010100"; //"11000001"; // C1 94 C1 + typedef struct { int sps; // header: symbol rate, baud @@ -140,28 +144,29 @@ static float lpFM_bw[2] = { 4e3, 10e3 }; // FM-audio lowpass bandwidth static float lpIQ_bw[N_bwIQ] = { 6e3, 12e3, 22e3, 200e3 }; // IF iq lowpass bandwidth static float set_lpIQ = 0.0; -#define tn_DFM 2 -#define tn_RS41 3 -#define tn_RS92 4 -#define tn_M10 5 -#define tn_M20 6 -#define tn_LMS6 8 -#define tn_MEISEI 9 -#define tn_MRZ 12 -#define tn_MTS01 13 -#define tn_C34C50 15 -#define tn_WXR301 16 -#define tn_MK2LMS 18 -#define tn_IMET5 24 -#define tn_IMETa 25 -#define tn_IMET4 26 -#define tn_IMET1rs 28 -#define tn_IMET1ab 29 - -#define Nrs 16 -#define idxIMETafsk 13 -#define idxRS 14 -#define idxI4 15 +#define tn_DFM 2 +#define tn_RS41 3 +#define tn_RS92 4 +#define tn_M10 5 +#define tn_M20 6 +#define tn_LMS6 8 +#define tn_MEISEI 9 +#define tn_MRZ 12 +#define tn_MTS01 13 +#define tn_C34C50 15 +#define tn_WXR301 16 +#define tn_WXRpn9 17 +#define tn_MK2LMS 18 +#define tn_IMET5 24 +#define tn_IMETa 25 +#define tn_IMET4 26 +#define tn_IMET1rs 28 +#define tn_IMET1ab 29 + +#define Nrs 17 +#define idxIMETafsk 14 +#define idxRS 15 +#define idxI4 16 static rsheader_t rs_hdr[Nrs] = { { 2500, 0, 0, dfm_header, 1.0, 0.0, 0.65, 2, NULL, "DFM9", tn_DFM, 0, 1, 0.0, 0.0}, // DFM6: -2 ? { 4800, 0, 0, rs41_header, 0.5, 0.0, 0.70, 2, NULL, "RS41", tn_RS41, 0, 1, 0.0, 0.0}, @@ -175,6 +180,7 @@ static rsheader_t rs_hdr[Nrs] = { { 1200, 0, 0, mts01_header, 1.0, 0.0, 0.65, 2, NULL, "MTS01", tn_MTS01, 0, 0, 0.0, 0.0}, { 5800, 0, 0, c34_preheader, 1.5, 0.0, 0.80, 2, NULL, "C34C50", tn_C34C50, 0, 2, 0.0, 0.0}, // C34/C50 2900 Hz tone { 4800, 0, 0, weathex_header, 1.0, 0.0, 0.65, 2, NULL, "WXR301", tn_WXR301, 0, 3, 0.0, 0.0}, + { 5000, 0, 0, wxr2pn9_header, 1.0, 0.0, 0.65, 2, NULL, "WXRPN9", tn_WXRpn9, 0, 3, 0.0, 0.0}, { 9600, 0, 0, imet1ab_header, 1.0, 0.0, 0.80, 2, NULL, "IMET1AB", tn_IMET1ab, 1, 3, 0.0, 0.0}, // (rs_hdr[idxAB]) { 9600, 0, 0, imet_preamble, 0.5, 0.0, 0.80, 4, NULL, "IMETafsk", tn_IMETa , 1, 1, 0.0, 0.0}, // IMET1AB, IMET1RS (IQ)IMET4 { 9600, 0, 0, imet1rs_header, 0.5, 0.0, 0.80, 2, NULL, "IMET1RS", tn_IMET1rs, 0, 3, 0.0, 0.0}, // (rs_hdr[idxRS]) IMET4: lpIQ=0 ... @@ -184,6 +190,7 @@ static rsheader_t rs_hdr[Nrs] = { static int idx_MTS01 = -1, idx_C34C50 = -1, idx_WXR301 = -1, + idx_WXRPN9 = -1, idx_IMET1AB = -1; @@ -1143,6 +1150,7 @@ static int init_buffers() { #endif #ifdef NOWXR301 if ( strncmp(rs_hdr[j].type, "WXR301", 5) == 0 ) idx_WXR301 = j; + if ( strncmp(rs_hdr[j].type, "WXRPN9", 5) == 0 ) idx_WXRPN9 = j; #endif #ifdef NOIMET1AB if ( strncmp(rs_hdr[j].type, "IMET1AB", 7) == 0 ) idx_IMET1AB = j; @@ -1153,7 +1161,7 @@ static int init_buffers() { rs_hdr[j].spb = sample_rate/(float)rs_hdr[j].sps; rs_hdr[j].hLen = strlen(rs_hdr[j].header); rs_hdr[j].L = rs_hdr[j].hLen * rs_hdr[j].spb + 0.5; - if (j != idx_MTS01 && j != idx_C34C50 && j != idx_WXR301 && j != idx_IMET1AB) { + if (j != idx_MTS01 && j != idx_C34C50 && j != idx_WXR301 && j != idx_WXRPN9 && j != idx_IMET1AB) { if (rs_hdr[j].hLen > hLen) hLen = rs_hdr[j].hLen; if (rs_hdr[j].L > Lmax) Lmax = rs_hdr[j].L; } @@ -1477,6 +1485,7 @@ int main(int argc, char **argv) { if ( j == idx_MTS01 ) continue; // only ifdef NOMTS01 if ( j == idx_C34C50 ) continue; // only ifdef NOC34C50 if ( j == idx_WXR301 ) continue; // only ifdef NOWXR301 + if ( j == idx_WXRPN9 ) continue; // only ifdef NOWXR301 if ( j == idx_IMET1AB ) continue; // only ifdef NOIMET1AB mv0_pos[j] = mv_pos[j]; diff --git a/weathex/weathex301d.c b/weathex/weathex301d.c index ff210b9b..d50a770d 100644 --- a/weathex/weathex301d.c +++ b/weathex/weathex301d.c @@ -1,10 +1,8 @@ /* - Malaysia - 401100 kHz (64kHz wide) - 2023-05-12 ([ 4400] 12:20:37 alt: 12616.6 lat: 2.6785 lon: 101.5827) - 2023-07-27 ([ 6402] 00:47:32 alt: 26835.9 lat: 2.6918 lon: 101.5025) - Weathex WxR-301D w/o PN9 + Weathex WxR-301D (64kHz wide) + UAII2022 Lindenberg: w/ PN9, 5000 baud + Malaysia: w/o PN9, 4800 baud */ #include @@ -38,25 +36,21 @@ int option_verbose = 0, wavloaded = 0; int wav_channel = 0; // audio channel: left +int option_pn9 = 0; -#define BAUD_RATE 4800.0 // (4997.2) // 5000 +#define BAUD_RATE 4800.0 +#define BAUD_RATE_PN9 5000.0 //(4997.2) // 5000 #define FRAMELEN 69 //64 #define BITFRAMELEN (8*FRAMELEN) -/* -#define HEADLEN 56 -#define HEADOFS 0 -char header[] = "10101010""10101010""10101010" // AA AA AA (preamble) - "11000001""10010100""11000001"; // C1 94 C1 -*/ -//preamble_header_sn1: 101010101010101010101010 1100000110010100110000011100011001111000 110001010110110111100100 -//preamble_header_sn2: 101010101010101010101010 1100000110010100110000011100011001111000 001100100110110111100100 -//preamble_header_sn3: 101010101010101010101010 1100000110010100110000011100011001111000 001010000110110111100100 -#define HEADLEN 40 //48 +#define HEADLEN 40 #define HEADOFS 0 +char header_pn9[] = "10101010""10101010""10101010"//"10101010" // AA AA AA (preamble) + "11000001""10010100"; //"11000001""11000110"; // C1 94 (C1 C6) + char header[] = "10101010""10101010""10101010" // AA AA AA (preamble) - "00101101""11010100"; //"10101010"; // 2D D4 55/AA + "00101101""11010100"; //"10101010"; // 2D D4 (55/AA) char buf[HEADLEN+1] = "xxxxxxxxxx\0"; int bufpos = 0; @@ -65,6 +59,7 @@ char frame_bits[BITFRAMELEN+1]; ui8_t frame_bytes[FRAMELEN+1]; ui8_t xframe[FRAMELEN+1]; +float baudrate = BAUD_RATE; /* ------------------------------------------------------------------------------------ */ @@ -131,7 +126,7 @@ int read_wav_header(FILE *fp) { if (sample_rate == 900001) sample_rate -= 1; - samples_per_bit = sample_rate/(float)BAUD_RATE; + samples_per_bit = sample_rate/(float)baudrate; fprintf(stderr, "samples/bit: %.2f\n", samples_per_bit); @@ -252,9 +247,9 @@ int f32soft_read(FILE *fp, float *s) { } -int compare() { +int compare(char *hdr) { int i=0; - while ((i < HEADLEN) && (buf[(bufpos+i) % HEADLEN] == header[HEADLEN+HEADOFS-1-i])) { + while ((i < HEADLEN) && (buf[(bufpos+i) % HEADLEN] == hdr[HEADLEN+HEADOFS-1-i])) { i++; } return i; @@ -266,9 +261,9 @@ char inv(char c) { return c; } -int compare2() { +int compare2(char *hdr) { int i=0; - while ((i < HEADLEN) && (buf[(bufpos+i) % HEADLEN] == inv(header[HEADLEN+HEADOFS-1-i]))) { + while ((i < HEADLEN) && (buf[(bufpos+i) % HEADLEN] == inv(hdr[HEADLEN+HEADOFS-1-i]))) { i++; } return i; @@ -307,8 +302,8 @@ int bits2bytes(char *bitstr, ui8_t *bytes) { // cf. https://www.ti.com/lit/an/swra322/swra322.pdf // https://destevez.net/2019/07/lucky-7-decoded/ // -// counter low byte: frame[OFS+4] XOR 0xCC -// zero bytes, frame[OFS+30]: 0C CA C9 FB 49 37 E5 A8 +// counter low byte: frame[ofs+4] XOR 0xCC +// zero bytes, frame[ofs+30]: 0C CA C9 FB 49 37 E5 A8 // ui8_t PN9b[64] = { 0xFF, 0x87, 0xB8, 0x59, 0xB7, 0xA1, 0xCC, 0x24, 0x57, 0x5E, 0x4B, 0x9C, 0x0E, 0xE9, 0xEA, 0x50, @@ -356,7 +351,10 @@ typedef struct { gpx_t gpx; -#define OFS 6 // xPN9: OFS=8, different baud +// xPN9: OFS=8, 5000 baud ; w/o PN9: OFS=6, 4800 baud +#define OFS 6 +#define OFS_PN9 8 +int ofs = OFS; int print_frame() { int j; @@ -366,12 +364,14 @@ int print_frame() { for (j = 0; j < FRAMELEN; j++) { ui8_t b = frame_bytes[j]; - //if (j >= 6) b ^= PN9b[(j-6)%64]; // PN9 baud diff + if (option_pn9) { + if (j >= 6) b ^= PN9b[(j-6)%64]; + } xframe[j] = b; } - chkval = xor8sum(xframe+OFS, 53); - chkdat = (xframe[OFS+53]<<8) | xframe[OFS+53+1]; + chkval = xor8sum(xframe+ofs, 53); + chkdat = (xframe[ofs+53]<<8) | xframe[ofs+53+1]; chk_ok = (chkdat == chkval); if (option_raw) { @@ -398,12 +398,12 @@ int print_frame() { int val; // SN - sn = xframe[OFS] | (xframe[OFS+1]<<8) | (xframe[OFS+2]<<16) | (xframe[OFS+3]<<24); + sn = xframe[ofs] | (xframe[ofs+1]<<8) | (xframe[ofs+2]<<16) | (xframe[ofs+3]<<24); // counter - cnt = xframe[OFS+4] | (xframe[OFS+5]<<8); + cnt = xframe[ofs+4] | (xframe[ofs+5]<<8); - ui8_t frid = xframe[OFS+6]; + ui8_t frid = xframe[ofs+6]; if (frid == 1) { @@ -436,7 +436,7 @@ int print_frame() { // time/UTC int hms; - hms = xframe[OFS+7] | (xframe[OFS+8]<<8) | (xframe[OFS+9]<<16); + hms = xframe[ofs+7] | (xframe[ofs+8]<<8) | (xframe[ofs+9]<<16); hms &= 0x3FFFF; //printf(" (%6d) ", hms); ui8_t h = hms / 10000; @@ -448,30 +448,35 @@ int print_frame() { gpx.sec = s; // alt - val = xframe[OFS+13] | (xframe[OFS+14]<<8) | (xframe[OFS+15]<<16); + val = xframe[ofs+13] | (xframe[ofs+14]<<8) | (xframe[ofs+15]<<16); val >>= 4; val &= 0x7FFFF; // int19 ? //if (val & 0x40000) val -= 0x80000; ?? or sign bit ? float alt = val / 10.0f; printf(" alt: %.1f ", alt); // MSL gpx.alt = alt; + int val_alt = val; // lat - val = xframe[OFS+15] | (xframe[OFS+16]<<8) | (xframe[OFS+17]<<16) | (xframe[OFS+18]<<24); + val = xframe[ofs+15] | (xframe[ofs+16]<<8) | (xframe[ofs+17]<<16) | (xframe[ofs+18]<<24); val >>= 7; val &= 0x1FFFFFF; // int25 ? ?? sign NMEA N/S ? //if (val & 0x1000000) val -= 0x2000000; // sign bit ? (or 90 -> -90 wrap ?) float lat = val / 1e5f; printf(" lat: %.4f ", lat); gpx.lat = lat; + int val_lat = val; // lon - val = xframe[OFS+19] | (xframe[OFS+20]<<8) | (xframe[OFS+21]<<16)| (xframe[OFS+22]<<24); + val = xframe[ofs+19] | (xframe[ofs+20]<<8) | (xframe[ofs+21]<<16)| (xframe[ofs+22]<<24); val &= 0x3FFFFFF; // int26 ? ?? sign NMEA E/W ? //if (val & 0x2000000) val -= 0x4000000; // or sign bit ? (or 180 -> -180 wrap ?) float lon = val / 1e5f; printf(" lon: %.4f ", lon); gpx.lon = lon; + int val_lon = val; + + int zero_pos = val_alt == 0 && val_lat == 0 && val_lon == 0; // checksum printf(" %s", chk_ok ? "[OK]" : "[NO]"); @@ -480,7 +485,7 @@ int print_frame() { printf("\n"); // JSON - if (option_json && gpx.chk2ok) { + if (option_json && gpx.chk2ok && !zero_pos) { if (gpx.chk1ok && gpx.sn2 == gpx.sn1 && gpx.cnt2 == gpx.cnt1) // double check, unreliable checksums { char *ver_jsn = NULL; @@ -493,6 +498,10 @@ int print_frame() { // if data from subframe1, // check gpx.chk1ok && gpx.sn1==gpx.sn2 && gpx.cnt1==gpx.cnt2 + if (option_pn9) { + fprintf(stdout, ", \"subtype\": \"WXR_PN9\""); + } + if (gpx.jsn_freq > 0) { fprintf(stdout, ", \"freq\": %d", gpx.jsn_freq ); } @@ -527,6 +536,7 @@ int main(int argc, char **argv) { int header_found = 0; int cfreq = -1; + char *hdr = header; fpname = argv[0]; ++argv; @@ -538,6 +548,7 @@ int main(int argc, char **argv) { fprintf(stderr, " -b\n"); return 0; } + else if (strcmp(*argv, "--pn9") == 0) { option_pn9 = 1; } else if ( (strcmp(*argv, "-i") == 0) || (strcmp(*argv, "--invert") == 0) ) { option_inv = 1; } @@ -575,6 +586,11 @@ int main(int argc, char **argv) { } if (!wavloaded) fp = stdin; + if (option_pn9) { + baudrate = BAUD_RATE_PN9; + hdr = header_pn9; + ofs = OFS_PN9; + } if ( !option_softin ) { i = read_wav_header(fp); @@ -592,7 +608,7 @@ int main(int argc, char **argv) { { float s = 0.0f; int bit = 0; - sample_rate = BAUD_RATE; + sample_rate = baudrate; sample_count = 0; while (!f32soft_read(fp, &s)) { @@ -605,12 +621,12 @@ int main(int argc, char **argv) { if (!header_found) { - h = compare(); //h2 = compare2(); + h = compare(hdr); //h2 = compare2(hdr); if ((h >= HEADLEN)) { header_found = 1; fflush(stdout); if (option_timestamp) printf("<%8.3f> ", sample_count/(double)sample_rate); - strncpy(frame_bits, header, HEADLEN); + strncpy(frame_bits, hdr, HEADLEN); bit_count += HEADLEN; frames++; } @@ -649,12 +665,12 @@ int main(int argc, char **argv) { if (!header_found) { - h = compare(); //h2 = compare2(); + h = compare(hdr); //h2 = compare2(hdr); if ((h >= HEADLEN)) { header_found = 1; fflush(stdout); if (option_timestamp) printf("<%8.3f> ", sample_count/(double)sample_rate); - strncpy(frame_bits, header, HEADLEN); + strncpy(frame_bits, hdr, HEADLEN); bit_count += HEADLEN; frames++; } From 9021501be12c178e4720bd023adcec4a3e7285ea Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Fri, 31 May 2024 21:13:17 +0930 Subject: [PATCH 36/42] Fix WXR301-PN9 baud rate --- auto_rx/autorx/__init__.py | 2 +- auto_rx/autorx/decode.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index 22d92de3..e2a0fda1 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -12,7 +12,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.7.3-beta12" +__version__ = "1.7.3-beta13" # Global Variables diff --git a/auto_rx/autorx/decode.py b/auto_rx/autorx/decode.py index 639b5b96..1b0bcb6e 100644 --- a/auto_rx/autorx/decode.py +++ b/auto_rx/autorx/decode.py @@ -1342,8 +1342,8 @@ def generate_decoder_command_experimental(self): elif self.sonde_type == "WXRPN9": # Weathex WxR-301D Sonde, PN9 variant - _baud_rate = 4800 - _sample_rate = 96000 + _baud_rate = 5000 + _sample_rate = 100000 # Limit FSK estimator window to roughly +/- 40 kHz _lower = -40000 From 9cdc2cd3b0fed48a323a962b1581e87fc2685010 Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Sun, 2 Jun 2024 09:36:04 +0930 Subject: [PATCH 37/42] Change PN9 variant sonde type name to WxR-301D-5k so it fits on the tracker sonde list. --- auto_rx/autorx/__init__.py | 2 +- auto_rx/autorx/sondehub.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index e2a0fda1..2d1ddcc0 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -12,7 +12,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.7.3-beta13" +__version__ = "1.7.3-beta14" # Global Variables diff --git a/auto_rx/autorx/sondehub.py b/auto_rx/autorx/sondehub.py index e5137dd3..804a9250 100644 --- a/auto_rx/autorx/sondehub.py +++ b/auto_rx/autorx/sondehub.py @@ -236,11 +236,11 @@ def reformat_data(self, telemetry): # Double check for the subtype being present, just in case... if "subtype" in telemetry: if telemetry["subtype"] == "WXR_PN9": - _output["type"] = "WxR-301D (PN9)" + _output["subtype"] = "WxR-301D-5k" elif telemetry["type"] == "WXRPN9": _output["manufacturer"] = "Weathex" - _output["type"] = "WxR-301D (PN9)" + _output["type"] = "WxR-301D-5k" _output["serial"] = telemetry["id"].split("-")[1] else: From 1b7b6399d3beba5756bb12326c206cca46b92a1d Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Tue, 4 Jun 2024 07:56:24 +0930 Subject: [PATCH 38/42] Narrow WXRPN9 mask estimator back to 50 khz --- auto_rx/autorx/__init__.py | 2 +- auto_rx/autorx/decode.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index 2d1ddcc0..348e57d1 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -12,7 +12,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.7.3-beta14" +__version__ = "1.7.3-beta15" # Global Variables diff --git a/auto_rx/autorx/decode.py b/auto_rx/autorx/decode.py index 1b0bcb6e..6f577174 100644 --- a/auto_rx/autorx/decode.py +++ b/auto_rx/autorx/decode.py @@ -1368,7 +1368,7 @@ def generate_decoder_command_experimental(self): demod_cmd += f" tee {self.save_decode_iq_path} |" # Trying out using the mask estimator here to reduce issues with interference - demod_cmd += "./fsk_demod --cs16 -s -b %d -u %d --mask 60000 --stats=%d 2 %d %d - -" % ( + demod_cmd += "./fsk_demod --cs16 -s -b %d -u %d --mask 50000 --stats=%d 2 %d %d - -" % ( _lower, _upper, _stats_rate, From 967741deeb37764c4b570bce939bf8ddefbfba99 Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Thu, 6 Jun 2024 18:04:15 +0930 Subject: [PATCH 39/42] Rebase dft_detect, bump beta version --- auto_rx/autorx/__init__.py | 2 +- scan/dft_detect.c | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index 348e57d1..961d8ae9 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -12,7 +12,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.7.3-beta15" +__version__ = "1.7.3-beta16" # Global Variables diff --git a/scan/dft_detect.c b/scan/dft_detect.c index ea39222d..701196fe 100644 --- a/scan/dft_detect.c +++ b/scan/dft_detect.c @@ -1018,9 +1018,12 @@ static int init_buffers() { float f_lp; // dec_lowpass: lowpass_bw/2 float t_bw; // dec_lowpass: transition_bw int taps; // dec_lowpass: taps + int wideIF = 0; if (set_lpIQ > IF_sr) IF_sr = set_lpIQ; + wideIF = IF_sr > 60e3; + sr_base = sample_rate; if (option_min) IF_sr = IF_SAMPLE_RATE_MIN; @@ -1030,8 +1033,13 @@ static int init_buffers() { decM = sr_base / IF_sr; } - f_lp = (IF_sr+20e3)/(4.0*sr_base); + f_lp = (IF_sr+20e3)/(4.0*sr_base); // IF=48k t_bw = (IF_sr-20e3)/*/2.0*/; + if (wideIF) { // IF=96k + f_lp = (IF_sr+60e3)/(4.0*sr_base); + t_bw = (IF_sr-60e3)/*/2.0*/; + } + else if (option_min) { t_bw = (IF_sr-12e3); } From bc2585cd4fb2aab72f579da422757b82befdd823 Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Sun, 16 Jun 2024 09:27:38 +0930 Subject: [PATCH 40/42] Fix KA9Q support to work with latest commit (2024-06-11) --- auto_rx/autorx/__init__.py | 2 +- auto_rx/autorx/ka9q.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index 961d8ae9..7f96ad97 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -12,7 +12,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.7.3-beta16" +__version__ = "1.7.3-beta17" # Global Variables diff --git a/auto_rx/autorx/ka9q.py b/auto_rx/autorx/ka9q.py index f4ca66f1..bae7fb28 100644 --- a/auto_rx/autorx/ka9q.py +++ b/auto_rx/autorx/ka9q.py @@ -119,9 +119,10 @@ def ka9q_get_iq_cmd( # Get the 'PCM' version of the server name, where as assume -pcm is added to the first part of the hostname. _pcm_host = sdr_hostname.split('.')[0] + "-pcm." + ".".join(sdr_hostname.split(".")[1:]) - # pcmcat -2 -s 404090000 sonde-pcm.local + # Example: pcmcat -s 404090000 sonde-pcm.local + # -2 option was removed sometime in early 2024. _cmd = ( - f"pcmcat -2 " + f"pcmcat " f"-s {int(frequency)} " f"{_pcm_host} |" ) From 6c2a1742041e6e52472a42ef7a7f94da2935b1db Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Sun, 16 Jun 2024 10:21:37 +0930 Subject: [PATCH 41/42] Make the test SSRC used for ka9q-radio testing more unique --- auto_rx/autorx/sdr_wrappers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/auto_rx/autorx/sdr_wrappers.py b/auto_rx/autorx/sdr_wrappers.py index 6883a116..d3cce978 100644 --- a/auto_rx/autorx/sdr_wrappers.py +++ b/auto_rx/autorx/sdr_wrappers.py @@ -75,7 +75,7 @@ def test_sdr( f"tune " f"--samprate 48000 --mode iq " f"--frequency {int(check_freq)} " - f"--ssrc {int(check_freq)} " + f"--ssrc {int(check_freq)}314 " f"--radio {sdr_hostname}" ) @@ -112,7 +112,7 @@ def test_sdr( f"tune " f"--samprate 48000 --mode iq " f"--frequency 0 " - f"--ssrc {int(check_freq)} " + f"--ssrc {int(check_freq)}314 " f"--radio {sdr_hostname}" ) From 224eb6af42439d431de2796dbb92cf35b5639d7f Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Sun, 16 Jun 2024 13:40:58 +0930 Subject: [PATCH 42/42] Prepare for 1.7.3 release --- auto_rx/autorx/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index 7f96ad97..e0fab7fb 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -12,7 +12,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.7.3-beta17" +__version__ = "1.7.3" # Global Variables