From f51658a7f3e877e414abebab030b20eef047b3a3 Mon Sep 17 00:00:00 2001 From: Greg Albrecht Date: Sat, 22 Jun 2024 13:15:57 -0700 Subject: [PATCH] Happy Summer Solstice - Fixes #72: Add a config variable for users to set expected CN when using CA verification. Thanks @ahoenerBE - Added configuration parameter: PYTAK_TLS_SERVER_EXPECTED_HOSTNAME - Rewrote GitHub actions, moved most logic to shell script and Makefile. - Renamed Debian package from python3-pytak to pytak. - Standardized Makefile for all PyTAK based programs. - Cleaned, simplified and expanded documentation. - Created Makefile jobs for Debian packaging and PyTAK customization. - Moved all media to media sub directory under docs/. - Converted README.rst to README.md. - Style & Linting of code. - Refactored TLS client creation, abstracted many functions. - Added TLS client cert and key checks and improved error messages. --- .github/workflows/debian.yml | 40 ++++-------- CHANGELOG.md | 18 ++++++ Makefile | 58 +++++++++++------- README.rst => README.md | 17 +++-- debian/install_pkg_build_deps.sh | 10 +++ debian/pytak.conf | 0 debian/pytak.postinst | 0 debian/pytak.service | 0 docs/index.md | 10 +-- docs/installation.md | 9 +-- .../atak_screenshot_with_pytak_logo-x25.jpg | Bin .../atak_screenshot_with_pytak_logo.jpg | Bin docs/{ => media}/pytak_logo-256x264.png | Bin docs/{ => media}/pytak_logo.png | Bin docs/{ => media}/takproto_chart.png | Bin docs/tak_protocols.md | 2 +- mkdocs.yml | 2 +- setup.cfg | 7 ++- setup.py | 6 +- takproto/__init__.py | 22 ++----- takproto/constants.py | 6 +- takproto/functions.py | 28 ++++++--- 22 files changed, 124 insertions(+), 111 deletions(-) rename README.rst => README.md (65%) create mode 100644 debian/install_pkg_build_deps.sh create mode 100644 debian/pytak.conf create mode 100644 debian/pytak.postinst create mode 100644 debian/pytak.service rename docs/{ => media}/atak_screenshot_with_pytak_logo-x25.jpg (100%) rename docs/{ => media}/atak_screenshot_with_pytak_logo.jpg (100%) rename docs/{ => media}/pytak_logo-256x264.png (100%) rename docs/{ => media}/pytak_logo.png (100%) rename docs/{ => media}/takproto_chart.png (100%) diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 035a5ba..64eb243 100755 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -1,4 +1,4 @@ -name: Build Debian package +name: Build Debian Package on: push: @@ -10,38 +10,24 @@ env: jobs: build-artifact: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@master - - name: Install packaging dependencies - run: | - sudo apt-get update -qq - sudo apt-get install -y \ - python3 python3-dev python3-pip python3-venv python3-all \ - dh-python debhelper devscripts dput software-properties-common \ - python3-distutils python3-setuptools python3-wheel python3-stdeb - - - name: Build Debian/Apt sdist_dsc - run: | - rm -Rf deb_dist/* - python3 setup.py --command-packages=stdeb.command sdist_dsc - - - name: Build Debian/Apt bdist_deb - run: | - export REPO_NAME=$(echo ${{ github.repository }} | awk -F"/" '{print $2}') - python3 setup.py --command-packages=stdeb.command bdist_deb - ls -al deb_dist/ - cp deb_dist/python3-${REPO_NAME}_*_all.deb deb_dist/python3-${REPO_NAME}_latest_all.deb - - - uses: actions/upload-artifact@master + - name: Install Debian Package Building Dependencies + run: sudo bash debian/install_pkg_build_deps.sh + + - name: Create Debian Package + run: make clean package + + - name: Upload Artifacts to GitHub + uses: actions/upload-artifact@master with: name: artifact-deb - path: | - deb_dist/*.deb + path: deb_dist/*.deb - - name: Create Release + - name: Create GitHub Release id: create_release uses: actions/create-release@master env: @@ -52,7 +38,7 @@ jobs: draft: false prerelease: false - - name: Upload Release Asset + - name: Upload Release Asset to GitHub id: upload-release-asset uses: svenstaro/upload-release-action@v2 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index e974870..0637261 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,21 @@ +## TAKProto 3.0.0 + +Happy Summer Solstice! + +- Fixes #15: Time data from iTak not parsed. +- Fixes #12: Add CHANGELOG +- Fixes #19: Support timestamps without microseconds. Thanks @sei-jmattson +- Fixes #17/#18: Support mixed-mode rx. Thanks @sei-jmattson +- Fixes #16: Fix datetime parsing for newer TAK clients that don't include microseconds in the timestamp. Thanks @brian7704 +- Rewrote GitHub actions, moved most logic to shell script and Makefile. +- Renamed Debian package from python3-takproto to takproto. +- Standardized Makefile for all PyTAK based programs. +- Cleaned, simplified and expanded documentation. +- Created Makefile jobs for Debian packaging and TAKProto customization. +- Moved all media to media sub directory under docs/. +- Converted README.rst to README.md. +- Style & Linting of code. + ## TAKProto 2.1.1 - Fixes #12: Add Changelog. diff --git a/Makefile b/Makefile index 72bd672..4205fea 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,11 @@ +# Makefile from https://github.com/snstac/pytak +# PyTAK Makefile # -# Copyright Sensors & Signals LLC https://www.snstac.com +# Copyright Sensors & Signals LLC https://www.snstac.com/ # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -13,15 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Author:: Greg Albrecht -# Copyright:: Copyright Sensors & Signals LLC https://www.snstac.com -# License:: Apache License, Version 2.0 -# -this_app = takproto -.DEFAULT_GOAL := all +REPO_NAME ?= $(shell echo $(wildcard */classes.py) | awk -F'/' '{print $$1}') +SHELL := /bin/bash +.DEFAULT_GOAL := editable +# postinst = $(wildcard debian/*.postinst.sh) +# service = $(wildcard debian/*.service) -all: editable +prepare: + mkdir -p build/ develop: python3 setup.py develop @@ -30,13 +30,13 @@ editable: python3 -m pip install -e . install_test_requirements: - python3 -m pip install -r requirements_test.txt + python3 -m pip install -r requirements_test.txt install: python3 setup.py install uninstall: - python3 -m pip uninstall -y $(this_app) + python3 -m pip uninstall -y $(REPO_NAME) reinstall: uninstall install @@ -47,16 +47,16 @@ clean: @rm -rf *.egg* build dist *.py[oc] */*.py[co] cover doctest_pypi.cfg \ nosetests.xml pylint.log output.xml flake8.log tests.log \ test-result.xml htmlcov fab.log .coverage __pycache__ \ - */__pycache__ .mypy_cache .pytest_cache + */__pycache__ deb_dist .mypy_cache pep8: - flake8 --max-line-length=88 --extend-ignore=E203,E231 --exit-zero $(this_app)/*.py + flake8 --max-line-length=88 --extend-ignore=E203 --exit-zero $(REPO_NAME)/*.py flake8: pep8 lint: pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \ - --max-line-length=88 -r n $(this_app)/*.py || exit 0 + --max-line-length=88 -r n $(REPO_NAME)/*.py || exit 0 pylint: lint @@ -72,7 +72,7 @@ pytest: test: editable install_test_requirements pytest test_cov: - pytest --cov=$(this_app) --cov-report term-missing + pytest --cov=$(REPO_NAME) --cov-report term-missing black: black . @@ -81,7 +81,23 @@ mkdocs: pip install -r docs/requirements.txt mkdocs serve -proto: - for p in src-protobuf/*.proto; do \ - protoc -v -I=src-protobuf --python_out=takprotobuf/proto $$p; \ - done +deb_dist: + python3 setup.py --command-packages=stdeb.command sdist_dsc + +deb_custom: + cp debian/$(REPO_NAME).conf $(wildcard deb_dist/*/debian)/$(REPO_NAME).default + cp debian/$(REPO_NAME).postinst $(wildcard deb_dist/*/debian)/$(REPO_NAME).postinst + cp debian/$(REPO_NAME).service $(wildcard deb_dist/*/debian)/$(REPO_NAME).service + +bdist_deb: deb_dist deb_custom + cd deb_dist/$(REPO_NAME)-*/ && dpkg-buildpackage -rfakeroot -uc -us + +faux_latest: + cp deb_dist/$(REPO_NAME)_*-1_all.deb deb_dist/$(REPO_NAME)_latest_all.deb + cp deb_dist/$(REPO_NAME)_*-1_all.deb deb_dist/python3-$(REPO_NAME)_latest_all.deb + +package: bdist_deb faux_latest + +extract: + dpkg-deb -e $(wildcard deb_dist/*latest_all.deb) deb_dist/extract + dpkg-deb -x $(wildcard deb_dist/*latest_all.deb) deb_dist/extract diff --git a/README.rst b/README.md similarity index 65% rename from README.rst rename to README.md index 0a1af55..53d4b6d 100644 --- a/README.rst +++ b/README.md @@ -1,16 +1,15 @@ -.. image:: https://takproto.readthedocs.io/en/latest/atak_screenshot_with_pytak_logo-x25.jpg - :alt: ATAK Screenshot with PyTAK Logo. - :target: https://github.com/snstac/takproto/blob/main/docs/atak_screenshot_with_pytak_logo.jpg +![ATAK Screenshot with PyTAK Logo.](https://takproto.readthedocs.io/en/latest/media/atak_screenshot_with_pytak_logo-x25.jpg) -TAKProto: TAK Protocol Python Module -************************************ +# Send & Receive data from TAK with Python -TAKProto is a Python module for encoding & decoding TAK Protocol Payloads, for use with `TAK Products `_ including ATAK, WinTAK, iTAK, TAKX, TAK Tracker & TAK Server. TAKProto includes functions for converting TAK Protocol Protobuf messages to Python objects, and serializing CoT XML messages as Protobuf. +TAKProto is a Python module for encoding & decoding TAK Protocol Payloads, for use +with [TAK Products](https://www.tak.gov/) including ATAK, WinTAK, iTAK, TAKX, TAK +Tracker & TAK Server. TAKProto includes functions for converting TAK Protocol +Protobuf messages to Python objects, and serializing CoT XML messages as Protobuf. -`Documentation is available here. `_ +[Documentation is available here.](https://takproto.rtfd.io/) -License -======= +## Copyright & License Copyright Sensors & Signals LLC https://www.snstac.com Copyright 2020 Delta Bravo-15 diff --git a/debian/install_pkg_build_deps.sh b/debian/install_pkg_build_deps.sh new file mode 100644 index 0000000..6713a78 --- /dev/null +++ b/debian/install_pkg_build_deps.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +echo "Installing Debian package build dependencies" + +apt-get update -qq + +apt-get install -y \ + python3 python3-dev python3-pip python3-venv python3-all \ + dh-python debhelper devscripts dput software-properties-common \ + python3-distutils python3-setuptools python3-wheel python3-stdeb diff --git a/debian/pytak.conf b/debian/pytak.conf new file mode 100644 index 0000000..e69de29 diff --git a/debian/pytak.postinst b/debian/pytak.postinst new file mode 100644 index 0000000..e69de29 diff --git a/debian/pytak.service b/debian/pytak.service new file mode 100644 index 0000000..e69de29 diff --git a/docs/index.md b/docs/index.md index c77849a..10882f6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,9 +1 @@ - -![ATAK Screenshot with PyTAK Logo.](atak_screenshot_with_pytak_logo-x25.jpg) - -# TAKProto - Send & Receive Team Awareness Kit (TAK) packets with Python. - -TAKProto is a Python module for encoding & decoding TAK Protocol Payloads, for use with [TAK Products](https://tak.gov) including ATAK, WinTAK, iTAK, TAKX, TAK Tracker & TAK Server. TAKProto includes functions for converting TAK Protocol Protobuf messages to Python objects, and serializing CoT XML messages as Protobuf. - -[Documentation is available here.](https://takproto.rtfd.io) - +{!README.md!} \ No newline at end of file diff --git a/docs/installation.md b/docs/installation.md index b67cb2c..75ec1fe 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,13 +1,14 @@ ## Install on Debian, Ubuntu or Raspberry Pi -TAKProto is distributed as a Debian package (``.deb``). takproto should be compatible with most contemporary system-Python versions from Python 3.6 onward. +TAKProto is distributed as a Debian package (``.deb``). takproto should be compatible +with most contemporary system-Python versions from Python 3.6 onward. To install takproto, download the takproto package and install using apt: ```sh linenums="1" -sudo apt update -y -wget https://github.com/snstac/takproto/releases/latest/download/python3-takproto_latest_all.deb -sudo apt install -f ./python3-takproto_latest_all.deb +sudo apt update -qq +wget https://github.com/snstac/takproto/releases/latest/download/takproto_latest_all.deb +sudo apt install -f ./takproto_latest_all.deb ``` ## Install from Python Package Index (PyPI) diff --git a/docs/atak_screenshot_with_pytak_logo-x25.jpg b/docs/media/atak_screenshot_with_pytak_logo-x25.jpg similarity index 100% rename from docs/atak_screenshot_with_pytak_logo-x25.jpg rename to docs/media/atak_screenshot_with_pytak_logo-x25.jpg diff --git a/docs/atak_screenshot_with_pytak_logo.jpg b/docs/media/atak_screenshot_with_pytak_logo.jpg similarity index 100% rename from docs/atak_screenshot_with_pytak_logo.jpg rename to docs/media/atak_screenshot_with_pytak_logo.jpg diff --git a/docs/pytak_logo-256x264.png b/docs/media/pytak_logo-256x264.png similarity index 100% rename from docs/pytak_logo-256x264.png rename to docs/media/pytak_logo-256x264.png diff --git a/docs/pytak_logo.png b/docs/media/pytak_logo.png similarity index 100% rename from docs/pytak_logo.png rename to docs/media/pytak_logo.png diff --git a/docs/takproto_chart.png b/docs/media/takproto_chart.png similarity index 100% rename from docs/takproto_chart.png rename to docs/media/takproto_chart.png diff --git a/docs/tak_protocols.md b/docs/tak_protocols.md index a529bb5..0c8d4d7 100644 --- a/docs/tak_protocols.md +++ b/docs/tak_protocols.md @@ -15,4 +15,4 @@ TAK Products configured for connecting to a TAK Server will send TAK Protocol Ve The `takproto` module supports encoding and decoding all 3 formats of CoT messages. -![TAK Protocol Chart](takproto_chart.png) +![TAK Protocol Chart](media/takproto_chart.png) diff --git a/mkdocs.yml b/mkdocs.yml index 5dd7cda..62ca354 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,4 +1,4 @@ -site_name: TAKProto - Send & Receive Team Awareness Kit (TAK) packets with Python. +site_name: A Python package for the TAK Protocol. site_url: https://takproto.rtfd.io/ repo_url: https://github.com/snstac/takproto/ site_description: A Python module to encode & decode Team Awareness Kit (TAK) Protocol-based Cursor on Target (CoT) messages. diff --git a/setup.cfg b/setup.cfg index fa90f54..ec5b88b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,5 @@ # Setup for the Python TAK Protocol Packet - Version 1 Module. +# setup.cfg from https://github.com/snstac/takproto # # Copyright Sensors & Signals LLC https://www.snstac.com # Copyright 2020 Delta Bravo-15 @@ -31,9 +32,9 @@ project_urls = CI: GitHub Actions = https://github.com/snstac/takproto/actions GitHub: issues = https://github.com/snstac/takproto/issues GitHub: repo = https://github.com/snstac/takproto -description = A Python module to encode & decode 'TAK Protocol Payload - Version 1' Protocol Buffer based Cursor on Target (CoT) messages. -long_description = file: README.rst -long_description_content_type = text/x-rst +description = TAKProto is a Python package for the TAK protocol. +long_description = file: README.md +long_description_content_type = text/markdown maintainer = Greg Albrecht maintainer_email = oss@undef.net license = MIT License diff --git a/setup.py b/setup.py index 9fbd2a7..717250e 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +# setup.py from https://github.com/snstac/takproto # # Copyright Sensors & Signals LLC https://www.snstac.com # Copyright 2020 Delta Bravo-15 @@ -24,10 +25,7 @@ # -"""Setup for the Python TAK Protocol Packet - Version 1 Module. - -:source: -""" +"""Setup for the Python TAK Protocol Package.""" from setuptools import setup diff --git a/takproto/__init__.py b/takproto/__init__.py index fd09b8b..3fcbbd9 100644 --- a/takproto/__init__.py +++ b/takproto/__init__.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +# __init__.py from https://github.com/snstac/takproto # # Copyright Sensors & Signals LLC https://www.snstac.com # @@ -22,17 +23,11 @@ # SOFTWARE. # -"""TAKProto: Encode & Decode TAK Protocol Payloads using Python. +"""TAKProto: Encode & Decode TAK Protocol Payloads using Python.""" -:author: Greg Albrecht -:copyright: Copyright Sensors & Signals LLC https://www.snstac.com -:license: MIT License -:source: -""" +__version__ = "3.0.0-beta1" -__version__ = "2.1.1" - -# Python 3.6 test/build work-around: +# COMPAT Python 3.6 import work-around. try: from .functions import ( # NOQA xml2proto, @@ -44,12 +39,5 @@ from .constants import TAKProtoVer # NOQA except ImportError as exc: import warnings - warnings.warn( - "Unable to import required modules, IGNORING for Python 3.6 compat. Original Exception: " - ) - warnings.warn(str(exc)) -__author__ = "Greg Albrecht " -__copyright__ = "Copyright Sensors & Signals LLC https://www.snstac.com" -__license__ = "MIT License" -__source__ = "https://github.com/snstac/takproto" + warnings.warn(f"COMPAT Python 3.6. Ignoring Exception: {str(exc)}") diff --git a/takproto/constants.py b/takproto/constants.py index ad75774..0339ca0 100644 --- a/takproto/constants.py +++ b/takproto/constants.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +# constants.py from https://github.com/snstac/takproto # # Copyright Sensors & Signals LLC https://www.snstac.com # @@ -26,11 +27,6 @@ from enum import Enum -__author__ = "Greg Albrecht " -__copyright__ = "Copyright Sensors & Signals LLC https://www.snstac.com" -__license__ = "Apache License, Version 2.0" - - DEFAULT_PROTO_HEADER = bytearray(b"\xbf") DEFAULT_MESH_HEADER = bytearray(b"\xbf\x01\xbf") DEFAULT_XML_HEADER = bytearray(b" @@ -22,6 +23,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # + """TAKProto Functions for manipulating TAK Protocol Version 1 messages.""" import re @@ -29,7 +31,7 @@ from datetime import datetime from io import BytesIO -from typing import Optional +from typing import Optional, Union import delimited_protobuf as dpb @@ -44,15 +46,15 @@ from takproto.proto import TakMessage -def parse_proto(msg: bytearray) -> Optional[bytearray]: +def parse_proto(msg: Union[bytearray, str]) -> Optional[bytearray]: """Parse TAK Protocol Version 1 Mesh & Stream message.""" parsed = None if msg[:3] == DEFAULT_MESH_HEADER: parsed = parse_mesh(msg) - elif msg[0] in DEFAULT_PROTO_HEADER: + elif msg[0] == DEFAULT_PROTO_HEADER: parsed = parse_stream(msg) - elif msg[:5] == DEFAULT_XML_HEADER: + elif msg[:5] == DEFAULT_XML_HEADER and isinstance(msg, str): parsed = xml2message(msg) return parsed @@ -81,7 +83,11 @@ def format_time(time: str) -> int: return int(s_time.timestamp() * 1000) -def xml2message(xml: str) -> TakMessage: # NOQA pylint: disable=too-many-locals,too-many-branches,too-many-statements +def xml2message( + xml: str, +) -> ( + TakMessage +): # NOQA pylint: disable=too-many-locals,too-many-branches,too-many-statements """Convert plain XML CoT to Protobuf.""" event = ET.fromstring(xml) tak_message = TakMessage() @@ -134,8 +140,10 @@ def xml2message(xml: str) -> TakMessage: # NOQA pylint: disable=too-many-locals if uid and "GeoChat." in uid: pattern = "(.*?)" target = ET.tostring(detail).decode("utf-8") - xmldetailstr = re.search(pattern, target).group(1) - new_detail.xmlDetail = xmldetailstr + re_search = re.search(pattern, target) + if re_search: + xmldetailstr = re_search.group(1) + new_detail.xmlDetail = xmldetailstr else: # Add unknown elements to xmlDetail field. known_elem = [ @@ -201,9 +209,9 @@ def xml2message(xml: str) -> TakMessage: # NOQA pylint: disable=too-many-locals return tak_message -def xml2proto( - xml: str, protover: Optional[TAKProtoVer] = None -): + +def xml2proto(xml: str, protover: Optional[TAKProtoVer] = None): + """Convert TAK XML COT to TAK Protobuf COT.""" tak_message = xml2message(xml) output = msg2proto(tak_message, protover) return output