Use venv for python

CHANGELOG.md

@@ -1,16 +1,17 @@
 # Changelog
-## [2.4.3] - not released
+## [2.5] - not released
 ### Added
  - Added UART connected F9P detection and configuration.
  - Some scripts for using a base with a 4G Simcom A76XX modem. (Beta)
 ### Changed
- - --detect-usb-gnss renamed to --detect-gnss
+ - RTKBase now use a virtual environnement for the python environnement.
+ - install.sh -> --detect-usb-gnss renamed to --detect-gnss
 ### Deprecated
 ### Removed
 ### Fixed
  - More tests before copying RTKLib binaries. #313
- - Pin Werkzeug version to fix incompatibility with Flask. #330
 ### Security
+ - Update of various python modules.
 
 ## [2.4.2] - 2023-11-10
 ### Fixed

README.md

@@ -326,6 +326,8 @@ A gnss receiver with a timepulse output is a very accurate [stratum 0](https://e
       ^- kalimantan.ordimatic.net      3   6   177    16    -27ms[  -27ms] +/-   64ms
 
    ```
+## Requirements:
+Python >= 3.7
 
 ## History:
 See the [changelog](./CHANGELOG.md)

tools/copy_unit.sh

@@ -6,6 +6,9 @@
 man_help() {
  echo 'Options:'
  echo '        -h | --help'
+ echo '        -p | --python_path'
+ echo '                        path to the web app python venv binary'
+ echo '                        (usually /home/your_username/rtkbase/venv/bin/python)'
  echo '        -u | --user  <username>'
  echo '                        Specify user used in the service unit. Without this argument'
  echo '                        the user return by the logname command will be used.'
@@ -14,8 +17,9 @@ man_help() {
 
 BASEDIR=$(dirname "$0")
 ARG_HELP=0
+ARG_PYPATH=0
 ARG_USER=0
-PARSED_ARGUMENTS=$(getopt --name copy_unit --options hu: --longoptions help,user: -- "$@")
+PARSED_ARGUMENTS=$(getopt --name copy_unit --options hp:u: --longoptions help,python_path:,user: -- "$@")
 VALID_ARGUMENTS=$?
 if [ "$VALID_ARGUMENTS" != "0" ]; then
     #man_help
@@ -27,8 +31,9 @@ if [ "$VALID_ARGUMENTS" != "0" ]; then
   while :
     do
       case "$1" in
-        -h | --help)   ARG_HELP=1                     ; shift   ;;
-        -u | --user)   ARG_USER="${2}"                ; shift 2 ;;
+        -h | --help)        ARG_HELP=1                     ; shift   ;;
+        -p | --python_path) ARG_PYPATH="${2}"              ; shift 2 ;;
+        -u | --user)        ARG_USER="${2}"                ; shift 2 ;;
         # -- means the end of the arguments; drop this, and break out of the while loop
         --) shift; break ;;
         # If invalid options were passed, then getopt should have reported an error,
@@ -38,6 +43,7 @@ if [ "$VALID_ARGUMENTS" != "0" ]; then
       esac
     done
 [ $ARG_HELP -eq 1 ] && man_help
+[ "${ARG_PYPATH}" == 0 ] && echo 'Please enter the python venv path with the -p argument' && exit
 [ "${ARG_USER}" == 0 ] && ARG_USER=$(logname)
 #echo 'user=' "${ARG_USER}"
 
@@ -50,7 +56,7 @@ for file_path in "${BASEDIR}"/../unit/*.service "${BASEDIR}"/../unit/*.timer
     do
         file_name=$(basename "${file_path}")
         echo copying "${file_name}"
-        sed -e 's|{script_path}|'"$(dirname "$(dirname "$(readlink -f "$0")")")"'|' -e 's|{user}|'"${ARG_USER}"'|' -e 's|{python_path}|'"$(which python3)"'|' "${file_path}" > /etc/systemd/system/"${file_name}"
+        sed -e 's|{script_path}|'"$(dirname "$(dirname "$(readlink -f "$0")")")"'|' -e 's|{user}|'"${ARG_USER}"'|' -e 's|{python_path}|'"${ARG_PYPATH}"'|' "${file_path}" > /etc/systemd/system/"${file_name}"
     done
 
 systemctl daemon-reload

tools/install.sh

@@ -104,7 +104,7 @@ install_dependencies() {
     echo 'INSTALLING DEPENDENCIES'
     echo '################################'
       apt-get "${APT_TIMEOUT}" update || exit 1
-      apt-get "${APT_TIMEOUT}" install -y git build-essential pps-tools python3-pip python3-dev python3-setuptools python3-wheel libsystemd-dev bc dos2unix socat zip unzip pkg-config psmisc || exit 1
+      apt-get "${APT_TIMEOUT}" install -y git build-essential pps-tools python3-pip python3-venv python3-dev python3-setuptools python3-wheel libsystemd-dev bc dos2unix socat zip unzip pkg-config psmisc || exit 1
       #apt-get "${APT_TIMEOUT}" upgrade -y
 }
 
@@ -333,8 +333,9 @@ rtkbase_requirements(){
     echo '################################'
     echo 'INSTALLING RTKBASE REQUIREMENTS'
     echo '################################'
-      #as we need to run the web server as root, we need to install the requirements with
-      #the same user
+      # create virtuel environnement for rtkbase
+      sudo -u "${RTKBASE_USER}" python3 -m venv "${rtkbase_path}"/venv
+      python_venv="${rtkbase_path}"/venv/bin/python
       platform=$(uname -m)
       if [[ $platform =~ 'aarch64' ]] || [[ $platform =~ 'x86_64' ]]
         then
@@ -343,19 +344,20 @@ rtkbase_requirements(){
       fi
       # Copying udev rules
       [[ ! -d /etc/udev/rules.d ]] && mkdir /etc/udev/rules.d/
-      cp "${rtkbase_path}"/tools/90-usb-simcom-at.rules /etc/rules.d/
+      cp "${rtkbase_path}"/tools/*.rules /etc/udev/rules.d/
       udevadm control --reload && udevadm trigger
 
-      python3 -m pip install --upgrade pip setuptools wheel  --extra-index-url https://www.piwheels.org/simple
+      sudo -u "${RTKBASE_USER}" "${python_venv}" -m pip install --upgrade pip setuptools wheel  --extra-index-url https://www.piwheels.org/simple
       # install prebuilt wheel for cryptography because it is unavailable on piwheels (2023/01)
-      if [[ $platform == 'armv7l' ]] && [[ $(python3 --version) =~ '3.7' ]]
-        then 
-          python3 -m pip install "${rtkbase_path}"/tools/wheel/cryptography-38.0.0-cp37-cp37m-linux_armv7l.whl
-      elif [[ $platform == 'armv6l' ]] && [[ $(python3 --version) =~ '3.7' ]]
-        then
-          python3 -m pip install "${rtkbase_path}"/tools/wheel/cryptography-38.0.0-cp37-cp37m-linux_armv6l.whl
-      fi
-      python3 -m pip install -r "${rtkbase_path}"/web_app/requirements.txt  --extra-index-url https://www.piwheels.org/simple
+      # not needed anymore (2023/11)
+      #if [[ $platform == 'armv7l' ]] && [[ $("${python_venv}" --version) =~ '3.7' ]]
+      #  then 
+      #    sudo -u "${RTKBASE_USER}" "${python_venv}" -m pip install "${rtkbase_path}"/tools/wheel/cryptography-38.0.0-cp37-cp37m-linux_armv7l.whl
+      #elif [[ $platform == 'armv6l' ]] && [[ $("${python_venv}" --version) =~ '3.7' ]]
+      #  then
+      #    sudo -u "${RTKBASE_USER}" "${python_venv}" -m pip install "${rtkbase_path}"/tools/wheel/cryptography-38.0.0-cp37-cp37m-linux_armv6l.whl
+      #fi
+      sudo -u "${RTKBASE_USER}" "${python_venv}" -m pip install -r "${rtkbase_path}"/web_app/requirements.txt  --extra-index-url https://www.piwheels.org/simple
       #when we will be able to launch the web server without root, we will use
       #sudo -u $(logname) python3 -m pip install -r requirements.txt --user.
 }
@@ -367,7 +369,7 @@ install_unit_files() {
       if [ -d "${rtkbase_path}" ]
       then 
         #Install unit files
-        "${rtkbase_path}"/tools/copy_unit.sh --user "${RTKBASE_USER}"
+        "${rtkbase_path}"/tools/copy_unit.sh --python_path "${rtkbase_path}"/venv/bin/python --user "${RTKBASE_USER}"
         systemctl enable rtkbase_web.service
         systemctl enable rtkbase_archive.timer
         systemctl daemon-reload

web_app/requirements.txt

@@ -1,23 +1,47 @@
-cryptography==38.0.0
-itsdangerous==2.1.2
-Werkzeug==2.2.2
-Flask==2.2.2
-Flask-SocketIO==5.3.2
+bidict==0.22.1
+blinker==1.6.3
+Bootstrap-Flask==2.3.2
+certifi==2023.7.22
+cffi==1.16.0;python_version>="3.8"
+cffi==1.15.1;python_version<"3.8"
+charset-normalizer==3.3.2
+click==8.1.7
+cryptography==41.0.5
+distro==1.8.0
+dnspython==2.4.2;python_version>="3.8"
+dnspython==2.3.0;python_version<"3.8"
 eventlet==0.33.3
-greenlet==1.1.3
-Bootstrap-Flask==2.2.0
-Flask-WTF==1.1.1
-Flask-Login==0.6.2
-#On some linux distribution, pexpect and psutil are already installed
-#and can't be upgraded with pip. In these cases, pip fail, leaving
-#some package uninstalled.
-#With pexpect <=4.8.0, pip will not fail.
-#With psutil <= 5.8.0, pip will not fail.
-pexpect<=4.8.0
-psutil<=5.9.4
-pyOpenSSL==23.0.0
+Flask==3.0.0;python_version>="3.8"
+Flask==2.2.5;python_version<"3.8"
+Flask-Login==0.6.3
+Flask-SocketIO==5.3.6
+Flask-WTF==1.2.1;python_version>="3.8"
+Flask-WTF==1.1.1;python_version<"3.8"
+greenlet==3.0.1
+h11==0.14.0
+idna==3.4
+importlib-metadata==6.7.0;python_version<"3.8"
+itsdangerous==2.1.2
+Jinja2==3.1.2
+lxml==4.9.3
+MarkupSafe==2.1.3
+pexpect==4.8.0
+psutil==5.9.6
+ptyprocess==0.7.0
+pycparser==2.21
+pyOpenSSL==23.3.0
 pyserial==3.5
-pystemd==0.10.0
-requests==2.28.1
-dnspython==2.3.0
-distro==1.8.0
+pystemd==0.13.2
+python-engineio==4.8.0
+python-socketio==5.10.0
+requests==2.31.0
+simple-websocket==1.0.0
+six==1.16.0
+typing_extensions==4.7.1;python_version<"3.8"
+urllib3==2.0.7
+Werkzeug==3.0.1;python_version>="3.8"
+Werkzeug==2.2.3;python_version<"3.8"
+wsproto==1.2.0
+WTForms==3.1.0;python_version>="3.8"
+WTForms==3.0.1;python_version<"3.8"
+zipp==3.15.0;python_version<"3.8"

web_app/server.py

@@ -65,6 +65,7 @@
 from flask_login import LoginManager, login_user, logout_user, login_required, current_user, UserMixin
 from wtforms.validators import ValidationError, DataRequired, EqualTo
 from flask_socketio import SocketIO, emit, disconnect
+import urllib
 import subprocess
 import psutil
 import distro