diff --git a/Makefile b/Makefile index 6701bcb9..0b3bb613 100644 --- a/Makefile +++ b/Makefile @@ -236,6 +236,21 @@ start-testing: compose-files aux_images ro_crates images reset_compose permissio exec -T lmtests /bin/bash -c "tests/wait-for-seek.sh 600"; \ printf "$(done)\n" +start-maintenance: compose-files aux_images ro_crates images reset_compose permissions ## Start LifeMonitor in a Testing environment + @printf "\n$(bold)Starting testing services...$(reset)\n" ; \ + base=$$(if [[ -f "docker-compose.yml" ]]; then echo "-f docker-compose.yml"; fi) ; \ + echo "$$(USER_UID=$$(id -u) USER_GID=$$(id -g) \ + $(docker_compose) $${base} \ + -f docker-compose.extra.yml \ + -f docker-compose.base.yml \ + -f docker-compose.monitoring.yml \ + -f docker-compose.dev.yml \ + -f docker-compose.maintenance.yml \ + config)" > docker-compose.yml \ + && cp {,.maintenance.}docker-compose.yml \ + && $(docker_compose) -f docker-compose.yml up -d db redis lm ws_server nginx console ;\ + printf "$(done)\n" + start-nginx: certs docker-compose.base.yml permissions ## Start a nginx front-end proxy for the LifeMonitor back-end @printf "\n$(bold)Starting nginx proxy...$(reset)\n" ; \ base=$$(if [[ -f "docker-compose.yml" ]]; then echo "-f docker-compose.yml"; fi) ; \ diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml index 0d6aae3a..2e17806e 100644 --- a/docker-compose.monitoring.yml +++ b/docker-compose.monitoring.yml @@ -2,7 +2,7 @@ version: "3.5" services: prometheus: - image: prom/prometheus:v2.24.1 + image: prom/prometheus:v2.48.0 ports: - "9090:9090" volumes: diff --git a/docker/worker_entrypoint.sh b/docker/worker_entrypoint.sh index cdc5544d..d82d65a4 100755 --- a/docker/worker_entrypoint.sh +++ b/docker/worker_entrypoint.sh @@ -84,7 +84,6 @@ while : ; do ${threads:-} \ lifemonitor.tasks.worker:broker lifemonitor.tasks ${queues} exit_code=$? - exit_code=$? if [[ $exit_code == 3 ]]; then log "dramatiq worker could not connect to message broker (exit code ${exit_code})" log "Restarting..." diff --git a/k8s/backup-key.secret.yaml b/k8s/backup-key.secret.yaml new file mode 100644 index 00000000..93276859 --- /dev/null +++ b/k8s/backup-key.secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: lifemonitor-api-backup-key +type: Opaque +data: + encryptionKey: \ No newline at end of file diff --git a/k8s/templates/_helpers.tpl b/k8s/templates/_helpers.tpl index 620afc27..cd046970 100644 --- a/k8s/templates/_helpers.tpl +++ b/k8s/templates/_helpers.tpl @@ -67,6 +67,13 @@ Define lifemonitor TLS secret name {{- printf "%s-tls" .Release.Name }} {{- end }} +{{/* +Define lifemonitor secret name for backup key +*/}} +{{- define "chart.lifemonitor.backup.key" -}} +{{- printf "%s-backup-key" .Release.Name }} +{{- end }} + {{/* Define volume name of LifeMonitor backup data diff --git a/k8s/templates/backend.deployment.yaml b/k8s/templates/backend.deployment.yaml index aaedf7f6..baea295c 100644 --- a/k8s/templates/backend.deployment.yaml +++ b/k8s/templates/backend.deployment.yaml @@ -63,7 +63,11 @@ spec: image: {{ include "chart.lifemonitor.image" . }} imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} env: - {{- include "lifemonitor.common-env" . | nindent 12 }} + {{- include "lifemonitor.common-env" . | nindent 12 }} + {{- if .Values.maintenanceMode.enabled }} + - name: FLASK_ENV + value: "maintenance" + {{- end }} volumeMounts: {{- include "lifemonitor.common-volume-mounts" . | nindent 12 }} ports: diff --git a/k8s/templates/backup.job.yaml b/k8s/templates/backup.job.yaml index 109e8d2d..f320994e 100644 --- a/k8s/templates/backup.job.yaml +++ b/k8s/templates/backup.job.yaml @@ -29,12 +29,22 @@ spec: {{- include "lifemonitor.common-volume-mounts" . | nindent 12 }} - name: lifemonitor-backup mountPath: "/var/data/backup" + {{- if .Values.backup.encryptionKeySecret }} + - name: lifemonitor-backup-encryption-key + mountPath: "/lm/backup/encryption.key" + subPath: encryptionKey + {{- end }} restartPolicy: OnFailure volumes: {{- include "lifemonitor.common-volume" . | nindent 10 }} - name: lifemonitor-backup persistentVolumeClaim: claimName: {{ .Values.backup.existingClaim }} + {{- if .Values.backup.encryptionKeySecret }} + - name: lifemonitor-backup-encryption-key + secret: + secretName: {{ .Values.backup.encryptionKeySecret }} + {{- end }} {{- with .Values.lifemonitor.nodeSelector }} nodeSelector: {{- toYaml . | nindent 10 }} diff --git a/k8s/templates/console.deployment.yaml b/k8s/templates/console.deployment.yaml new file mode 100644 index 00000000..4ead8a13 --- /dev/null +++ b/k8s/templates/console.deployment.yaml @@ -0,0 +1,73 @@ +{{- if or (.Values.maintenanceMode.enabled) (.Values.console.enabled ) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "chart.fullname" . }}-console + labels: + {{- include "chart.labels" . | nindent 4 }} +spec: + {{- if not .Values.lifemonitor.autoscaling.enabled }} + replicas: {{ .Values.lifemonitor.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "chart.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/settings: {{ include (print $.Template.BasePath "/settings.secret.yaml") . | sha256sum }} + {{- with .Values.lifemonitor.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "chart.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.lifemonitor.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "chart.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.lifemonitor.podSecurityContext | nindent 8 }} + containers: + - name: app + securityContext: + {{- toYaml .Values.lifemonitor.securityContext | nindent 12 }} + image: {{ include "chart.lifemonitor.image" . }} + imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} + command: ["/bin/sh","-c"] + args: ["wait-for-redis.sh && wait-for-postgres.sh && sleep infinity"] + env: + {{- include "lifemonitor.common-env" . | nindent 12 }} + - name: LIFEMONITOR_MAINTENANCE_MODE + value: {{ .Values.maintenanceMode.enabled | quote }} + - name: LIFEMONITOR_CONSOLE_ENABLED + value: {{ .Values.console.enabled | quote }} + volumeMounts: + {{- include "lifemonitor.common-volume-mounts" . | nindent 12 }} + - name: lifemonitor-backup + mountPath: "/var/data/backup" + ports: + - name: http + containerPort: 8000 + protocol: TCP + resources: + {{- toYaml .Values.lifemonitor.resources | nindent 12 }} + volumes: + - name: lifemonitor-backup + persistentVolumeClaim: + claimName: {{ .Values.backup.existingClaim }} + {{- include "lifemonitor.common-volume" . | nindent 8 }} + {{- with .Values.lifemonitor.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.lifemonitor.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.lifemonitor.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/k8s/templates/settings.secret.yaml b/k8s/templates/settings.secret.yaml index 8c19c8c0..7a0e00d9 100644 --- a/k8s/templates/settings.secret.yaml +++ b/k8s/templates/settings.secret.yaml @@ -78,6 +78,9 @@ stringData: {{- if .Values.backup.retain_days }} BACKUP_RETAIN_DAYS={{ .Values.backup.retain_days }} {{- end }} + {{- if .Values.backup.encryptionKeySecret }} + BACKUP_ENCRYPTION_KEY_PATH=/lm/backup/encryption.key + {{- end }} {{- if .Values.backup.remote.enabled }} BACKUP_REMOTE_PATH={{ .Values.backup.remote.path }} BACKUP_REMOTE_HOST={{ .Values.backup.remote.host }} @@ -87,6 +90,12 @@ stringData: {{- end }} {{- end }} + # Maintenance Mode Settings + {{- if .Values.maintenanceMode.enabled -}} + MAINTENANCE_MODE={{.Values.maintenanceMode.enabled}} + MAINTENANCE_MODE_MAIN_MESSAGE={{.Values.maintenanceMode.mainMessage}} + MAINTENANCE_MODE_SECONDARY_MESSAGE={{.Values.maintenanceMode.secondaryMessage}} + {{- end }} # Set admin credentials LIFEMONITOR_ADMIN_PASSWORD={{ .Values.lifemonitor.administrator.password }} diff --git a/k8s/templates/worker.deployment.yaml b/k8s/templates/worker.deployment.yaml index 7d06c3ca..e622bde2 100644 --- a/k8s/templates/worker.deployment.yaml +++ b/k8s/templates/worker.deployment.yaml @@ -72,6 +72,10 @@ spec: {{ else }} value: {{ $queue.name }} {{ end }} + {{- if $.Values.maintenanceMode.enabled }} + - name: FLASK_ENV + value: "maintenance" + {{- end }} ports: - containerPort: 9191 volumeMounts: diff --git a/k8s/templates/wss.deployment.yaml b/k8s/templates/wss.deployment.yaml index a127f74e..8bae0a96 100644 --- a/k8s/templates/wss.deployment.yaml +++ b/k8s/templates/wss.deployment.yaml @@ -60,6 +60,10 @@ spec: imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} env: {{- include "lifemonitor.common-env" . | nindent 12 }} + {{- if .Values.maintenanceMode.enabled }} + - name: FLASK_ENV + value: "maintenance" + {{- end }} volumeMounts: {{- include "lifemonitor.common-volume-mounts" . | nindent 12 }} ports: diff --git a/k8s/values.yaml b/k8s/values.yaml index 6dbc2303..b9052247 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -6,6 +6,12 @@ nameOverride: "" fullnameOverride: "" +# manage maintenance mode +maintenanceMode: + enabled: false + # mainMessage: "We're busy updating the Life-Monitor service for you." + # secondaryMessage: "We'll be back shortly." + # The name and port number of the server (e.g.: 'lm.local:8000'), # used as base_url on all the links returned by the API externalServerName: &hostname api.lifemonitor.eu @@ -120,6 +126,7 @@ backup: successfulJobsHistoryLimit: 30 failedJobsHistoryLimit: 30 existingClaim: data-api-backup + # encryptionKeySecret: lifemonitor-api-backup-key # Settings to mirror the (cluster) local backup # to a remote site via FTPS or SFTP remote: @@ -150,6 +157,10 @@ monitoring: memory: 256Mi cpu: 0.2 +# management console settings +console: + enabled: false + rateLimiting: zone: accounts: diff --git a/lifemonitor/app.py b/lifemonitor/app.py index 2074dce5..6917ce07 100644 --- a/lifemonitor/app.py +++ b/lifemonitor/app.py @@ -27,14 +27,13 @@ from flask_migrate import Migrate import lifemonitor.config as config -from lifemonitor import redis from lifemonitor import errors as errors_controller +from lifemonitor import redis from lifemonitor.auth.services import current_user from lifemonitor.integrations import init_integrations from lifemonitor.metrics import init_metrics from lifemonitor.routes import register_routes from lifemonitor.tasks import init_task_queues -from lifemonitor.utils import get_domain from . import commands from .cache import init_cache @@ -47,8 +46,15 @@ logger = logging.getLogger(__name__) -def create_app(env=None, settings=None, init_app=True, init_integrations=True, - worker=False, load_jobs=True, **kwargs): +def create_app( + env=None, + settings=None, + init_app=True, + init_integrations=True, + worker=False, + load_jobs=True, + **kwargs, +): """ App factory method :param env: @@ -58,10 +64,10 @@ def create_app(env=None, settings=None, init_app=True, init_integrations=True, """ # set app env app_env = env or os.environ.get("FLASK_ENV", "production") - if app_env != 'production': + if app_env != "production": # Set the DEBUG_METRICS env var to also enable the # prometheus metrics exporter when running in development mode - os.environ['DEBUG_METRICS'] = 'true' + os.environ["DEBUG_METRICS"] = "true" # load app config app_config = config.get_config_by_name(app_env, settings=settings) # set the FlaskApp instance path @@ -77,24 +83,26 @@ def create_app(env=None, settings=None, init_app=True, init_integrations=True, if os.environ.get("FLASK_APP_CONFIG_FILE", None): app.config.from_envvar("FLASK_APP_CONFIG_FILE") # set worker flag - app.config['WORKER'] = worker + app.config["WORKER"] = worker # append proxy settings - app.config['PROXY_ENTRIES'] = config.load_proxy_entries(app.config) + app.config["PROXY_ENTRIES"] = config.load_proxy_entries(app.config) # initialize the application if init_app: with app.app_context() as ctx: - initialize_app(app, ctx, load_jobs=load_jobs, load_integrations=init_integrations) + initialize_app( + app, ctx, load_jobs=load_jobs, load_integrations=init_integrations + ) @app.route("/") def index(): if not current_user.is_authenticated: return render_template("index.j2") - return redirect(url_for('auth.index')) + return redirect(url_for("auth.index")) @app.route("/profile") def profile(): - return redirect(url_for('auth.index', back=request.args.get('back', False))) + return redirect(url_for("auth.index", back=request.args.get("back", False))) # append routes to check app health @app.route("/health") @@ -103,11 +111,23 @@ def health(): @app.route("/openapi.html") def openapi(): - return redirect('/static/specs/apidocs.html', code=302) + return redirect("/static/specs/apidocs.html", code=302) + + @app.route("/maintenance") + def maintenance(): + if not app.config.get("MAINTENANCE_MODE", False): + return redirect(url_for("index")) + return render_template("maintenance/maintenance.j2", + main_message=app.config.get("MAINTENANCE_MODE_MAIN_MESSAGE", None), + secondary_message=app.config.get("MAINTENANCE_MODE_SECONDARY_MESSAGE", None)) @app.before_request def set_request_start_time(): request.start_time = time.time() + if app.config.get("MAINTENANCE_MODE", False): + logger.debug("Application is running in maintenance mode. Request %s cannot be served!", request.path) + if not request.path.startswith("/maintenance"): + return redirect('/maintenance') @app.after_request def log_response(response): @@ -120,7 +140,7 @@ def log_response(response): # for h in request.headers: # logger.debug("header: %s %s", h, request.headers.get(h, None)) # log the request - processing_time = (time.time() * 1000.0 - request.start_time * 1000.0) + processing_time = time.time() * 1000.0 - request.start_time * 1000.0 logger.info( "resp: %s %s %s %s %s %s %s %s %0.3fms", request.remote_addr, @@ -131,7 +151,7 @@ def log_response(response): response.content_length, request.referrer, request.user_agent, - processing_time + processing_time, ) # return the response return response @@ -139,37 +159,45 @@ def log_response(response): return app -def initialize_app(app: Flask, app_context, prom_registry=None, load_jobs: bool = True, load_integrations: bool = True): +def initialize_app( + app: Flask, + app_context, + prom_registry=None, + load_jobs: bool = True, + load_integrations: bool = True, +): # init tmp folder - os.makedirs(app.config.get('BASE_TEMP_FOLDER'), exist_ok=True) + os.makedirs(app.config.get("BASE_TEMP_FOLDER"), exist_ok=True) # enable CORS CORS(app, expose_headers=["Content-Type", "X-CSRFToken"], supports_credentials=True) # configure logging config.configure_logging(app) - # register error handlers - errors_controller.register_api(app) - # init Redis connection - redis.init(app) - # configure app DB - db.init_app(app) - # initialize Migration engine - Migrate(app, db) - # initialize cache - init_cache(app) - # configure serializer engine (Flask Marshmallow) - ma.init_app(app) - # configure app routes - register_routes(app) - # init scheduler/worker for async tasks - init_task_queues(app, load_jobs=load_jobs) - # init mail system - init_mail(app) - # initialize integrations - if load_integrations: - init_integrations(app) - # initialize metrics engine - init_metrics(app, prom_registry) - # register commands - commands.register_commands(app) - # register the domain filter with Jinja - app.jinja_env.filters['domain'] = get_domain + # check if the app is running in maintenance mode + if app.config.get("MAINTENANCE_MODE", False): + logger.warning("Application is running in maintenance mode") + else: + # register error handlers + errors_controller.register_api(app) + # init Redis connection + redis.init(app) + # configure app DB + db.init_app(app) + # initialize Migration engine + Migrate(app, db) + # initialize cache + init_cache(app) + # configure serializer engine (Flask Marshmallow) + ma.init_app(app) + # configure app routes + register_routes(app) + # init scheduler/worker for async tasks + init_task_queues(app, load_jobs=load_jobs) + # init mail system + init_mail(app) + # initialize integrations + if load_integrations: + init_integrations(app) + # initialize metrics engine + init_metrics(app, prom_registry) + # register commands + commands.register_commands(app) diff --git a/lifemonitor/commands/backup.py b/lifemonitor/commands/backup.py index 0ad6020c..9acb367c 100644 --- a/lifemonitor/commands/backup.py +++ b/lifemonitor/commands/backup.py @@ -25,6 +25,7 @@ import sys import time from pathlib import Path +from typing import BinaryIO import click from click_option_group import GroupedOption, optgroup @@ -32,7 +33,8 @@ from flask.blueprints import Blueprint from flask.cli import with_appcontext from flask.config import Config -from lifemonitor.utils import FtpUtils + +from lifemonitor.utils import FtpUtils, encrypt_folder from .db import backup, backup_options @@ -45,6 +47,15 @@ # set help for the CLI command _blueprint.cli.help = "Manage backups of database and RO-Crates" +# define the encryption key options +encryption_key_option = click.option("-k", "--encryption-key", default=None, help="Encryption key") +encryption_key_file_option = click.option("-kf", "--encryption-key-file", + type=click.File("rb"), default=None, + help="File containing the encryption key") +encryption_asymmetric_option = click.option("-a", "--encryption-asymmetric", is_flag=True, default=False, + show_default=True, + help="Use asymmetric encryption") + class RequiredIf(GroupedOption): def __init__(self, *args, **kwargs): @@ -113,17 +124,29 @@ def bck(ctx): @backup_options @synch_otptions @with_appcontext -def db_cmd(file, directory, verbose, *args, **kwargs): +def db_cmd(file, directory, + encryption_key, encryption_key_file, encryption_asymmetric, + verbose, *args, **kwargs): """ Make a backup of the database """ - result = backup_db(directory, file, verbose, *args, **kwargs) + result = backup_db(directory, file, + encryption_key=encryption_key, + encryption_key_file=encryption_key_file, + encryption_asymmetric=encryption_asymmetric, + verbose=verbose, *args, **kwargs) sys.exit(result) -def backup_db(directory, file=None, verbose=False, *args, **kwargs): +def backup_db(directory, file=None, + encryption_key=None, encryption_key_file=None, encryption_asymmetric=False, + verbose=False, *args, **kwargs): logger.debug(sys.argv) - result = backup(directory, file, verbose) + logger.debug("Backup DB: %r - %r - %r - %r - %r - %r - %r",) + logger.warning(f"Encryption asymmetric: {encryption_asymmetric}") + result = backup(directory, file, + encryption_key=encryption_key, encryption_key_file=encryption_key_file, + encryption_asymmetric=encryption_asymmetric, verbose=verbose) if result.returncode == 0: synch = kwargs.pop('synch', False) if synch: @@ -134,21 +157,45 @@ def backup_db(directory, file=None, verbose=False, *args, **kwargs): @bck.command("crates") @click.option("-d", "--directory", default="./", show_default=True, help="Local path to store RO-Crates") +@encryption_key_option +@encryption_key_file_option +@encryption_asymmetric_option @synch_otptions @with_appcontext -def crates_cmd(directory, *args, **kwargs): +def crates_cmd(directory, + encryption_key, encryption_key_file, encryption_asymmetric, + *args, **kwargs): """ Make a backup of the registered workflow RO-Crates """ - result = backup_crates(current_app.config, directory, *args, **kwargs) - sys.exit(result) + result = backup_crates(current_app.config, directory, + encryption_key=encryption_key, encryption_key_file=encryption_key_file, + encryption_asymmetric=encryption_asymmetric, + *args, **kwargs) + sys.exit(result.returncode) -def backup_crates(config, directory, *args, **kwargs): +def backup_crates(config, directory, + encryption_key: bytes = None, encryption_key_file: BinaryIO = None, + encryption_asymmetric: bool = False, + *args, **kwargs) -> subprocess.CompletedProcess: assert config.get("DATA_WORKFLOWS", None), "DATA_WORKFLOWS not configured" + # get the path of the RO-Crates rocrate_source_path = config.get("DATA_WORKFLOWS").removesuffix('/') + # create the directory if not exists os.makedirs(directory, exist_ok=True) - result = subprocess.run(f'rsync -avh --delete {rocrate_source_path}/ {directory} ', shell=True, capture_output=True) + # flag to check the result of the rsync or encrypt command + result = False + # encrypt the RO-Crates if an encryption key is provided + if encryption_key or encryption_key_file: + if not encryption_key: + encryption_key = encryption_key_file.read() + result = encrypt_folder(rocrate_source_path, directory, encryption_key, + encryption_asymmetric=encryption_asymmetric) + result = subprocess.CompletedProcess(returncode=0 if result else 1, args=()) + else: + result = subprocess.run(f'rsync -avh --delete {rocrate_source_path}/ {directory} ', + shell=True, capture_output=True) if result.returncode == 0: print("Created backup of workflow RO-Crates @ '%s'" % directory) synch = kwargs.pop('synch', False) @@ -156,8 +203,11 @@ def backup_crates(config, directory, *args, **kwargs): logger.debug("Remaining args: %r", kwargs) return __remote_synch__(source=directory, **kwargs) else: - print("Unable to backup workflow RO-Crates\n%s", result.stderr.decode()) - return result.returncode + try: + print("Unable to backup workflow RO-Crates\n%s", result.stderr.decode()) + except Exception: + print("Unable to backup workflow RO-Crates\n") + return result def auto(config: Config): @@ -167,19 +217,38 @@ def auto(config: Config): click.echo("No BACKUP_LOCAL_PATH found in your settings") sys.exit(0) + # search for an encryption key file + encryption_key = None + encryption_key_file = config.get("BACKUP_ENCRYPTION_KEY_PATH", None) + if not encryption_key_file: + click.echo("WARNING: No BACKUP_ENCRYPTION_KEY_PATH found in your settings") + logger.warning("No BACKUP_ENCRYPTION_KEY_PATH found in your settings") + else: + # read the encryption key from the file if the key is not provided + if isinstance(encryption_key_file, str): + with open(encryption_key_file, "rb") as encryption_key_file: + encryption_key = encryption_key_file.read() + elif isinstance(encryption_key_file, BinaryIO): + encryption_key = encryption_key_file.read() + else: + raise ValueError("Invalid encryption key file") + # set paths base_path = base_path.removesuffix('/') # remove trailing '/' db_backups = f"{base_path}/db" rc_backups = f"{base_path}/crates" logger.debug("Backup paths: %r - %r - %r", base_path, db_backups, rc_backups) # backup database - result = backup(db_backups) + result = backup(db_backups, + encryption_key=encryption_key, + encryption_asymmetric=True) if result.returncode != 0: sys.exit(result.returncode) # backup crates - result = backup_crates(config, rc_backups) - if result != 0: - sys.exit(result) + result = backup_crates(config, rc_backups, + encryption_key=encryption_key, encryption_asymmetric=True) + if result.returncode != 0: + sys.exit(result.returncode) # clean up old files retain_days = int(config.get("BACKUP_RETAIN_DAYS", -1)) logger.debug("RETAIN DAYS: %d", retain_days) diff --git a/lifemonitor/commands/db.py b/lifemonitor/commands/db.py index 13f55194..7bd09bfd 100644 --- a/lifemonitor/commands/db.py +++ b/lifemonitor/commands/db.py @@ -24,13 +24,15 @@ import subprocess import sys from datetime import datetime +from typing import BinaryIO import click from flask import current_app from flask.cli import with_appcontext from flask_migrate import cli, current, stamp, upgrade + from lifemonitor.auth.models import User -from lifemonitor.utils import hide_secret +from lifemonitor.utils import decrypt_file, encrypt_file, hide_secret # set module level logger logger = logging.getLogger() @@ -105,11 +107,20 @@ def wait_for_db(): # define common options verbose_option = click.option("-v", "--verbose", default=False, is_flag=True, help="Enable verbose mode") +encryption_key_option = click.option("-k", "--encryption-key", default=None, help="Encryption key") +encryption_key_file_option = click.option("-kf", "--encryption-key-file", + type=click.File("rb"), + default=None, help="File containing the encryption key") +encryption_asymmetric_option = click.option("-a", "--encryption-asymmetric", is_flag=True, default=False, + help="Use asymmetric encryption", show_default=True) def backup_options(func): # backup command options (evaluated in reverse order!) func = verbose_option(func) + func = encryption_asymmetric_option(func) + func = encryption_key_file_option(func) + func = encryption_key_option(func) func = click.option("-f", "--file", default=None, help="Backup filename (default 'hhmmss_yyyymmdd.tar')")(func) func = click.option("-d", "--directory", default="./", help="Directory path for the backup file (default '.')")(func) return func @@ -118,20 +129,34 @@ def backup_options(func): @cli.db.command("backup") @backup_options @with_appcontext -def backup_cmd(directory, file, verbose): +def backup_cmd(directory, file, + encryption_key, encryption_key_file, encryption_asymmetric, + verbose): """ Make a backup of the current app database """ - result = backup(directory, file, verbose) + logger.debug("%r - %r - %r - %r - %r - %r ", file, directory, + encryption_key, encryption_key_file, encryption_asymmetric, verbose) + result = backup(directory, file, + encryption_key=encryption_key, + encryption_key_file=encryption_key_file, + encryption_asymmetric=encryption_asymmetric, + verbose=verbose) # report exit code to the main process sys.exit(result.returncode) -def backup(directory, file=None, verbose=False) -> subprocess.CompletedProcess: +def backup(directory, file=None, + encryption_key=None, encryption_key_file: BinaryIO = None, + encryption_asymmetric=False, + verbose=False) -> subprocess.CompletedProcess: """ Make a backup of the current app database """ - logger.debug("%r - %r - %r", file, directory, verbose) + logger.debug("%r - %r - %r - %r - %r - %r", + file, directory, + encryption_key, encryption_key_file, encryption_asymmetric, + verbose) from lifemonitor.db import db_connection_params params = db_connection_params() if not file: @@ -148,6 +173,30 @@ def backup(directory, file=None, verbose=False) -> subprocess.CompletedProcess: msg = f"Created backup of database {params['dbname']} @ {target_path}" logger.debug(msg) print(msg) + if encryption_key is not None or encryption_key_file is not None: + msg = f"Encrypting backup file {target_path}..." + logger.debug(msg) + print(msg) + # read the encryption key from the file if the key is not provided + if encryption_key is None: + encryption_key = encryption_key_file.read() + # encrypt the backup file using the encryption key with the Fernet algorithm + try: + with open(target_path, "rb") as input_file: + with open(target_path + ".enc", "wb") as output_file: + encrypt_file(input_file, output_file, encryption_key, + encryption_asymmetric=encryption_asymmetric, + raise_error=True) + # remove the original backup file + os.remove(target_path) + msg = f"Backup file {target_path} encrypted" + logger.debug(msg) + print(msg) + except ValueError as e: + logger.error("Unable to encrypt backup file '%s'. ERROR: %s", target_path, str(e)) + except Exception as e: + print("Unable to encrypt backup file '%s'. ERROR: %s" % (target_path, str(e))) + sys.exit(1) else: click.echo("\nERROR Unable to backup the database: %s" % result.stderr.decode()) if verbose and result.stderr: @@ -159,79 +208,117 @@ def backup(directory, file=None, verbose=False) -> subprocess.CompletedProcess: @click.argument("file") @click.option("-s", "--safe", default=False, is_flag=True, help="Preserve the current database renaming it as '_yyyymmdd_hhmmss'") +@encryption_key_option +@encryption_key_file_option +@encryption_asymmetric_option @verbose_option @with_appcontext -def restore(file, safe, verbose): +def restore(file, safe, + encryption_key, encryption_key_file, encryption_asymmetric, + verbose): """ Restore a backup of the app database """ from lifemonitor.db import (create_db, db_connection_params, db_exists, drop_db, rename_db) + # initialize the encrypted file reference + encrypted_file = None + # check if DB file exists if not os.path.isfile(file): print("File '%s' not found!" % file) sys.exit(128) - # check if delete or preserve the current app database (if exists) - new_db_name = None - params = db_connection_params() - db_copied = False - if db_exists(params['dbname']): - if safe: - answer = input(f"The database '{params['dbname']}' will be renamed. Continue? (y/n): ") - if not answer.lower() in ('y', 'yes'): - sys.exit(0) - else: - answer = input(f"The database '{params['dbname']}' will be delete. Continue? (y/n): ") - if not answer.lower() in ('y', 'yes'): - sys.exit(0) - # create a snapshot of the current database - new_db_name = f"{params['dbname']}_{datetime.now().strftime('%Y%m%d_%H%M%S')}" - rename_db(params['dbname'], new_db_name) - db_copied = True - msg = f"Created a DB snapshot: data '{params['dbname']}' temporarily renamed as '{new_db_name}'" - logger.debug(msg) - if verbose: - print(msg) - # restore database - create_db(current_app.config) - cmd = f"PGPASSWORD={params['password']} pg_restore -h {params['host']} -U {params['user']} -d {params['dbname']} -v {file}" - if verbose: - print("Dabaset file: %s" % file) - print("Backup command: %s" % hide_secret(cmd, params['password'])) - result = subprocess.run(cmd, shell=True) - logger.debug("Restore result: %r", hide_secret(cmd, params['password'])) - if result.returncode == 0: - if db_copied and safe: - print(f"Existing database '{params['dbname']}' renamed as '{new_db_name}'") - msg = f"Backup {file} restored to database '{params['dbname']}'" - logger.debug(msg) - print(msg) - # if mode is set to 'not safe' - # delete the temp snapshot of the current database - if not safe: - drop_db(db_name=new_db_name) - msg = f"Current database '{params['dbname']}' deleted" - logger.debug(msg) - if verbose: - print(msg) - else: - # if any error occurs - # restore the previous latest version of the DB - # previously saved as temp snapshot - if new_db_name: - # delete the db just created - drop_db() - # restore the old database snapshot - rename_db(new_db_name, params['dbname']) + + try: + + # check if the DB backup is encrypted and the key or key file is provided + if file.endswith(".enc"): + if encryption_key is None and encryption_key_file is None: + print("The backup file '%s' is encrypted but no encryption key is provided!" % file) + sys.exit(128) + + # read the encryption key from the file if the key is not provided + if encryption_key is None: + encryption_key = encryption_key_file.read() + + # Set the reference to the encrypted file + encrypted_file = file + + # decrypt the backup file using the encryption key with the Fernet algorithm + file = file.removesuffix(".enc") + with open(encrypted_file, "rb") as input_file: + with open(file, "wb") as output_file: + decrypt_file(input_file, output_file, + encryption_key, encryption_asymmetric=encryption_asymmetric) + logger.debug("Decrypted backup file '%s' to '%s'", encrypted_file, file) + + # check if delete or preserve the current app database (if exists) + new_db_name = None + params = db_connection_params() + db_copied = False + if db_exists(params['dbname']): + if safe: + answer = input(f"The database '{params['dbname']}' will be renamed. Continue? (y/n): ") + if not answer.lower() in ('y', 'yes'): + sys.exit(0) + else: + answer = input(f"The database '{params['dbname']}' will be delete. Continue? (y/n): ") + if not answer.lower() in ('y', 'yes'): + sys.exit(0) + # create a snapshot of the current database + new_db_name = f"{params['dbname']}_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + rename_db(params['dbname'], new_db_name) db_copied = True - msg = f"Database restored '{params['dbname']}' renamed as '{new_db_name}'" + msg = f"Created a DB snapshot: data '{params['dbname']}' temporarily renamed as '{new_db_name}'" logger.debug(msg) if verbose: print(msg) - print("ERROR: Unable to restore the database backup") - if verbose and result.stderr: - print("ERROR [stderr]: %s" % result.stderr.decode()) + # restore database + create_db(current_app.config) + cmd = f"PGPASSWORD={params['password']} pg_restore -h {params['host']} -U {params['user']} -d {params['dbname']} -v {file}" + if verbose: + print("Dabaset file: %s" % file) + print("Backup command: %s" % hide_secret(cmd, params['password'])) + result = subprocess.run(cmd, shell=True) + logger.debug("Restore result: %r", hide_secret(cmd, params['password'])) + if result.returncode == 0: + if db_copied and safe: + print(f"Existing database '{params['dbname']}' renamed as '{new_db_name}'") + msg = f"Backup {file} restored to database '{params['dbname']}'" + logger.debug(msg) + print(msg) + # if mode is set to 'not safe' + # delete the temp snapshot of the current database + if not safe: + drop_db(db_name=new_db_name) + msg = f"Current database '{params['dbname']}' deleted" + logger.debug(msg) + if verbose: + print(msg) + else: + # if any error occurs + # restore the previous latest version of the DB + # previously saved as temp snapshot + if new_db_name: + # delete the db just created + drop_db() + # restore the old database snapshot + rename_db(new_db_name, params['dbname']) + db_copied = True + msg = f"Database restored '{params['dbname']}' renamed as '{new_db_name}'" + logger.debug(msg) + if verbose: + print(msg) + print("ERROR: Unable to restore the database backup") + if verbose and result.stderr: + print("ERROR [stderr]: %s" % result.stderr.decode()) + finally: + if encrypted_file and os.path.isfile(file): + # remove the decrypted file + os.remove(file) + logger.debug("Removed decrypted backup file '%s'", file) + # report exit code to the main process sys.exit(result.returncode) diff --git a/lifemonitor/commands/encrypt.py b/lifemonitor/commands/encrypt.py new file mode 100644 index 00000000..3f7b3f7e --- /dev/null +++ b/lifemonitor/commands/encrypt.py @@ -0,0 +1,251 @@ +# Copyright (c) 2020-2022 CRS4 +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import base64 +import logging +import os +import sys + +import click +from flask import Blueprint + +from ..utils import (decrypt_file, decrypt_folder, encrypt_file, + encrypt_folder, generate_asymmetric_encryption_keys, + generate_symmetric_encryption_key, serialization) + +# set module level logger +logger = logging.getLogger(__name__) + +# define the blueprint for DB commands +blueprint = Blueprint('ed', __name__) + +# set CLI help +blueprint.cli.help = "Manage files encryption/decryption" + +# define the encryption key options +encryption_asymmetric_option = click.option("-a", "--encryption-asymmetric", is_flag=True, default=False, + help="Use asymmetric encryption", show_default=True) +encryption_key_option = click.option("-k", "--encryption-key", default=None, help="Encryption key") +encryption_key_file_option = click.option("-kf", "--encryption-key-file", + type=click.File("rb"), default="lifemonitor.key", + help="File containing the encryption key") + + +@blueprint.cli.command('gen-keys') +@click.option("-f", "--key-file", type=click.Path(exists=False), default="lifemonitor.key", show_default=True) +@encryption_asymmetric_option +def generate_encryption_keys_cmd(key_file, encryption_asymmetric): + """Generate a new pair of encryption keys""" + try: + # init reference to the key (symmetric or asymmetric) bytes + key = None + # check if the key file already exists + if os.path.exists(key_file): + print("Key file '%s' already exists" % os.path.abspath(key_file)) + sys.exit(1) + if not encryption_asymmetric: + # generate the key + key = generate_symmetric_encryption_key() + print("Key generated: %s" % key.decode("utf-8")) + # save the key + with open(key_file, "wb") as f: + f.write(key) + print("Key saved in '%s'" % os.path.abspath(key_file)) + else: + # generate the key pair + priv, pub = generate_asymmetric_encryption_keys(key_filename=key_file) + logger.debug(f"Keys saved: private={key_file}, public={key_file + '.pub'}") + logger.debug(f"Private key: {priv}") + logger.debug(f"Public key: {pub}") + print("Keys saved: private=%s, public=%s" % (key_file, key_file + ".pub")) + # set reference to the public key + key = pub.public_bytes(encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo) + # generate the kubernetes secret containing the key + with open(key_file + ".secret.yaml", "w") as f: + with open(os.path.join("k8s", "backup-key.secret.yaml"), "r") as t: + # base 64 encode the key + f.write(t.read().replace("", + base64.b64encode(key).decode("utf-8"))) + sys.exit(0) + except Exception as e: + if logger.isEnabledFor(logging.DEBUG): + logger.exception(e) + else: + logger.error(f"Error generating key: {e}") + sys.exit(1) + + +@blueprint.cli.command('encrypt') +@click.argument("input_file", metavar="input", type=click.File("rb")) +@click.option("-o", "--out", type=click.File("wb"), default=".enc", show_default=True, help="Output file") +@encryption_key_option +@encryption_key_file_option +@encryption_asymmetric_option +def encrypt_cmd(input_file, out, encryption_key, encryption_key_file, encryption_asymmetric): + """Encrypt a file""" + try: + # log the parameters + logger.debug(f"Input file: {input_file.name}") + logger.debug(f"Output file: {out.name}") + logger.debug(f"Encryption key: {encryption_key}") + logger.debug(f"Encryption key file: {encryption_key_file.name}") + + # # check if the key or key file are not set + if encryption_key is None and encryption_key_file is None: + print("ERROR: Key or key file should be set") + sys.exit(1) + # check if the output file already exists + if os.path.exists(out.name): + print("ERROR: Output file '%s' already exists" % os.path.abspath(out.name)) + sys.exit(1) + # initialize the output file + if out.name == ".enc": + out.name = "%s.enc" % os.path.abspath(input_file.name) + + # read the encryption key from the file if the key is not provided + if encryption_key is None: + encryption_key = encryption_key_file.read() + + # encrypt the file + encrypt_file(input_file, out, encryption_key, encryption_asymmetric=encryption_asymmetric) + logger.debug(f"File encrypted: {out.name}") + print(f"File encrypted: {out.name}") + sys.exit(0) + except Exception as e: + if logger.isEnabledFor(logging.DEBUG): + logger.exception(e) + else: + logger.error(f"Error encrypting file: {e}") + sys.exit(1) + + +@blueprint.cli.command('encrypt-folder') +@click.argument("input_folder", type=click.Path(exists=True)) +@click.option("-o", "--output_folder", type=click.Path(exists=False), default="", show_default=True, help="Output file") +@encryption_key_option +@encryption_key_file_option +@encryption_asymmetric_option +def encrypt_folder_cmd(input_folder, output_folder, + encryption_key, encryption_key_file, encryption_asymmetric): + + # log the parameters + logger.debug(f"Input folder: {input_folder}") + logger.debug(f"Output file: {output_folder}") + logger.debug(f"Encryption key: {encryption_key}") + logger.debug(f"Encryption key file: {encryption_key_file.name}") + + # # check if the key or key file are not set + if encryption_key is None and encryption_key_file is None: + print("ERROR: Key or key file should be set") + sys.exit(1) + + # init the output folder + if output_folder == "": + output_folder = input_folder + logger.debug(f"Using Output folder: {output_folder}") + + # read the encryption key from the file if the key is not provided + if encryption_key is None: + encryption_key = encryption_key_file.read() + + # encrypt the folder + encrypted_files = encrypt_folder(input_folder, output_folder, encryption_key, encryption_asymmetric=encryption_asymmetric) + print(f"Encryption completed: {encrypted_files} files encrypted on {output_folder}") + sys.exit(0) + + +@blueprint.cli.command('decrypt') +@click.argument("input_file", metavar="input", type=click.File("rb")) +@click.option("-o", "--out", type=click.File("wb"), default="", show_default=True, help="Output file") +@encryption_key_option +@encryption_key_file_option +@encryption_asymmetric_option +def decrypt_cmd(input_file, out, encryption_key, encryption_key_file, encryption_asymmetric): + """Decrypt a file""" + try: + # log the parameters + logger.debug(f"Input file: {input_file.name}") + logger.debug(f"Output file: {out.name}") + logger.debug(f"Encryption key: {encryption_key}") + logger.debug(f"Encryption key file: {encryption_key_file.name}") + + # check if the key or key file are not set + if encryption_key is None and encryption_key_file is None: + print("ERROR: Key or key file should be set") + sys.exit(1) + + # check if the output file already exists + if os.path.exists(out.name): + print("Output file '%s' already exists" % os.path.abspath(out.name)) + sys.exit(1) + # initialize the output file + if out.name == "": + out.name = "%s" % os.path.abspath(input_file.name).removesuffix(".enc") + # read the encryption key from the file if the key is not provided + if encryption_key is None: + encryption_key = encryption_key_file.read() + # decrypt the file + decrypt_file(input_file, out, encryption_key, encryption_asymmetric=encryption_asymmetric) + logger.debug(f"File decrypted: {out.name}") + print(f"File decrypted: {out.name}") + sys.exit(0) + except Exception as e: + if logger.isEnabledFor(logging.DEBUG): + logger.exception(e) + else: + logger.error(f"Error decrypting file: {e}") + sys.exit(1) + + +@blueprint.cli.command('decrypt-folder') +@click.argument("input_folder", type=click.Path(exists=True)) +@click.option("-o", "--output_folder", type=click.Path(exists=False), default="", show_default=True, help="Output file") +@encryption_key_option +@encryption_key_file_option +@encryption_asymmetric_option +def decrypt_folder_cmd(input_folder, output_folder, + encryption_key, encryption_key_file, encryption_asymmetric): + + # log the parameters + logger.debug(f"Input folder: {input_folder}") + logger.debug(f"Output file: {output_folder}") + logger.debug(f"Encryption key: {encryption_key}") + logger.debug(f"Encryption key file: {encryption_key_file.name}") + + # check if the key or key file are not set + if encryption_key is None and encryption_key_file is None: + print("ERROR: Key or key file should be set") + sys.exit(1) + + # init the output folder + if output_folder == "": + output_folder = input_folder + logger.debug(f"Using Output folder: {output_folder}") + + # read the encryption key from the file if the key is not provided + if encryption_key is None: + encryption_key = encryption_key_file.read() + + # decrypt the folder + decrypted_files = decrypt_folder(input_folder, output_folder, + encryption_key, asymmetric_encryption=encryption_asymmetric) + print(f"Decryption completed: {decrypted_files} files decrypted on {output_folder}") + sys.exit(0) diff --git a/lifemonitor/config.py b/lifemonitor/config.py index 463c0a1f..069cf498 100644 --- a/lifemonitor/config.py +++ b/lifemonitor/config.py @@ -130,6 +130,8 @@ class BaseConfig: SERVICE_AVAILABILITY_TIMEOUT = 1 # Cookie Settings SESSION_COOKIE_NAME = 'lifemonitor_session' + # Disable Maintenance Mode by default + MAINTENANCE_MODE = False class DevelopmentConfig(BaseConfig): @@ -149,6 +151,13 @@ class ProductionConfig(BaseConfig): CACHE_TYPE = "flask_caching.backends.rediscache.RedisCache" +class MaintenanceConfig(BaseConfig): + CONFIG_NAME = "maintenance" + TESTING = False + CACHE_TYPE = "flask_caching.backends.rediscache.RedisCache" + MAINTENANCE_MODE = True + + class TestingConfig(BaseConfig): CONFIG_NAME = "testing" SETTINGS_FILE = "tests/settings.conf" @@ -178,7 +187,8 @@ class TestingSupportConfig(TestingConfig): DevelopmentConfig, TestingConfig, ProductionConfig, - TestingSupportConfig + TestingSupportConfig, + MaintenanceConfig ] _config_by_name = {cfg.CONFIG_NAME: cfg for cfg in _EXPORT_CONFIGS} diff --git a/lifemonitor/static/img/icons/maintenance-1.png b/lifemonitor/static/img/icons/maintenance-1.png new file mode 100644 index 00000000..485af25c Binary files /dev/null and b/lifemonitor/static/img/icons/maintenance-1.png differ diff --git a/lifemonitor/static/img/icons/maintenance-2.png b/lifemonitor/static/img/icons/maintenance-2.png new file mode 100644 index 00000000..485af25c Binary files /dev/null and b/lifemonitor/static/img/icons/maintenance-2.png differ diff --git a/lifemonitor/tasks/worker.py b/lifemonitor/tasks/worker.py index e8e5fb63..8888c20c 100644 --- a/lifemonitor/tasks/worker.py +++ b/lifemonitor/tasks/worker.py @@ -5,9 +5,13 @@ logger = logging.getLogger(__name__) - app = create_app(worker=True, load_jobs=True) app.app_context().push() - -broker = app.broker +# check if the app is in maintenance mode +if app.config.get("MAINTENANCE_MODE", False): + logger.warning("Application is in maintenance mode") + app.run() +else: + # initialise the message broker + broker = app.broker diff --git a/lifemonitor/templates/maintenance/base.j2 b/lifemonitor/templates/maintenance/base.j2 new file mode 100644 index 00000000..8f2229b9 --- /dev/null +++ b/lifemonitor/templates/maintenance/base.j2 @@ -0,0 +1,58 @@ + + + + + {% block title %}Life Monitor{% endblock %} + + + + {% block stylesheets %} + + + + + + + + + + + + + + + + + + + + + + {% endblock stylesheets %} + + {% block extra_stylesheets %} {%endblock extra_stylesheets %} + + + + {% block body %}{% endblock %} + + + {% block javascripts_libraries %} + + + + + + + + + {# Enable notifications #} + {{ macros.messages() }} + + {% endblock javascripts_libraries %} + + {% block javascripts %} {% endblock javascripts %} + + + diff --git a/lifemonitor/templates/maintenance/maintenance.j2 b/lifemonitor/templates/maintenance/maintenance.j2 new file mode 100644 index 00000000..f460b586 --- /dev/null +++ b/lifemonitor/templates/maintenance/maintenance.j2 @@ -0,0 +1,59 @@ +{% extends 'maintenance/base.j2' %} +{% import 'macros.j2' as macros %} + + +{% block title %} {{ title }} {% endblock title %} + + +{% block extra_stylesheets %} {% endblock extra_stylesheets %} + + +{% block body_class %} error-page {% endblock %} + + +{% block body %} + +
+
+ {{ macros.render_logo(style="width: 200px; margin: auto;") }} +
+

+ We'll be back soon! +

+
+ +
+ + {% if not main_message %} + +

+ We're busy updating the + Life-Monitor + service for you.
Please check back soon! +

+ {% else %} +

+ {{ main_message }} +

+ {% endif %} + + + {% if secondary_message %} +

+ {{ secondary_message }} +

+ {% endif %} + + + + + +
+ +{% endblock body %} + + +{% block javascripts %} +{% endblock javascripts %} \ No newline at end of file diff --git a/lifemonitor/utils.py b/lifemonitor/utils.py index a912ce02..b19438fa 100644 --- a/lifemonitor/utils.py +++ b/lifemonitor/utils.py @@ -34,6 +34,7 @@ import shutil import socket import string +import struct import subprocess import tempfile import time @@ -43,7 +44,8 @@ from datetime import datetime, timezone from importlib import import_module from os.path import basename, dirname, isfile, join -from typing import Dict, Iterable, List, Literal, Optional, Tuple, Type +from typing import (BinaryIO, Dict, Iterable, List, Literal, Optional, Tuple, + Type) from urllib.parse import urlparse import flask @@ -53,6 +55,9 @@ import pygit2 import requests import yaml +from cryptography.fernet import Fernet +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import padding, rsa from dateutil import parser from lifemonitor.cache import cached @@ -1248,3 +1253,257 @@ def rm_tree(self, path): self.ftp.rmd(path) except ftplib.all_errors as e: logger.debug('Could not remove {0}: {1}'.format(path, e)) + + +def generate_symmetric_encryption_key() -> bytes: + """Generate a new encryption key""" + key = None + try: + key = Fernet.generate_key() + logger.debug("Encryption key generated: %r", key) + except Exception as e: + if logger.isEnabledFor(logging.DEBUG): + logger.exception(e) + return key + + +def generate_asymmetric_encryption_keys( + key_filename: str = "lifemonitor.key", + public_exponent=65537, key_size=2048) -> Tuple[rsa.RSAPrivateKey, rsa.RSAPublicKey]: + + # Generate the RSA private key + private_key = rsa.generate_private_key( + public_exponent=public_exponent, + key_size=key_size, + ) + + # Write the private key to a file + with open(f"{key_filename}", "wb") as key_file: + key_file.write( + private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption() + ) + ) + + # Extract the corresponding public key + public_key = private_key.public_key() + + # Write the public key to a file + with open(f"{key_filename}.pub", "wb") as key_file: + key_file.write( + public_key.public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo + ) + ) + + return private_key, public_key + + +def encrypt_file(input_file: BinaryIO, output_file: BinaryIO, key: bytes, + encryption_asymmetric: bool = False, + raise_error: bool = True, block=65536) -> bool: + """Encrypt a file using AES-256-CBC""" + # check if input and output are valid + if not input_file or not output_file: + raise ValueError("Invalid input/output file") + # check if the input file exists + if not os.path.exists(input_file.name): + raise ValueError(f"Input file {input_file.name} does not exist") + # check if the key is valid + if not key: + raise ValueError("Invalid encryption key") + try: + logger.warning("Encryption asymmetric: %r", encryption_asymmetric) + # encrypt the file chunk by chunk + # using a symmetric encryption algorithm + if not encryption_asymmetric: + cipher = Fernet(key) + while True: + chunk = input_file.read(block) + if not chunk or len(chunk) == 0: + break + enc = cipher.encrypt(chunk) + output_file.write(struct.pack(' bool: + + # check if the input folder exists + if not os.path.exists(input_folder): + raise ValueError(f"Input folder {input_folder} does not exist") + + # check if the key is valid + if not key: + raise ValueError("Invalid encryption key") + + # initialize the counter + count = 0 + try: + # walk on the input folder + for root, dirs, files in os.walk(input_folder): + for file in files: + input_file = os.path.join(root, file) + logger.debug(f"Input file: {input_file}") + file_output_folder = root.replace(input_folder, output_folder) + logger.debug(f"File output folder: {file_output_folder}") + if not os.path.exists(file_output_folder): + os.makedirs(file_output_folder, exist_ok=True) + logger.debug(f"Created folder: {file_output_folder}") + output_file = f"{os.path.join(file_output_folder, file)}.enc" + logger.debug(f"Encrypting file: {input_file}") + logger.debug(f"Output file: {output_file}") + with open(input_file, "rb") as f: + with open(output_file, "wb") as o: + encrypt_file(f, o, key, raise_error=raise_error, block=block, + encryption_asymmetric=encryption_asymmetric) + logger.debug(f"File encrypted: {output_file}") + print(f"File encrypted: {output_file}") + count += 1 + logger.debug(f"File encrypted: {count}") + logger.debug(f"Encryption completed: {count} files encrypted on {output_folder}") + except Exception as e: + if logger.isEnabledFor(logging.DEBUG): + logger.exception(e) + if raise_error: + raise lm_exceptions.LifeMonitorException(detail=str(e)) + return count + + +def decrypt_file(input_file: BinaryIO, output_file: BinaryIO, key: bytes, + encryption_asymmetric: bool = False, block=65536, + raise_error: bool = True) -> bool: + """Decrypt a file using AES-256-CBC""" + # check if input and output are valid + if not input_file or not output_file: + raise ValueError("Invalid input/output file") + # check if the input file exists + if not os.path.exists(input_file.name): + raise ValueError(f"Input file {input_file.name} does not exist") + # check if the key is valid + if not key: + raise ValueError("Invalid encryption key") + try: + # decrypt the file chunk by chunk + # using a symmetric encryption algorithm + if not encryption_asymmetric: + cipher = Fernet(key) + while True: + size_data = input_file.read(4) + if len(size_data) == 0: + break + chunk = input_file.read(struct.unpack(' int: + + # check if the input folder exists + if not os.path.exists(input_folder): + raise ValueError(f"Input folder {input_folder} does not exist") + + # check if the key is valid + if not key: + raise ValueError("Invalid encryption key") + + # walk on the input folder + count = 0 + try: + for root, dirs, files in os.walk(input_folder): + for file in files: + input_file = os.path.join(root, file) + file_output_folder = root.replace(input_folder, output_folder) + logger.debug(f"File output folder: {file_output_folder}") + if not os.path.exists(file_output_folder): + os.makedirs(file_output_folder, exist_ok=True) + logger.debug(f"Created folder: {file_output_folder}") + output_file = f"{os.path.join(file_output_folder, file).removesuffix('.enc')}" + logger.debug(f"Decrypting file: {input_file}") + logger.debug(f"Output file: {output_file}") + with open(input_file, "rb") as f: + with open(output_file, "wb") as o: + decrypt_file(f, o, key, raise_error=raise_error, + encryption_asymmetric=asymmetric_encryption) + logger.debug(f"File decrypted: {output_file}") + print(f"File decrypted: {output_file}") + count += 1 + logger.debug(f"File decrypted: {count}") + logger.debug(f"Decryption completed: {count} files decrypted on {output_folder}") + except Exception as e: + if logger.isEnabledFor(logging.DEBUG): + logger.exception(e) + if raise_error: + raise lm_exceptions.LifeMonitorException(detail=str(e)) + return count diff --git a/settings.conf b/settings.conf index d493aeb9..6953728f 100644 --- a/settings.conf +++ b/settings.conf @@ -7,6 +7,11 @@ LOG_LEVEL=INFO # default: 'INFO' on production, 'DEBUG' on development # Set the path for the log file # LOG_FILE_PATH=/var/log/lm # default: /var/log/lm +# Manage the maintenance mode +# MAINTENANCE_MODE=True # default: False +# MAINTENANCE_MODE_MAIN_MESSAGE="We're busy updating the Life-Monitor service for you.Please check back soon!" +# MAINTENANCE_MODE_SECONDARY_MESSAGE="We are currently performing maintenance on the LifeMonitor service. Please try again later." + # The name and port number of the back-end server (e.g., 'localhost:8000'). # If the back-end is served through a reverse proxy, # then you have to set SERVER_NAME to the appropriate proxy entry @@ -94,6 +99,7 @@ CACHE_WORKFLOW_TIMEOUT=1800 # Backup settings BACKUP_LOCAL_PATH="./backups" BACKUP_RETAIN_DAYS=30 +# BACKUP_ENCRYPTION_KEY_PATH= # BACKUP_REMOTE_PATH="lm-backups" # BACKUP_REMOTE_HOST="ftp-site.domain.it" # BACKUP_REMOTE_USER="lm" diff --git a/ws.py b/ws.py index 944854dc..0561f450 100644 --- a/ws.py +++ b/ws.py @@ -30,5 +30,9 @@ # create an app instance application = create_app(init_app=True, load_jobs=False, init_integrations=False) + +# initialise the websocket socketIO = initialise_ws(application) -start_brodcaster(application, max_age=5) # use default ws_channel +# initialise the message broker only if the app is not in maintenance mode +if not application.config.get("MAINTENANCE_MODE", False): + start_brodcaster(application, max_age=5) # use default ws_channel