Deployment/auto update

.github/workflows/docker.yml

@@ -42,8 +42,8 @@ jobs:
           - image: grafana
             dockerfile: docker/grafana.Dockerfile
             target:
-          - image: cron
-            dockerfile: docker/cron.Dockerfile
+          - image: util
+            dockerfile: docker/util.Dockerfile
             target:
 
     outputs:
@@ -93,7 +93,7 @@ jobs:
         # https://github.com/docker/login-action?tab=readme-ov-file#dockerhub
         uses: docker/login-action@v3
         with:
-          username: ${{ vars.DOCKERHUB_USERNAME }}
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
         # worst case of this failing is running into Docker Hub rate limit
         continue-on-error: true
@@ -115,16 +115,18 @@ jobs:
           # and tag with PR source branch (eg: feature-x)
           tags: |
             ${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:${{ steps.get_version.outputs.internetnl_version }}
+            ${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:pr-${{ github.event_name == 'pull_request' && github.event.number || github.ref_name}}
           # use latest build from main, or image previously build by this PR for caching
           cache-from: |
             ${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:main
+            ${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:pr-${{ github.event_name == 'pull_request' && github.event.number || github.ref_name}}
           # push images to registry
           push: true
           # makes build images better usable as cache by allowing individual layers to be pulled from cache
           # pass in version information
           build-args: |
             BUILDKIT_INLINE_CACHE=1
-            INTERNETNL_VERSION=${{ steps.get_version.outputs.internetnl_version }}
+            RELEASE=${{ steps.get_version.outputs.internetnl_version }}
 
       - name: Build ${{ matrix.image }} (for main)
         # build for pushes to the main branch
@@ -150,7 +152,7 @@ jobs:
           # pass in version information
           build-args: |
             BUILDKIT_INLINE_CACHE=1
-            INTERNETNL_VERSION=${{ steps.get_version.outputs.internetnl_version }}
+            RELEASE=${{ steps.get_version.outputs.internetnl_version }}
 
       - name: Build ${{ matrix.image }} (for release)
         # build for tagged releases
@@ -175,7 +177,7 @@ jobs:
           # pass in version information
           build-args: |
             BUILDKIT_INLINE_CACHE=1
-            INTERNETNL_VERSION=${{ steps.get_version.outputs.internetnl_version }}
+            RELEASE=${{ steps.get_version.outputs.internetnl_version }}
 
       - name: Build ${{ matrix.image }} (for forked PR's or dependabot)
         # build for forked PR's that don't have permissions to push to the container registry or dependabot PR's
@@ -198,7 +200,7 @@ jobs:
           # makes build images better usable as cache by allowing individual layers to be pulled from cache
           # pass in version information
           build-args: |
-            INTERNETNL_VERSION=${{ steps.get_version.outputs.internetnl_version }}
+            RELEASE=${{ steps.get_version.outputs.internetnl_version }}
 
       - name: Save image to disk (for forked PR's or dependabot)
         # trigger only for forked PR's that don't have permissions to push to the container registry
@@ -220,46 +222,17 @@ jobs:
     needs: [build-docker]
     steps:
       - name: Branch deployment docs
-        if: ${{ env.use_registry == 'true' }}
         run: |
           cat >> $GITHUB_STEP_SUMMARY <<EOF
 
-          To deploy this specific build to a existing deployment run the following update commands:
-
-              export BRANCH="${{ github.sha }}" && \\
-              export RELEASE="${{ needs.build-docker.outputs.internetnl_version }}" && \\
-              export DOCKER_REGISTRY="${{env.DOCKER_REGISTRY}}" && \\
-              cd /opt/Internet.nl/ && \\
-              curl -sSfO --output-dir docker https://raw.githubusercontent.com/${{github.repository}}/\$BRANCH/docker/defaults.env && \\
-              curl -sSfO --output-dir docker https://raw.githubusercontent.com/${{github.repository}}/\$BRANCH/docker/docker-compose.yml && \\
-              curl -sSfO https://raw.githubusercontent.com/${{github.repository}}/\$BRANCH/docker/user_manage.sh && \\
-              chmod 755 user_manage.sh && \\
-              env -i RELEASE="\$RELEASE" DOCKER_REGISTRY="\$DOCKER_REGISTRY" docker compose --env-file=docker/defaults.env --env-file=docker/host.env --env-file=docker/local.env pull && \\
-              # temporary solution to recreate containers when configs change: https://github.com/internetstandards/Internet.nl/issues/1490 \\
-              env -i RELEASE="\$RELEASE" DOCKER_REGISTRY="\$DOCKER_REGISTRY" docker compose --env-file=docker/defaults.env --env-file=docker/host.env --env-file=docker/local.env rm --stop --force cron-docker prometheus alertmanager nginx_logs_exporter && \\
-              env -i RELEASE="\$RELEASE" DOCKER_REGISTRY="\$DOCKER_REGISTRY" docker compose --env-file=docker/defaults.env --env-file=docker/host.env --env-file=docker/local.env up --remove-orphans --wait --no-build
-
-          EOF
-
-      - name: Release deployment docs
-        if: github.event_name == 'release'
-        run: |
-          cat >> $GITHUB_STEP_SUMMARY <<EOF
+          To deploy this release to a existing deployment run the following update command:
 
-          To deploy this release to an existing deployment run the following update commands:
-
-              export RELEASE="${{ needs.build-docker.outputs.internetnl_version }}" && \\
-              export TAG="v\$RELEASE" && \\
-              export DOCKER_REGISTRY="${{env.DOCKER_REGISTRY}}" && \\
-              cd /opt/Internet.nl/ && \\
-              curl -sSfO --output-dir docker https://raw.githubusercontent.com/${{github.repository}}/\$TAG/docker/defaults.env && \\
-              curl -sSfO --output-dir docker https://raw.githubusercontent.com/${{github.repository}}/\$TAG/docker/docker-compose.yml && \\
-              curl -sSfO https://raw.githubusercontent.com/${{github.repository}}/\$BRANCH/docker/user_manage.sh && \\
-              chmod 755 user_manage.sh && \\
-              env -i RELEASE="\$RELEASE" DOCKER_REGISTRY="\$DOCKER_REGISTRY" docker compose --env-file=docker/defaults.env --env-file=docker/host.env --env-file=docker/local.env pull && \\
-              # temporary solution to recreate containers when configs change: https://github.com/internetstandards/Internet.nl/issues/1490 \\
-              env -i RELEASE="\$RELEASE" DOCKER_REGISTRY="\$DOCKER_REGISTRY" docker compose --env-file=docker/defaults.env --env-file=docker/host.env --env-file=docker/local.env rm --stop --force cron-docker prometheus alertmanager nginx_logs_exporter && \\
-              env -i RELEASE="\$RELEASE" DOCKER_REGISTRY="\$DOCKER_REGISTRY" docker compose --env-file=docker/defaults.env --env-file=docker/host.env --env-file=docker/local.env up --remove-orphans --wait --no-build
+              docker run -ti --rm --pull=always \\
+                --volume /var/run/docker.sock:/var/run/docker.sock \\
+                --volume /opt/Internet.nl:/opt/Internet.nl \\
+                --network none \\
+                ${{ env.DOCKER_REGISTRY }}/util:${{ needs.build-docker.outputs.internetnl_version }} \\
+                /deploy.sh
           EOF
 
   integration-test:
@@ -315,7 +288,7 @@ jobs:
         # https://github.com/docker/login-action?tab=readme-ov-file#dockerhub
         uses: docker/login-action@v3
         with:
-          username: ${{ vars.DOCKERHUB_USERNAME }}
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
         # worst case of this failing is running into Docker Hub rate limit
         continue-on-error: true
@@ -413,7 +386,7 @@ jobs:
         # https://github.com/docker/login-action?tab=readme-ov-file#dockerhub
         uses: docker/login-action@v3
         with:
-          username: ${{ vars.DOCKERHUB_USERNAME }}
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
         # worst case of this failing is running into Docker Hub rate limit
         continue-on-error: true
@@ -482,7 +455,7 @@ jobs:
         # https://github.com/docker/login-action?tab=readme-ov-file#dockerhub
         uses: docker/login-action@v3
         with:
-          username: ${{ vars.DOCKERHUB_USERNAME }}
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
         # worst case of this failing is running into Docker Hub rate limit
         continue-on-error: true
@@ -565,7 +538,7 @@ jobs:
         # https://github.com/docker/login-action?tab=readme-ov-file#dockerhub
         uses: docker/login-action@v3
         with:
-          username: ${{ vars.DOCKERHUB_USERNAME }}
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
         # worst case of this failing is running into Docker Hub rate limit
         continue-on-error: true
@@ -685,7 +658,7 @@ jobs:
         # https://github.com/docker/login-action?tab=readme-ov-file#dockerhub
         uses: docker/login-action@v3
         with:
-          username: ${{ vars.DOCKERHUB_USERNAME }}
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
         # worst case of this failing is running into Docker Hub rate limit
         continue-on-error: true

Makefile

@@ -12,7 +12,7 @@ CERTSSDIR=$(REMOTEDATADIR)/certs
 DNSDIR=$(REMOTEDATADIR)/dns
 
 # default version if nothing is provided by environment
-INTERNETNL_VERSION ?= 0.0.0-dev0
+RELEASE ?= 0.0.0-dev0
 
 ifeq ($(shell uname -m),arm64)
 _env = env PATH="${bin}:$$PATH /usr/bin/arch -x86_64"
@@ -400,7 +400,15 @@ endif
 
 # allow overriding settings
 ifneq (,$(wildcard docker/local.env))
-	localenv=--env-file=docker/local.env
+$(info )
+$(info A `docker/local.env` exists which may override default behaviour!)
+$(info )
+$(info File contents:)
+$(info )
+$(info $(shell cat docker/local.env))
+$(info )
+
+localenv =--env-file=docker/local.env
 endif
 
 # command used to bring projects up
@@ -418,7 +426,7 @@ DOCKER_COMPOSE_BUILD_CMD=docker compose ${compose_args} \
 	--env-file=docker/build.env
 
 build docker-compose-build:
-	${DOCKER_COMPOSE_BUILD_CMD} build ${build_args} --build-arg=INTERNETNL_VERSION=${INTERNETNL_VERSION} ${services}
+	${DOCKER_COMPOSE_BUILD_CMD} build ${build_args} --build-arg=RELEASE=${RELEASE} ${services}
 
 build-no-cache docker-compose-build-no-cache: build
 build-no-cache docker-compose-build-no-cache: build_args=--no-cache
@@ -439,7 +447,7 @@ restart docker-compose-restart:
 	${DOCKER_COMPOSE_CMD} restart --no-deps ${services}
 
 docker-compose-up-build-no-deps:
-	${DOCKER_COMPOSE_UP_PULL_CMD} up --wait --build --no-deps --build-arg=INTERNETNL_VERSION=${INTERNETNL_VERSION} ${services}
+	${DOCKER_COMPOSE_UP_PULL_CMD} up --wait --build --no-deps --build-arg=RELEASE=${RELEASE} ${services}
 
 up-no-deps:
 	${DOCKER_COMPOSE_UP_PULL_CMD} up --wait --no-deps ${services}

bin/lint.sh

@@ -8,7 +8,8 @@ fail=0
 
 pylama --skip "**/migrations/*" ${@} || fail=1
 black --line-length 120 --check ${@} || fail=1
+shellcheck -e SC1071 docker/cron/periodic/*/* || fail=1
 
 SKIP_SECRET_KEY_CHECK=True CACHE_LOCATION= ENABLE_BATCH= ./manage.py makemigrations --noinput --check --dry-run || fail=1
 
-exit "$fail"
+exit "$fail"

docker/Dockerfile

@@ -10,7 +10,7 @@ ARG PYTHON_VERSION=3.9
 # hashes/log output prefix build command with: `DOCKER_BUILDKIT=0`
 
 # generic stage used by all build stages
-FROM debian:bullseye-20230502-slim as build-deps
+FROM debian:bullseye-20230502-slim AS build-deps
 ARG PYTHON_VERSION
 
 RUN apt update && \
@@ -33,7 +33,7 @@ RUN apt update && \
       flex \
       bison
 
-FROM build-deps as build-unbound
+FROM build-deps AS build-unbound
 ARG PYTHON_VERSION
 
 COPY vendor/unbound /src/vendor/unbound
@@ -48,7 +48,7 @@ RUN ./configure \
 RUN make
 RUN make install
 
-FROM build-deps as build-nassl
+FROM build-deps AS build-nassl
 
 COPY vendor/nassl /src/vendor/nassl
 WORKDIR /src/vendor/nassl
@@ -61,7 +61,7 @@ RUN python3 build_from_scratch.py
 RUN python3 setup.py install
 
 # intermediate stage with apt and python dependencies
-FROM build-deps as build-app-deps
+FROM build-deps AS build-app-deps
 
 COPY requirements.txt /src/requirements.txt
 
@@ -70,15 +70,15 @@ WORKDIR /src
 RUN pip3 install -r requirements.txt
 
 # stage with app dependencies and lint/test depencencies
-FROM build-app-deps as linttest-deps
+FROM build-app-deps AS linttest-deps
 ARG PYTHON_VERSION
 
 COPY requirements-dev.txt /src/requirements-dev.txt
 
 RUN pip3 install -r requirements-dev.txt
 
 # build unbound target
-FROM debian:bullseye-20230502-slim as unbound
+FROM debian:bullseye-20230502-slim AS unbound
 
 COPY --from=build-unbound /opt/unbound /opt/unbound
 
@@ -131,7 +131,7 @@ USER unbound
 ENTRYPOINT ["/entrypoint.sh"]
 
 # build main application image target
-FROM debian:bullseye-20230502-slim as build-app
+FROM debian:bullseye-20230502-slim AS build-app
 ARG PYTHON_VERSION
 
 RUN apt update && \
@@ -215,17 +215,17 @@ RUN SKIP_SECRET_KEY_CHECK=True CACHE_LOCATION= ENABLE_BATCH= ./manage.py compile
 RUN SKIP_SECRET_KEY_CHECK=True CACHE_LOCATION= ENABLE_BATCH= ./manage.py collectstatic --no-input
 RUN SKIP_SECRET_KEY_CHECK=True CACHE_LOCATION= ENABLE_BATCH= ./manage.py api_generate_doc
 
-FROM build-app as app
+FROM build-app AS app
 
-ARG INTERNETNL_VERSION
-ENV SETUPTOOLS_SCM_PRETEND_VERSION=$INTERNETNL_VERSION
+ARG RELEASE
+ENV SETUPTOOLS_SCM_PRETEND_VERSION=$RELEASE
 
 # define default entrypoint and command
 ENTRYPOINT [ "python3", "./manage.py"]
 CMD ["runserver", "0.0.0.0:8080"]
 
 # supplement application with unittest, lint dependencies
-FROM build-app as linttest
+FROM build-app AS linttest
 ARG PYTHON_VERSION
 
 USER root
@@ -236,6 +236,7 @@ RUN apt update && \
     make \
     python3-pip \
     docker.io \
+    shellcheck \
     # since this stage ends up in the final image we care about size and remove cache files
     && rm -rf /var/lib/apt/lists/*
 

docker/cron-docker/periodic/15min/auto_update

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+set -e
+
+if [ -z "$AUTO_UPDATE_TO" ];then
+  # auto update not configured
+  exit 0
+fi
+
+cd /opt/Internet.nl/
+
+CURRENT_RELEASE="$RELEASE"
+
+IMAGE="${DOCKER_REGISTRY:-ghcr.io/internetstandards}/util:${AUTO_UPDATE_TO}"
+
+# pull latest image
+docker pull "$IMAGE"
+
+# get release info from image
+UPSTREAM_RELEASE="$(docker image inspect "$IMAGE" | jq -r '.[0].Config.Labels.release')"
+
+# check if update is required
+if [ "$CURRENT_RELEASE" = "$UPSTREAM_RELEASE" ];then
+  echo "No update available"
+  exit 0
+fi
+
+echo "Updating to: $UPSTREAM_RELEASE"
+
+# initiate update, don't remove container when done to preserve logging
+# as the initiator (cron-docker) will be killed during the deploy process
+docker run --pull=always --network none \
+  --volume /var/run/docker.sock:/var/run/docker.sock \
+  --volume /opt/Internet.nl:/opt/Internet.nl \
+  "ghcr.io/internetstandards/util:$UPSTREAM_RELEASE" \
+  /deploy.sh

docker/cron-docker/periodic/15min/restart_nassl_worker

@@ -0,0 +1,4 @@
+#!/bin/sh
+set -e
+# find nassl worker and restart the container(s)
+docker ps --filter label=com.docker.compose.service=worker-nassl --quiet |  xargs --no-run-if-empty docker restart

docker/cron-docker/periodic/daily/restart_slow_worker

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+# restart slow worker every day to prevent slow memory leaks
+# as the slow worker can run very long tasks (eg: report generation)
+# we first start  a new container before stopping the previous one
+
+set -e
+
+cd /opt/Internet.nl
+
+SERVICE=worker-slow
+REPLICAS=$WORKER_SLOW_REPLICAS
+COMPOSE_CMD="docker compose --env-file=docker/defaults.env --env-file=docker/host.env --env-file=docker/local.env"
+
+OLD_CONTAINERS=$($COMPOSE_CMD ps --format "{{ .Name }}"|grep "$SERVICE")
+
+# bring up new containers, wait until healthy
+$COMPOSE_CMD up --no-deps --no-recreate --wait --scale="$SERVICE=$(($REPLICAS*2))" "$SERVICE"
+
+# graceful shutdown and remove old containers
+docker rm --force "$OLD_CONTAINERS"
+
+# restore replica number to original
+$COMPOSE_CMD scale $SERVICE=$REPLICAS

docker/cron-docker/periodic/hourly/restart_worker

@@ -0,0 +1,4 @@
+#!/bin/sh
+set -e
+# find worker and restart the container(s)
+docker ps --filter label=com.docker.compose.service=worker --quiet | xargs --no-run-if-empty docker restart

docker/cron-docker/periodic/weekly/docker_system_prune

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+# remove unused images, stoped containers, unused networks and build cache
+# --all to remove images not used by containers, --force to skip confirm prompt
+docker system prune --all --force