From f4269412e3fe897cdbd2a1c52d6fd7074b581374 Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 6 Feb 2024 16:18:02 +0100 Subject: [PATCH] change backend endpoint in openapi --- .env.example | 7 ++ Makefile | 11 +++ README.md | 147 ++++++++++++++++++++++++++++++++++++++++- docker-compose.dev.yml | 64 ++++++++++++++++++ docker-compose.yml | 133 +++++++++++++++++++++++++++++++++++++ frontend/openapi.json | 2 +- makefile.js | 60 +++++++++++++++++ 7 files changed, 422 insertions(+), 2 deletions(-) create mode 100644 .env.example create mode 100644 Makefile create mode 100644 docker-compose.dev.yml create mode 100644 docker-compose.yml create mode 100644 makefile.js diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..26d9419 --- /dev/null +++ b/.env.example @@ -0,0 +1,7 @@ +CONTAINER_REGISTRY_TOKEN=insert_token_here +DOCKER_SERVICES="rabbitmq scanner database backend frontend" +RABBITMQ_USER=barrierelos +RABBITMQ_PASSWORD=barrierelos +RABBITMQ_PORT=5050 +POSTGRES_USER=barrierelos +POSTGRES_PASSWORD=barrierelos diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..7133d77 --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +docker-login: + (. $(PWD)/.env ; echo "$$CONTAINER_REGISTRY_TOKEN" | docker login registry.gitlab.ost.ch:45023 -u doesntmatter --password-stdin) + +docker-pull: + (. $(PWD)/.env ; docker compose -f compose.yml -f compose-dev.yml pull $$DOCKER_SERVICES) + +docker-up: + (. $(PWD)/.env ; docker compose -f compose.yml -f compose-dev.yml up -d $$DOCKER_SERVICES) + +docker-down: + docker compose -f compose.yml -f compose-dev.yml down diff --git a/README.md b/README.md index d819c34..6d8874e 100644 --- a/README.md +++ b/README.md @@ -1 +1,146 @@ -# Barrierelos +# Barrierelos Deployment + +This repo includes the deployment configuration for the Barrierelos project. +That includes both the logic to deploy to the server and to run specific components locally in Docker. +Running specific components locally in Docker is useful for development when you want to work on a single component and +start the other components locally in Docker. We were developing on Linux and as such, some scripts might not work on +Windows or macOS. MacOS should technically work, but we didn't test it. + +If you're interested in the architecture of the Barrierelos project, decision records, and other documentation, please +refer to the [project documentation](http://barrierelos.pages.gitlab.ost.ch/documentation/index.html) on GitLab Pages. + +## Repositories + +The Barrierelos project consists of the three primary repos: + +- [scanner](https://gitlab.ost.ch/barrierelos/scanner): + The Typescript / Node.js application that runs the Axe-core a11y scan jobs submitted by the backend via RabbitMQ. +- [backend](https://gitlab.ost.ch/barrierelos/backend): + The Kotlin / Spring backend API used by the frontend, which also submits Axe-core a11y scan jobs to the scanner via + RabbitMQ. +- [frontend](https://gitlab.ost.ch/barrierelos/frontend): + The Typescript / React frontend which uses client-side-rendering (CSR). + +The Barrierelos project also includes the following additional repos: + +- [documentation](https://gitlab.ost.ch/barrierelos/documentation): + The ReStructuredText / Sphinx documentation for the Barrierelos project which is published to GitLab Pages. +- [tools/scripts](https://gitlab.ost.ch/barrierelos/tools/scripts): + Contains some helper scripts to add example websites via the backend API and to scrape information we needed while + developing the project. +- [tools/bruno](https://gitlab.ost.ch/barrierelos/tools/bruno): + Contains the [Bruno](https://www.usebruno.com/) configuration. Bruno is an open-source API tool similar to Postman. + +## File descriptions + +- `.gitlab-ci-docker-publish-deploy.yml`: + Includes the shared GitLab CI logic to build a Docker image and publish it to the GitLab registry. + This file is included in the `gitlab-ci.yml` files of the other repos. +- `compose.yml`: + The Docker Compose file to deploy the Barrierelos project on the server. +- `compose-dev.yml`: + This Docker Compose file provides overrides for the `compose.yml` to run the Barrierelos project locally in Docker. +- `Makefile`: + This Makefile provides commands to simplify running the Barrierelos project locally in Docker. +- `.env`: + This file contains the [environment variables](#environment-variables) to configure how the Barrierelos project is run + locally in Docker. + +## Environment Variables + +The environment variables in `.env` are used to configure how the Barrierelos project is run locally in Docker. + +- `CONTAINER_REGISTRY_TOKEN`: + The token to log in to the GitLab container registry via `make docker-login`. + A new Group Access Token for this purpose can be + created [here](https://gitlab.ost.ch/groups/barrierelos/-/settings/access_tokens). +- `DOCKER_SERVICES`: + The Docker services to run locally in Docker via `make docker-up`. + If you want to work on the backend for example, you can remove the `backend` from the list so that you can launch it + locally outside of Docker in your IDE. + +The other environment variables are used in the Docker Compose files to configure credentials and ports of the services. + +## Run the project locally in Docker + +To run the Barrierelos project locally in Docker, you need to have Docker and [make](https://www.gnu.org/software/make/) +installed. + +> Make sure that you have configured [environment variables](#environment-variables) in the `.env` file as desired +> before +> running the commands below. + +In order for the Docker Compose file to access the images in the GitLab registry, +you need to log in to the GitLab registry. This step is only required once. + +> Note: If you have never logged in to a Docker registry before, you may first need to configure a Docker +> [credential store](https://docs.docker.com/engine/reference/commandline/login/#credential-stores). + +```bash +make docker-login +``` + +To pull the latest images from the GitLab registry, run: + +```bash +make docker-pull +``` + +> Note: This will only pull the images that are listed in the `DOCKER_SERVICES` environment variable. +> This is useful for example if you don't want to pull the most recent `scanner` image yet because your +> `backend` branch is not compatible with it yet. + +To start the Barrierelos project locally in Docker, run: + +```bash +make docker-up +``` + +To stop the Barrierelos project locally in Docker, run: + +> Note: This command will also remove all containers, networks, volumes, and images created by `make docker-up`. +> This includes the database and all data stored in it. + +```bash +make docker-down +``` + +## Accessing the RabbitMQ web interface + +After starting the `rabbitmq` service locally in Docker, you can access the RabbitMQ web interface at the following URL: +http://localhost:15672/. The default username and password are both `barrierelos`. + +The `scanner` and the `backend` will automatically create the required queues and exchanges. +You can use the web interface to manually insert website scan jobs or results into the queues. As well as to inspect the +messages in the queues. And lastly you can purge all messages in the queues in case some of them remained there during +development. + +## Accessing the PostgreSQL database + +The [backend](https://gitlab.ost.ch/barrierelos/backend) project pre-configured data sources to access the local +database and the one on the server. This is documented +[here](https://gitlab.ost.ch/barrierelos/backend/-/blob/main/README.md#accessing-the-postgresql-database). + +## GitLab CI/CD Pipeline + +Whenever you push a commit to any branch of one of the primary [repositories](#repositories), a GitLab CI pipeline is +triggered. Each pipeline consists of multiple jobs that accomplish the same task but for the different technologies used +in these components. The procedure goes as follows: + +- Build the application +- Test / lint the application +- Build a Docker image and publish it to the GitLab registry +- Deploy the image to the server + +The first two steps are implemented in the `.gitlab-ci.yml` file of the corresponding repository. The last two steps are +shared between all primary repositories and is therefore implemented in this repository, in the +`.gitlab-ci-docker-publish-deploy.yml` file. The two last steps are only executed automatically when pushing to +the `main` branch. When pushing a commit to another branch the auto-build and -deploy jobs are still available but must +be triggered manually, since commits to feature branches should not be deployed to the server automatically. + +Since the different primary repos depend on each other, you can also disable the auto-build and -deploy pipelines on +the `main` branch by including the string `no_auto_deploy` in the commit message. This way you can for example finish +merging all merge requests in different repos before triggering the auto-build and -deploy jobs. + +The `documentation` repo also has an auto-build and -deploy pipeline, which functions similar to the pipelines described +above. It builds the documentation as HTML and PDF and publishes it to GitLab Pages. diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..56a59cf --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,64 @@ +# This file is used for development only and cloned into the local repositories +# It extends the production configuration additional configuration and reset values + +services: + rabbitmq: + volumes: !reset [] + networks: + - default + ports: + - "${RABBITMQ_PORT:-5672}:5672" + environment: + - RABBITMQ_DEFAULT_USER=${RABBITMQ_USER:-barrierelos} + - RABBITMQ_DEFAULT_PASS=${RABBITMQ_PASSWORD:-barrierelos} + + scanner: + image: !reset + build: + context: scanner + dockerfile: ./Dockerfile + networks: + - default + environment: + - RABBITMQ_HOSTNAME=${RABBITMQ_HOSTNAME:-barrierelos-rabbitmq} + - RABBITMQ_PORT=5672 + - RABBITMQ_USER=${RABBITMQ_USER:-barrierelos} + - RABBITMQ_PASSWORD=${RABBITMQ_PASSWORD:-barrierelos} + deploy: !reset {} + + + database: + volumes: !reset [] + networks: + - default + ports: + - "5040:5432" + + backend: + image: !reset + build: + context: backend + dockerfile: ./Dockerfile + networks: + - default + ports: + - "8030:8030" + + frontend: + image: !reset + build: + context: frontend + dockerfile: ./Dockerfile + depends_on: !reset [] + networks: !reset + - hostnet + extra_hosts: + # Enables using "host.docker.internal" to access the host machine + - "host.docker.internal:host-gateway" + ports: + - "5000:5000" + +networks: + hostnet: + external: true + name: host diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..610c90a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,133 @@ +version: "3.8" +# name: barrierelos + +services: + proxy: + image: geometalab/env-configurable-caddy:latest + networks: + - web + - default + environment: + CADDY_CONFIG: | + :8080 { + route { + handle /api/* { + uri strip_prefix /api + reverse_proxy backend:8030 + } + redir /api /api/ + reverse_proxy /* { + to frontend:5000 + } + } + } + deploy: + labels: + # caddy: barrierelos.projects-ost.ch + caddy: barrierelos.sifs0003.infs.ch + caddy.reverse_proxy: "{{upstreams 8080}}" + restart_policy: + condition: "on-failure" + + rabbitmq: + container_name: barrierelos-rabbitmq + image: rabbitmq:3-management + restart: always + volumes: + - rabbitmq-data-dir:/var/lib/rabbitmq/mnesia/ + networks: + - default + hostname: rabbitmq + # expose: + # - 5672 + # ports: + # - "15672:15672" + environment: + - RABBITMQ_DEFAULT_USER=$RABBITMQ_USER + - RABBITMQ_DEFAULT_PASS=$RABBITMQ_PASSWORD + healthcheck: + test: rabbitmq-diagnostics -q ping + interval: 10s + timeout: 3s + retries: 3 + + scanner: + container_name: barrierelos-scanner + image: $CONTAINER_REGISTRY_PATH/scanner:latest + restart: always + depends_on: + - rabbitmq + networks: + - default + environment: + - NODE_ENV=production + - RABBITMQ_HOSTNAME=rabbitmq + - RABBITMQ_PORT=5672 + - RABBITMQ_USER=$RABBITMQ_USER + - RABBITMQ_PASSWORD=$RABBITMQ_PASSWORD + deploy: + resources: + limits: + cpus: "1" + + database: + container_name: barrierelos-database + image: postgres:16.0 + restart: always + # ports: + # - "5432:5432" + environment: + - POSTGRES_DB=barrierelos-db + - POSTGRES_USER=$POSTGRES_USER + - POSTGRES_PASSWORD=$POSTGRES_PASSWORD + volumes: + - barrierelos-data:/var/lib/postgresql/data/ + networks: + - default + healthcheck: + test: pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER} + interval: 10s + timeout: 3s + retries: 3 + + backend: + container_name: barrierelos-backend + image: $CONTAINER_REGISTRY_PATH/backend:latest + restart: always + # expose: + # - 8030 + # ports: + # - "40001:8030" + networks: + - default + environment: + - RABBITMQ_USER=$RABBITMQ_USER + - RABBITMQ_PASSWORD=$RABBITMQ_PASSWORD + - SPRING_RABBITMQ_HOST=rabbitmq + - SPRING_RABBITMQ_PORT=5672 + - POSTGRES_USER=$POSTGRES_USER + - POSTGRES_PASSWORD=$POSTGRES_PASSWORD + - SPRING_DATASOURCE_URL=jdbc:postgresql://database:5432/barrierelos-db?stringtype=unspecified + depends_on: + - database + - rabbitmq + + frontend: + container_name: barrierelos-frontend + image: $CONTAINER_REGISTRY_PATH/frontend:latest + restart: always + networks: + - default + # ports: + # - "40000:5000" + depends_on: + - backend + +volumes: + rabbitmq-data-dir: + barrierelos-data: + +networks: + web: + external: true + name: web \ No newline at end of file diff --git a/frontend/openapi.json b/frontend/openapi.json index 40f48bc..f1cc2d4 100644 --- a/frontend/openapi.json +++ b/frontend/openapi.json @@ -7,7 +7,7 @@ }, "servers": [ { - "url": "http://barrierelos.ch:40001", + "url": "https://barrierelos.sifs0003.infs.ch/api", "description": "Generated server url" } ], diff --git a/makefile.js b/makefile.js new file mode 100644 index 0000000..223224b --- /dev/null +++ b/makefile.js @@ -0,0 +1,60 @@ +const { execSync } = require('child_process'); +const fs = require('fs'); + + +process.chdir(__dirname); + +const envPath = `.env`; +const envContent = fs.readFileSync(envPath, 'utf8'); + +function executeCommand(command) { + try { + execSync(command, {stdio: 'inherit'}); + } catch (error) { + console.error(`Error executing command: ${command}`); + process.exit(1); + } +} + +function dockerLogin() { + const containerRegistryToken = envContent.match(/CONTAINER_REGISTRY_TOKEN=(.*)/)[1].trim(); + const loginCommand = `echo | set /p="${containerRegistryToken}" | docker login registry.gitlab.ost.ch:45023 -u doesntmatter --password-stdin`; + executeCommand(loginCommand); +} + +function dockerPull() { + const dockerServices = envContent.match(/DOCKER_SERVICES=(.*)/)[1].trim().replace('"', ''); + const pullCommand = `docker compose -f compose.yml -f compose-dev.yml pull ${dockerServices}`; + executeCommand(pullCommand); +} + +function dockerUp() { + const dockerServices = envContent.match(/DOCKER_SERVICES=(.*)/)[1].trim().replace('"', ''); + const upCommand = `docker compose -f compose.yml -f compose-dev.yml up -d ${dockerServices}`; + executeCommand(upCommand); +} + +function dockerDown() { + executeCommand('docker compose -f compose.yml -f compose-dev.yml down'); +} + +// Run the appropriate command based on the script argument +const scriptArgument = process.argv[2]; + +switch (scriptArgument) { + case 'docker-login': + dockerLogin(); + break; + case 'docker-pull': + dockerPull(); + break; + case 'docker-up': + dockerUp(); + break; + case 'docker-down': + dockerDown(); + break; + default: + console.error('Invalid script argument. Use one of: docker-login, docker-pull, docker-up, docker-down'); + process.exit(1); +}