From 90170833a751276418aab5af176dfade8c56de70 Mon Sep 17 00:00:00 2001 From: Andreas Resch Date: Tue, 26 Dec 2023 17:25:41 +0100 Subject: [PATCH] fix ci (#50) * try to fix ci * copy deployment directory * add stuff * create directory * fix ssh command --- .github/workflows/build-and-push.yaml | 32 ++++++++++++++++++++++--- .github/workflows/deploy-production.yml | 2 +- .github/workflows/deploy-test.yaml | 2 +- .github/workflows/deploy.yaml | 13 ++++++++++ cli/generators/jenkins.py | 2 +- deployment/aeolus-deployment | 3 +-- deployment/aeolus-docker | 23 ++++++++++++------ deployment/traefik/docker-compose.yml | 8 +++---- deployment/traefik/traefik.yaml | 15 +++++------- 9 files changed, 72 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build-and-push.yaml b/.github/workflows/build-and-push.yaml index faa3e10e..1e48f2c8 100644 --- a/.github/workflows/build-and-push.yaml +++ b/.github/workflows/build-and-push.yaml @@ -9,6 +9,8 @@ on: paths-ignore: - "README.md" - ".github/workflows/python-**.yaml" + - ".github/workflows/deploy-**.yaml" + - ".github/workflows/json-schema-generation.yaml" env: REGISTRY: ghcr.io @@ -40,6 +42,7 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -53,6 +56,27 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Compute Tag + uses: actions/github-script@v7 + id: compute-tag + with: + result-encoding: string + script: | + if (context.eventName === "pull_request") { + return "pr-" + context.issue.number; + } + if (context.eventName === "release") { + return "latest"; + } + if (context.eventName === "push") { + if (context.ref.startsWith("refs/tags/")) { + return context.ref.slice(10); + } + if (context.ref === "refs/heads/develop") { + return "nightly"; + } + } + return "FALSE"; - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v4 @@ -63,15 +87,17 @@ jobs: type=raw,value={{date 'YYYYMMDD-hhmmss' tz='Europe/Berlin'}} type=ref,event=pr type=sha + type=raw,value={{steps.compute-tag.outputs.result}} type=raw,value=nightly,enable={{is_default_branch}} - - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . file: ${{ matrix.dockerfile }} platforms: linux/amd64,linux/arm64 - push: ${{ github.ref_name == 'develop' }} + push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=min diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index f8667166..8623c076 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -18,6 +18,6 @@ jobs: with: docker-tag: latest branch-name: develop - environment-name: Aeolus Production + environment-name: aeolus-prod environment-url: https://aeolus.artemis.cit.tum.de secrets: inherit \ No newline at end of file diff --git a/.github/workflows/deploy-test.yaml b/.github/workflows/deploy-test.yaml index bb076dc4..4a0c44a9 100644 --- a/.github/workflows/deploy-test.yaml +++ b/.github/workflows/deploy-test.yaml @@ -155,7 +155,7 @@ jobs: with: docker-tag: ${{ needs.compute-tag.outputs.tag }} branch-name: ${{ github.event.pull_request.head.ref }} - environment-name: aeolus test + environment-name: aeolus-dev environment-url: https://aeolus-test.artemis.cit.tum.de secrets: inherit diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 60f6ec77..edaeaab2 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -49,6 +49,7 @@ jobs: - name: Fetch Aeolus Deployment CLI run: | wget ${{ env.RAW_URL }}/deployment/aeolus-deployment + wget ${{ env.RAW_URL }}/deployment/aeolus-docker chmod +x aeolus-deployment # Configure SSH Key @@ -65,6 +66,18 @@ jobs: cat - <<< $GATEWAY_HOST_PUBLIC_KEY >> ~/.ssh/known_hosts cat - <<< $(sed 's/\\n/\n/g' <<< "$DEPLOYMENT_HOST_PUBLIC_KEYS") >> ~/.ssh/known_hosts + - name: Prepare directory + env: + SSH_AUTH_SOCK: /tmp/ssh_agent.sock + run: | + ssh -J "$GATEWAY_USER@$GATEWAY_HOST" "$DEPLOYMENT_USER@$DEPLOYMENT_HOST" "mkdir -p $DEPLOYMENT_DIRECTORY" + + - name: Copy deployment scripts + env: + SSH_AUTH_SOCK: /tmp/ssh_agent.sock + run: | + scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=~/.ssh/known_hosts -o ProxyJump="$GATEWAY_USER@$GATEWAY_HOST" aeolus-docker "$DEPLOYMENT_USER@$DEPLOYMENT_HOST:$DEPLOYMENT_DIRECTORY" + - name: Deploy Aeolus with Docker env: SSH_AUTH_SOCK: /tmp/ssh_agent.sock diff --git a/cli/generators/jenkins.py b/cli/generators/jenkins.py index 821ffbf0..98b97d6f 100644 --- a/cli/generators/jenkins.py +++ b/cli/generators/jenkins.py @@ -131,7 +131,7 @@ def handle_always_step(self, name: str, step: ScriptAction, indentation: int = 4 self.add_script( wrapper="post", name=name, - original_type=original_type, + original_type="custom", script=script, indentation=indentation, workdir=None, diff --git a/deployment/aeolus-deployment b/deployment/aeolus-deployment index a60afd12..b2d0742f 100644 --- a/deployment/aeolus-deployment +++ b/deployment/aeolus-deployment @@ -48,8 +48,7 @@ function docker_deploy { ssh -J "$_gateway_host" -o "StrictHostKeyChecking=no" "$_deployment_host" << COMMAND mkdir -p $_deployment_directory -cd $_deployment_directory -sudo /usr/bin/bash $_deployment_directory/pyris-docker.sh restart $3 $4 +/usr/bin/bash $_deployment_directory/aeolus-docker restart $_pr_tag $_pr_branch COMMAND } diff --git a/deployment/aeolus-docker b/deployment/aeolus-docker index 89302741..1aebc8b3 100644 --- a/deployment/aeolus-docker +++ b/deployment/aeolus-docker @@ -3,7 +3,8 @@ # Adapted from https://github.com/ls1intum/Pyris # this script is used to start the Aeolus docker containers, and must be available on the deployment server -PROJECT_DIR="/opt/aeolus/deployment" +WORKING_DIR="/opt/aeolus" +PROJECT_DIR="/opt/aeolus/Aeolus/deployment" COMPOSE_FILE="traefik/docker-compose.yml" ENV_FILE="/opt/aeolus/docker.env" @@ -26,17 +27,25 @@ function start { local pr_branch=$2 echo "Starting aeolus with PR tag: $pr_tag and branch: $pr_branch" - rm -rf Pyris + rm -rf Aeolus + cd $WORKING_DIR || exit git clone https://github.com/ls1intum/Aeolus.git -b "$pr_branch" Aeolus sed -i "s/AEOLUS_DOCKER_TAG=.*/AEOLUS_DOCKER_TAG='$pr_tag'/g" $ENV_FILE - docker compose --project-directory "$PROJECT_DIR" -f "$PROJECT_DIR/$COMPOSE_FILE" --env-file "$ENV_FILE" up -d --pull always --no-build + if [ ! -f "$WORKING_DIR/acme.json" ]; then + touch "$WORKING_DIR/acme.json" + chmod 600 "$WORKING_DIR/acme.json" + fi + if [ ! -f "$WORKING_DIR/traefik.log" ]; then + touch "$WORKING_DIR/traefik.log" + fi + docker compose -f "$PROJECT_DIR/$COMPOSE_FILE" --env-file "$ENV_FILE" up -d --pull always --no-build } function stop { # TODO: In the future extract pr_tag and pr_branch from env echo "Stopping aeolus" - docker compose --project-directory "$PROJECT_DIR" -f "$PROJECT_DIR/$COMPOSE_FILE" --env-file "$ENV_FILE" stop pyris-app + docker compose -f "$PROJECT_DIR/$COMPOSE_FILE" --env-file "$ENV_FILE" down } function restart { @@ -45,15 +54,15 @@ function restart { } function aeolus_logs { - docker compose --project-directory "$PROJECT_DIR" -f "$PROJECT_DIR/$COMPOSE_FILE" --env-file "$ENV_FILE" logs -f aeolus-api + docker compose -f "$PROJECT_DIR/$COMPOSE_FILE" --env-file "$ENV_FILE" logs -f aeolus-api } function all_logs { - docker compose --project-directory "$PROJECT_DIR" -f "$PROJECT_DIR/$COMPOSE_FILE" --env-file "$ENV_FILE" logs -f + docker compose -f "$PROJECT_DIR/$COMPOSE_FILE" --env-file "$ENV_FILE" logs -f } function run_docker_compose_cmd { - docker compose --project-directory "$PROJECT_DIR" -f "$PROJECT_DIR/$COMPOSE_FILE" --env-file "$ENV_FILE" "$@" + docker compose -f "$PROJECT_DIR/$COMPOSE_FILE" --env-file "$ENV_FILE" "$@" } # read subcommand `aeolus-docker subcommand server` in variable and remove base command from argument list diff --git a/deployment/traefik/docker-compose.yml b/deployment/traefik/docker-compose.yml index f70d7a7f..2bb88ad4 100644 --- a/deployment/traefik/docker-compose.yml +++ b/deployment/traefik/docker-compose.yml @@ -10,7 +10,7 @@ services: labels: - "traefik.enable=true" - "traefik.docker.network=traefik" - - 'traefik.http.routers.aeolus-playground.rule=(HostRegexp(`{host:.+}`) && !PathPrefix("/api"))' + - 'traefik.http.routers.aeolus-playground.rule=(Host(`${AEOLUS_HOST}`) && !PathPrefix("/api"))' - "traefik.http.routers.aeolus-playground.entrypoints=https" - "traefik.http.routers.aeolus-playground.tls=true" - "traefik.http.routers.aeolus-playground.tls.certResolver=leresolver" @@ -26,7 +26,7 @@ services: labels: - "traefik.enable=true" - "traefik.docker.network=traefik" - - 'traefik.http.routers.aeolus-api.rule=(HostRegexp(`{host:.+}`) && PathPrefix("/api"))' + - 'traefik.http.routers.aeolus-api.rule=(Host(`${AEOLUS_HOST}`) && PathPrefix("/api"))' - "traefik.http.routers.aeolus-api.entrypoints=https" - "traefik.http.routers.aeolus-api.tls=true" - "traefik.http.routers.aeolus-api.tls.certResolver=leresolver" @@ -50,8 +50,8 @@ services: - ./traefik.yaml:/traefik.yaml # Configuration for the file provider (needed for host networking and default TLS Options) - ./traefik-provider.yaml:/traefik-provider.yaml - - ./acme.json:/acme.json - - ./traefik.log:/traefik.log + - ../../../acme.json:/acme.json + - ../../../traefik.log:/traefik.log labels: - 'traefik.enable=false' - "traefik.docker.network=traefik" diff --git a/deployment/traefik/traefik.yaml b/deployment/traefik/traefik.yaml index 1e452844..9ab3dcab 100644 --- a/deployment/traefik/traefik.yaml +++ b/deployment/traefik/traefik.yaml @@ -1,14 +1,17 @@ log: level: DEBUG filePath: /traefik.log -docker: - watch: true - exposedbydefault: false providers: docker: endpoint: unix:///var/run/docker.sock file: filename: /traefik-provider.yaml +certificatesResolvers: + leresolver: + acme: + email: letsencrypt@resch.io + storage: acme.json + tlsChallenge: {} entryPoints: http: address: :80/tcp @@ -22,10 +25,4 @@ entryPoints: http: tls: certResolver: leresolver -certificatesResolvers: - leresolver: - acme: - email: letsencrypt@resch.io - storage: acme.json - tlsChallenge: {} retry: {} \ No newline at end of file