diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fded4f7376c..46ac384c5b7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,136 +1,615 @@ -name: Docker - -# initially we would us on: [release] as well, the problem is that -# the code in clowder would not know what branch the code is in, -# and would not set the right version flags. - -# This will run when: -# - when new code is pushed to master/develop to push the tags -# latest and develop -# - when a pull request is created and updated to make sure the -# Dockerfile is still valid. -# To be able to push to dockerhub, this expects the following -# secrets to be set in the project: -# - DOCKERHUB_USERNAME : username that can push to the org -# - DOCKERHUB_PASSWORD : password asscoaited with the username +name: Docker GHA + on: push: branches: + - main - master - develop + tags: + - "v*.*.*" pull_request: - merge_group: - issue_comment: - types: - - created workflow_dispatch: + inputs: + r_version: + description: 'R version to use' + required: true + type: choice + default: "4.1" + options: + - 4.1 + - 4.2 + - 4.3 + - 4.4 + - devel -# Certain actions will only run when this is the master repo. +# set up the environment, either input or default env: - MASTER_REPO: PecanProject/pecan - DOCKERHUB_ORG: pecan - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + R_VERSION: ${{ github.event.inputs.r_version || '4.1' }} +# there are 3 jobs to build different images jobs: - docker: - if: github.event_name != 'issue_comment' || startsWith(github.event.comment.body, '/build') + # ---------------------------------------------------------------------- + # depends image has all the dependencies installed + # ---------------------------------------------------------------------- + depends: runs-on: ubuntu-latest + permissions: + packages: write steps: - - name: Work around https://github.com/actions/checkout/issues/766 - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - uses: actions/checkout@v4 + + # create metadata for image + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 with: - set-safe-directory: false + # list of Docker images to use as base name for tags + images: | + pecan/depends + ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/depends + # generate Docker tags based on the following events/attributes + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + # setup docker build + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + id: buildx uses: docker/setup-buildx-action@v3 + + - name: Inspect Builder + run: | + echo "Name: ${{ steps.buildx.outputs.name }}" + echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" + echo "Status: ${{ steps.buildx.outputs.status }}" + echo "Flags: ${{ steps.buildx.outputs.flags }}" + echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + + # login to registries + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # build the docker images + - name: Build and push depends + uses: docker/build-push-action@v6 + with: + context: docker/depends + file: docker/depends/Dockerfile + push: true + platforms: "linux/amd64" + cache-from: type=registry,ref=pecan/depends:buildcache + cache-to: type=registry,ref=pecan/depends:buildcache,mode=max + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ steps.meta.outputs.version }} + R_VERSION=${{ env.R_VERSION }} + GITHUB_PAT=${{ secrets.GITHUB_TOKEN }} + + # ---------------------------------------------------------------------- + # base image has PEcAn compiled and installed, and depends on depends + # ---------------------------------------------------------------------- + base: + runs-on: ubuntu-latest + needs: depends + permissions: + packages: write + + steps: + - uses: actions/checkout@v4 + + # create metadata for image + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 with: - driver: docker - buildkitd-flags: --debug - install: true + # list of Docker images to use as base name for tags + images: | + pecan/base + ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/base + # generate Docker tags based on the following events/attributes + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + # setup docker build + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 - # calculate some variables that are used later - - name: get version tag + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Inspect Builder run: | - BRANCH=${GITHUB_REF##*/} - echo "GITHUB_BRANCH=${BRANCH}" >> $GITHUB_ENV - if [ "$BRANCH" == "master" ]; then - version="$(awk '/Version:/ { print $2 }' base/all/DESCRIPTION)" - tags="latest" - oldversion="" - while [ "${oldversion}" != "${version}" ]; do - oldversion="${version}" - tags="${tags},${version}" - version=${version%.*} - done - echo "PECAN_VERSION=$(awk '/Version:/ { print $2 }' base/all/DESCRIPTION)" >> $GITHUB_ENV - echo "PECAN_TAGS=${tags}" >> $GITHUB_ENV - elif [ "$BRANCH" == "develop" ]; then - echo "PECAN_VERSION=develop" >> $GITHUB_ENV - echo "PECAN_TAGS=develop" >> $GITHUB_ENV - else - echo "PECAN_VERSION=develop" >> $GITHUB_ENV - echo "PECAN_TAGS=develop" >> $GITHUB_ENV - fi - # If any dependencies changed in this PR, try to update depends image - - name: check for changed dependencies - uses: dorny/paths-filter@v3 - id: findchanges - with: - filters: | - deps: - - docker/depends/** - - if: steps.findchanges.outputs.deps == 'true' - # NB: GITHUB_BASE_REF is only set on pull requests, - # so non-PR builds will find no existing tag and build fresh + echo "Name: ${{ steps.buildx.outputs.name }}" + echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" + echo "Status: ${{ steps.buildx.outputs.status }}" + echo "Flags: ${{ steps.buildx.outputs.flags }}" + echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + + # login to registries + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # build the docker images + - name: Build and push base + uses: docker/build-push-action@v6 + with: + context: . + file: docker/base/Dockerfile + push: true + platforms: "linux/amd64" + cache-from: type=registry,ref=pecan/base:buildcache + cache-to: type=registry,ref=pecan/base:buildcache,mode=max + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ steps.meta.outputs.version }} + R_VERSION=${{ env.R_VERSION }} + FROM_IMAGE=depends + IMAGE_VERSION=${{ steps.meta.outputs.version }} + GITHUB_PAT=${{ secrets.GITHUB_TOKEN }} + PECAN_VERSION=${{ steps.meta.outputs.version }} + PECAN_GIT_BRANCH= ${{ github.head_ref || github.ref_name }} + PECAN_GIT_CHECKSUM=${{ github.sha }} + PECAN_GIT_DATE=${{ github.event.repository.updated_at }} + +# ---------------------------------------------------------------------- +# models image has some python installed to run models, depends on base +# ---------------------------------------------------------------------- + models: + runs-on: ubuntu-latest + needs: base + permissions: + packages: write + + steps: + - uses: actions/checkout@v4 + + # create metadata for image + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + # list of Docker images to use as models name for tags + images: | + pecan/models + ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/models + # generate Docker tags based on the following events/attributes + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + # setup docker build + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Inspect Builder run: | - if $(docker manifest inspect pecan/depends:${GITHUB_BASE_REF##*/} > /dev/null 2>&1); then - echo "UPDATE_DEPENDS_FROM_TAG=${GITHUB_BASE_REF##*/}" >> $GITHUB_ENV - else - echo "BUILD_DEPENDS_FRESH=true" >> GITHUB_ENV - fi - env: - GITHUB_BASE_REF: ${{ github.base_ref }} - # use shell script to build, there is some complexity in this - - name: create images - run: ./docker.sh -i github - env: - PECAN_GIT_CHECKSUM: ${{ github.sha }} - PECAN_GIT_BRANCH: ${GITHUB_BRANCH} - VERSION: ${{ env.PECAN_VERSION }} - UPDATE_DEPENDS_FROM_TAG: ${{ env.UPDATE_DEPENDS_FROM_TAG }} - BUILD: ${{ env.BUILD_DEPENDS_FRESH }} - - # push all images to github - - name: Publish to GitHub - if: github.event_name == 'push' && github.repository == env.MASTER_REPO + echo "Name: ${{ steps.buildx.outputs.name }}" + echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" + echo "Status: ${{ steps.buildx.outputs.status }}" + echo "Flags: ${{ steps.buildx.outputs.flags }}" + echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + + # login to registries + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # build the docker images + - name: Build and push models + uses: docker/build-push-action@v6 + with: + context: docker/models + file: docker/models/Dockerfile + push: true + platforms: "linux/amd64" + cache-from: type=registry,ref=pecan/models:buildcache + cache-to: type=registry,ref=pecan/models:buildcache,mode=max + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ steps.meta.outputs.version }} + R_VERSION=${{ env.R_VERSION }} + FROM_IMAGE=depends + IMAGE_VERSION=${{ steps.meta.outputs.version }} + GITHUB_PAT=${{ secrets.GITHUB_TOKEN }} + PECAN_VERSION=${{ steps.meta.outputs.version }} + PECAN_GIT_BRANCH= ${{ github.head_ref || github.ref_name }} + PECAN_GIT_CHECKSUM=${{ github.sha }} + PECAN_GIT_DATE=${{ github.event.repository.updated_at }} + +# ---------------------------------------------------------------------- +# Next are images that have models installed +# ---------------------------------------------------------------------- + modelsbinary: + runs-on: ubuntu-latest + needs: models + permissions: + packages: write + strategy: + fail-fast: false + matrix: + name: + - basgra + - biocro + # - ed2_2.2.0 + - ed2_git + - maespa + - sipnet + include: + - name: basgra + CONTEXT: models/basgra + DOCKERFILE: models/basgra/Dockerfile + PLATFORM: "linux/amd64" + MODEL: basgra + VERSION: BASGRA_N_v1 + - name: biocro + CONTEXT: models/biocro + DOCKERFILE: models/biocro/Dockerfile + PLATFORM: "linux/amd64" + MODEL: biocro + VERSION: "0.95" + # - name: ed2_2.2.0 + # CONTEXT: models/ed + # DOCKERFILE: models/ed/Dockerfile + # PLATFORM: "linux/amd64" + # MODEL: ed2 + # VERSION: "2.2.0" + - name: ed2_git + CONTEXT: models/ed + DOCKERFILE: models/ed/Dockerfile + PLATFORM: "linux/amd64" + MODEL: ed2 + VERSION: "git" + - name: maespa + CONTEXT: models/maespa + DOCKERFILE: models/maespa/Dockerfile + PLATFORM: "linux/amd64" + MODEL: maespa + VERSION: "git" + - name: sipnet + CONTEXT: models/sipnet + DOCKERFILE: models/sipnet/Dockerfile + PLATFORM: "linux/amd64" + MODEL: sipnet + VERSION: "git" + + steps: + - uses: actions/checkout@v4 + + # lower case name for docker + - name: docker image name + id: lower + run: echo "image_name=$(echo model-${{ matrix.MODEL }}-${{ matrix.VERSION }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT + + # create metadata for image + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + # list of Docker images to use as base name for tags + images: | + pecan/${{ steps.lower.outputs.image_name }} + ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/${{ steps.lower.outputs.image_name }} + # generate Docker tags based on the following events/attributes + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + # setup docker build + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Inspect Builder run: | - echo "${INPUT_PASSWORD}" | docker login -u ${INPUT_USERNAME} --password-stdin ${INPUT_REGISTRY} - repo=$(echo ${{ github.repository_owner }} | tr 'A-Z' 'a-z') - for image in $(docker image ls pecan/*:github --format "{{ .Repository }}"); do - for v in ${PECAN_TAGS}; do - docker tag ${image}:github ${INPUT_REGISTRY}/${repo}/${image#pecan/}:${v} - docker push ${INPUT_REGISTRY}/${repo}/${image#pecan/}:${v} - done - done - docker logout - env: - INPUT_REGISTRY: ghcr.io - INPUT_USERNAME: ${{ secrets.GHCR_USERNAME }} - INPUT_PASSWORD: ${{ secrets.GHCR_PASSWORD }} - - # push all images to dockerhub - - name: Publish to DockerHub - if: github.event_name == 'push' && github.repository == env.MASTER_REPO + echo "Name: ${{ steps.buildx.outputs.name }}" + echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" + echo "Status: ${{ steps.buildx.outputs.status }}" + echo "Flags: ${{ steps.buildx.outputs.flags }}" + echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + + # login to registries + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: docker image ls + run: docker image ls + + # build the docker images + - name: Build and push ${{ matrix.MODEL }} ${{ matrix.VERSION }} + uses: docker/build-push-action@v6 + with: + context: ${{ matrix.CONTEXT }} + file: ${{ matrix.DOCKERFILE }} + push: true + platforms: ${{ matrix.PLATFORM }} + cache-from: type=registry,ref=pecan/${{ steps.lower.outputs.image_name }}:buildcache + cache-to: type=registry,ref=pecan/${{ steps.lower.outputs.image_name }}:buildcache,mode=max + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ steps.meta.outputs.version }} + R_VERSION=${{ env.R_VERSION }} + MODEL_VERSION=${{ matrix.VERSION }} + IMAGE_VERSION=${{ steps.meta.outputs.version }} + + # ---------------------------------------------------------------------- + # Next are images that depend on base image + # ---------------------------------------------------------------------- + baseplus: + runs-on: ubuntu-latest + needs: base + permissions: + packages: write + strategy: + fail-fast: false + matrix: + name: + - docs + - executor + - api + include: + - name: docs + CONTEXT: . + DOCKERFILE: docker/docs/Dockerfile + PLATFORM: "linux/amd64" + IMAGE: docs + - name: executor + CONTEXT: docker/executor + DOCKERFILE: docker/executor/Dockerfile + PLATFORM: "linux/amd64" + IMAGE: executor + - name: api + CONTEXT: apps/api + DOCKERFILE: apps/api/Dockerfile + PLATFORM: "linux/amd64" + IMAGE: api + + steps: + - uses: actions/checkout@v4 + + # create metadata for image + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + # list of Docker images to use as base name for tags + images: | + pecan/${{ matrix.IMAGE }} + ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/${{ matrix.IMAGE }} + # generate Docker tags based on the following events/attributes + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + # setup docker build + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Inspect Builder run: | - echo "${INPUT_PASSWORD}" | docker login -u ${INPUT_USERNAME} --password-stdin - for image in $(docker image ls pecan/*:github --format "{{ .Repository }}"); do - for v in ${PECAN_TAGS}; do - docker tag ${image}:github ${{ env.DOCKERHUB_ORG }}/${image#pecan/}:${v} - docker push ${{ env.DOCKERHUB_ORG }}/${image#pecan/}:${v} - done - done - docker logout - env: - INPUT_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - INPUT_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + echo "Name: ${{ steps.buildx.outputs.name }}" + echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" + echo "Status: ${{ steps.buildx.outputs.status }}" + echo "Flags: ${{ steps.buildx.outputs.flags }}" + echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + + # login to registries + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # build the docker images + - name: Build and push ${{ matrix.name }} + uses: docker/build-push-action@v6 + with: + context: ${{ matrix.CONTEXT }} + file: ${{ matrix.DOCKERFILE }} + push: true + platforms: ${{ matrix.PLATFORM }} + cache-from: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache + cache-to: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache,mode=max + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ steps.meta.outputs.version }} + R_VERSION=${{ env.R_VERSION }} + IMAGE_VERSION=${{ steps.meta.outputs.version }} + + # ---------------------------------------------------------------------- + # Next are images that do not depend on either depends or base image + # ---------------------------------------------------------------------- + extras: + runs-on: ubuntu-latest + permissions: + packages: write + strategy: + fail-fast: false + matrix: + name: + - web + - dbsync + - data + - monitor + - rstudio-nginx + include: + - name: web + CONTEXT: . + DOCKERFILE: docker/web/Dockerfile + PLATFORM: "linux/amd64,linux/arm64" + IMAGE: web + - name: dbsync + CONTEXT: . + DOCKERFILE: shiny/dbsync/Dockerfile + PLATFORM: "linux/amd64" + IMAGE: shiny-dbsync + - name: data + CONTEXT: docker/data + DOCKERFILE: docker/data/Dockerfile + PLATFORM: "linux/amd64,linux/arm64" + IMAGE: data + - name: monitor + CONTEXT: docker/monitor + DOCKERFILE: docker/monitor/Dockerfile + PLATFORM: "linux/amd64,linux/arm64" + IMAGE: monitor + - name: rstudio-nginx + CONTEXT: docker/rstudio-nginx + DOCKERFILE: docker/rstudio-nginx/Dockerfile + PLATFORM: "linux/amd64,linux/arm64" + IMAGE: rstudio-nginx + + steps: + - uses: actions/checkout@v4 + + # create metadata for image + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + # list of Docker images to use as base name for tags + images: | + pecan/${{ matrix.IMAGE }} + ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/${{ matrix.IMAGE }} + # generate Docker tags based on the following events/attributes + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + # setup docker build + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Inspect Builder + run: | + echo "Name: ${{ steps.buildx.outputs.name }}" + echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" + echo "Status: ${{ steps.buildx.outputs.status }}" + echo "Flags: ${{ steps.buildx.outputs.flags }}" + echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + + # login to registries + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # build the docker images + - name: Build and push ${{ matrix.name }} + uses: docker/build-push-action@v6 + with: + context: ${{ matrix.CONTEXT }} + file: ${{ matrix.DOCKERFILE }} + push: true + platforms: ${{ matrix.PLATFORM }} + cache-from: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache + cache-to: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache,mode=max + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ steps.meta.outputs.version }} + R_VERSION=${{ env.R_VERSION }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 3717d3372ea..f2a265d2ee9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ For more information about this file see also [Keep a Changelog](http://keepacha ### Added ### Fixed +- updated github action to build docker images ### Changed diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 4aa5a6a2d9a..059e14f0c0c 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -6,7 +6,6 @@ ARG IMAGE_VERSION="latest" # PECAN FOR MODEL BASE IMAGE # -------------------------------------------------------------------------- FROM pecan/base:${IMAGE_VERSION} -LABEL maintainer="Tezan Sahu " EXPOSE 8000 @@ -15,9 +14,7 @@ EXPOSE 8000 # -------------------------------------------------------------------------- # COMMAND TO RUN -RUN --mount=type=secret,id=github_token \ - export GITHUB_PAT=`cat /run/secrets/github_token` \ - && apt-get update \ +RUN apt-get update \ && apt-get install libsodium-dev -y \ && rm -rf /var/lib/apt/lists/* \ && Rscript -e "devtools::install_version('promises', '1.1.0')" \ @@ -35,7 +32,6 @@ ENV AUTH_REQ="TRUE" \ SECRET_KEY_BASE="thisisnotasecret" WORKDIR /api/R - -CMD Rscript entrypoint.R - COPY ./ /api + +CMD ["Rscript", "entrypoint.R"] diff --git a/docker-compose.yml b/docker-compose.yml index 92af1a03a95..c5c77c37daa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,6 +52,11 @@ services: - "traefik.http.routers.rabbitmq.rule=Host(`rabbitmq.pecan.localhost`)" volumes: - rabbitmq:/var/lib/rabbitmq + healthcheck: + test: rabbitmqctl ping + interval: 10s + timeout: 5s + retries: 5 # ---------------------------------------------------------------------- # Database to hold the data from PEcAn and BETY. @@ -65,6 +70,11 @@ services: - pecan volumes: - postgres:/var/lib/postgresql/data + healthcheck: + test: pg_isready -U postgres + interval: 10s + timeout: 5s + retries: 5 # ---------------------------------------------------------------------- # BETY rails frontend to the database @@ -82,11 +92,17 @@ services: volumes: - bety:/home/bety/log depends_on: - - postgres + postgres: + condition: service_healthy labels: - "traefik.enable=true" - "traefik.http.services.bety.loadbalancer.server.port=8000" - "traefik.http.routers.bety.rule=Host(`${TRAEFIK_HOST:-pecan.localhost}`) && PathPrefix(`/bety/`)" + healthcheck: + test: "curl --silent --fail http://localhost:8000/${RAILS_RELATIVE_URL_ROOT} > /dev/null || exit 1" + interval: 10s + timeout: 5s + retries: 5 # ---------------------------------------------------------------------- # RStudio @@ -98,8 +114,10 @@ services: networks: - pecan depends_on: - - rabbitmq - - postgres + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy environment: - KEEP_ENV=RABBITMQ_URI RABBITMQ_PREFIX RABBITMQ_PORT FQDN NAME - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F} @@ -141,6 +159,11 @@ services: - "traefik.enable=true" - "traefik.http.services.docs.loadbalancer.server.port=80" - "traefik.http.routers.docs.rule=Host(`${TRAEFIK_HOST:-pecan.localhost}`) && PathPrefix(`/`)" + healthcheck: + test: "curl --silent --fail http://localhost/ > /dev/null || exit 1" + interval: 10s + timeout: 5s + retries: 5 # PEcAn web front end, this is just the PHP code pecan: @@ -155,8 +178,10 @@ services: - NAME=${PECAN_NAME:-docker} - SECRET_KEY_BASE=${BETY_SECRET_KEY:-thisisnotasecret} depends_on: - - postgres - - rabbitmq + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy labels: - "traefik.enable=true" - "traefik.http.services.pecan.loadbalancer.server.port=8080" @@ -164,6 +189,11 @@ services: volumes: - pecan:/data - pecan:/var/www/html/pecan/data + healthcheck: + test: "curl --silent --fail http://localhost/pecan/ > /dev/null || exit 1" + interval: 10s + timeout: 5s + retries: 5 # PEcAn model monitor monitor: @@ -176,7 +206,10 @@ services: - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F} - FQDN=${PECAN_FQDN:-docker} depends_on: - - rabbitmq + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy labels: - "traefik.enable=true" - "traefik.http.routers.monitor.rule=Host(`${TRAEFIK_HOST:-pecan.localhost}`) && PathPrefix(`/monitor/`)" @@ -184,6 +217,11 @@ services: - "traefik.http.middlewares.monitor-stripprefix.stripprefix.prefixes=/monitor" volumes: - pecan:/data + healthcheck: + test: "curl --silent --fail http://localhost:9999/monitor/ > /dev/null || exit 1" + interval: 10s + timeout: 5s + retries: 5 # PEcAn executor, executes jobs. Does not the actual models executor: @@ -198,8 +236,10 @@ services: - RABBITMQ_PORT=15672 - FQDN=${PECAN_FQDN:-docker} depends_on: - - postgres - - rabbitmq + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy volumes: - pecan:/data @@ -216,7 +256,8 @@ services: environment: - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F} depends_on: - - rabbitmq + rabbitmq: + condition: service_healthy volumes: - pecan:/data @@ -230,7 +271,8 @@ services: environment: - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F} depends_on: - - rabbitmq + rabbitmq: + condition: service_healthy volumes: - pecan:/data @@ -244,7 +286,8 @@ services: environment: - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F} depends_on: - - rabbitmq + rabbitmq: + condition: service_healthy volumes: - pecan:/data @@ -258,7 +301,8 @@ services: environment: - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F} depends_on: - - rabbitmq + rabbitmq: + condition: service_healthy volumes: - pecan:/data @@ -272,7 +316,8 @@ services: environment: - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F} depends_on: - - rabbitmq + rabbitmq: + condition: service_healthy volumes: - pecan:/data @@ -286,7 +331,8 @@ services: environment: - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F} depends_on: - - rabbitmq + rabbitmq: + condition: service_healthy volumes: - pecan:/data @@ -300,12 +346,18 @@ services: networks: - pecan depends_on: - - postgres + postgres: + condition: service_healthy labels: - "traefik.enable=true" - "traefik.http.routers.dbsync.rule=Host(`${TRAEFIK_HOST:-pecan.localhost}`) && PathPrefix(`/dbsync/`)" - "traefik.http.routers.dbsync.middlewares=dbsync-stripprefix" - "traefik.http.middlewares.dbsync-stripprefix.stripprefix.prefixes=/monitor" + healthcheck: + test: "curl --silent --fail http://localhost/dbsync/ > /dev/null || exit 1" + interval: 10s + timeout: 5s + retries: 5 # ---------------------------------------------------------------------- # PEcAn API @@ -329,9 +381,15 @@ services: - "traefik.http.routers.api.rule=Host(`${TRAEFIK_HOST:-pecan.localhost}`) && PathPrefix(`/api/`)" - "traefik.http.services.api.loadbalancer.server.port=8000" depends_on: - - postgres + postgres: + condition: service_healthy volumes: - pecan:/data/ + healthcheck: + test: "curl --silent --fail http://localhost/api/ping > /dev/null || exit 1" + interval: 10s + timeout: 5s + retries: 5 # ---------------------------------------------------------------------- # Name of network to be used by all containers diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index 8ec9f3347ef..abbffd47c9b 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -2,7 +2,6 @@ ARG IMAGE_VERSION="latest" ARG FROM_IMAGE="depends" FROM pecan/${FROM_IMAGE}:${IMAGE_VERSION} -MAINTAINER Rob Kooper # ---------------------------------------------------------------------- # PEcAn version information @@ -26,9 +25,7 @@ COPY models /pecan/models/ # install all PEcAn packages # `make clean` is to remove artifacts copied in from host system # (e.g. basgra.so) -RUN --mount=type=secret,id=github_token \ - export GITHUB_PAT=`cat /run/secrets/github_token` \ - && cd /pecan \ +RUN cd /pecan \ && make clean \ && make \ && rm -rf /tmp/downloaded_packages @@ -38,7 +35,7 @@ WORKDIR /work COPY web/workflow.R docker/base/rstudio.sh /work/ # COMMAND TO RUN -CMD Rscript --vanilla workflow.R | tee workflow.Rout +CMD ["bash", "-c", "Rscript --vanilla workflow.R | tee workflow.Rout"] # variables to store in docker image ENV PECAN_VERSION=${PECAN_VERSION} \ diff --git a/docker/data/Dockerfile b/docker/data/Dockerfile index 2ac60cd7e83..b0581386039 100644 --- a/docker/data/Dockerfile +++ b/docker/data/Dockerfile @@ -1,5 +1,4 @@ FROM alpine -MAINTAINER Rob Kooper # name to use in the machines table FQDN when registering the data files ENV FQDN="" \ @@ -19,4 +18,4 @@ RUN apk --no-cache add bash curl rsync postgresql-client unzip \ && curl -s -o create_met_driver.tar.gz http://isda.ncsa.illinois.edu/~kooper/EBI/create_met_driver.tar.gz COPY add-data.sh add.util.sh /work/ -CMD bash /work/add-data.sh +CMD ["bash", "-c", "/work/add-data.sh"] diff --git a/docker/depends/Dockerfile b/docker/depends/Dockerfile index 8afa79b5ad7..b784bcd3bfa 100644 --- a/docker/depends/Dockerfile +++ b/docker/depends/Dockerfile @@ -5,8 +5,6 @@ ARG FROM_IMAGE="rocker/tidyverse" # PECAN FOR MODEL BASE IMAGE # ---------------------------------------------------------------------- FROM ${FROM_IMAGE}:${R_VERSION} -MAINTAINER Rob Kooper - # ---------------------------------------------------------------------- # INSTALL BINARY/LIBRARY DEPENDENCIES @@ -33,9 +31,7 @@ RUN apt-get update \ # INSTALL DEPENDENCIES # ---------------------------------------------------------------------- COPY pecan.depends.R pecan_package_dependencies.csv pecan_deps_from_github.txt / -RUN --mount=type=secret,id=github_token \ - export GITHUB_PAT=`cat /run/secrets/github_token` \ - && Rscript -e "install.packages(c('desc', 'remotes'))" \ +RUN Rscript -e "install.packages(c('desc', 'remotes'))" \ && R_LIBS_USER='/usr/local/lib/R/site-library' Rscript /pecan.depends.R \ && rm -rf /tmp/* diff --git a/docker/docs/Dockerfile b/docker/docs/Dockerfile index 2c98d076a03..dcc7955b185 100644 --- a/docker/docs/Dockerfile +++ b/docker/docs/Dockerfile @@ -5,7 +5,6 @@ ARG IMAGE_VERSION="latest" # compile bookdown to html # ---------------------------------------------------------------------- FROM pecan/base:${IMAGE_VERSION} AS pecandocs -MAINTAINER Rob Kooper RUN apt-get update \ && apt-get install -y --no-install-recommends pandoc \ @@ -28,7 +27,6 @@ RUN make build # copy html pages to container # ---------------------------------------------------------------------- FROM httpd -MAINTAINER Rob Kooper COPY docker/docs/index.html /usr/local/apache2/htdocs/ COPY --from=pecandocs /src/book_source/_book/ /usr/local/apache2/htdocs/docs/pecan/ diff --git a/docker/executor/Dockerfile b/docker/executor/Dockerfile index 34722783909..19cecced3cd 100644 --- a/docker/executor/Dockerfile +++ b/docker/executor/Dockerfile @@ -5,7 +5,6 @@ ARG IMAGE_VERSION="latest" # PECAN FOR MODEL BASE IMAGE # ---------------------------------------------------------------------- FROM pecan/base:${IMAGE_VERSION} -MAINTAINER Rob Kooper # ---------------------------------------------------------------------- # SETUP FOR PYTHON CODE @@ -24,4 +23,4 @@ ENV RABBITMQ_URI="amqp://guest:guest@rabbitmq/%2F" \ # actual application that will be executed COPY executor.py /work/ -CMD python3 /work/executor.py +CMD ["python3", "/work/executor.py"] diff --git a/docker/models/Dockerfile b/docker/models/Dockerfile index 1870fba9a03..f76f51382e1 100644 --- a/docker/models/Dockerfile +++ b/docker/models/Dockerfile @@ -5,7 +5,6 @@ ARG IMAGE_VERSION="latest" # PECAN FOR MODEL BASE IMAGE # ---------------------------------------------------------------------- FROM pecan/base:${IMAGE_VERSION} -MAINTAINER Rob Kooper # ---------------------------------------------------------------------- # SETUP FOR PYTHON CODE @@ -23,4 +22,4 @@ ENV RABBITMQ_URI="amqp://guest:guest@rabbitmq/%2F" \ # actual application that will be executed COPY model.py /work/ -CMD python3 /work/model.py +CMD ["python3", "/work/model.py"] diff --git a/docker/monitor/Dockerfile b/docker/monitor/Dockerfile index 24232352fca..ec479cc2458 100644 --- a/docker/monitor/Dockerfile +++ b/docker/monitor/Dockerfile @@ -17,4 +17,4 @@ COPY requirements.txt /src/ RUN pip3 install -r /src/requirements.txt COPY . /src/ -CMD python3 monitor.py +CMD ["python3", "./monitor.py"] diff --git a/docker/rstudio-nginx/Dockerfile b/docker/rstudio-nginx/Dockerfile index 600c1da6484..af33c2c3d71 100644 --- a/docker/rstudio-nginx/Dockerfile +++ b/docker/rstudio-nginx/Dockerfile @@ -1,4 +1,3 @@ FROM nginx:alpine -MAINTAINER Rob Kooper COPY nginx.conf /etc/nginx/conf.d/default.conf diff --git a/docker/web/Dockerfile b/docker/web/Dockerfile index f22315cf3d5..9382561e6e7 100644 --- a/docker/web/Dockerfile +++ b/docker/web/Dockerfile @@ -1,5 +1,4 @@ FROM php:8-apache -MAINTAINER Rob Kooper # ---------------------------------------------------------------------- # install rabbitmq and postgresql extentions diff --git a/models/ed/Dockerfile b/models/ed/Dockerfile index 282a5b5d391..a3873e896a7 100644 --- a/models/ed/Dockerfile +++ b/models/ed/Dockerfile @@ -4,15 +4,12 @@ ARG IMAGE_VERSION="latest" # ---------------------------------------------------------------------- # BUILD MODEL BINARY # ---------------------------------------------------------------------- -FROM pecan/models:${IMAGE_VERSION} as model-binary +FROM pecan/models:${IMAGE_VERSION} AS model-binary # Some variables that can be used to set control the docker build ARG MODEL_VERSION="2.2.0" ARG BINARY_VERSION="2.2" -# specify fortran compiler -ENV FC_TYPE=GNU - # install dependencies RUN apt-get update \ && apt-get install -y --no-install-recommends \ @@ -20,17 +17,16 @@ RUN apt-get update \ curl \ gfortran \ git \ - libhdf5-dev \ - libopenmpi-dev \ + libhdf5-openmpi-dev \ && rm -rf /var/lib/apt/lists/* # download, unzip and build ed2 WORKDIR /src -RUN git -c http.sslVerify=false clone https://github.com/EDmodel/ED2.git \ - && cd ED2/ED/build \ - && curl -o make/include.mk.VM http://isda.ncsa.illinois.edu/~kooper/EBI/include.mk.opt.`uname -s` \ +RUN git -c http.sslVerify=false clone https://github.com/EDmodel/ED2.git +COPY data-raw/include.mk.opt /src/ED2/ED/build/make/include.mk.pecan +RUN cd ED2/ED/build \ && if [ "${MODEL_VERSION}" != "git" ]; then git checkout "v.${MODEL_VERSION}"; fi \ - && ./install.sh -g -p VM \ + && ./install.sh -g -p pecan \ && mv /src/ED2/ED/build/ed_${BINARY_VERSION}-opt /src/ED2/ED/build/ed ######################################################################## @@ -46,7 +42,8 @@ FROM pecan/models:${IMAGE_VERSION} RUN apt-get update \ && apt-get install -y --no-install-recommends \ - libopenmpi3 \ + libhdf5-openmpi-103 \ + libgomp1 \ && rm -rf /var/lib/apt/lists/* # INSTALL PEcAn.ED2 diff --git a/models/ed/data-raw/include.mk.opt b/models/ed/data-raw/include.mk.opt new file mode 100644 index 00000000000..da68d2f1455 --- /dev/null +++ b/models/ed/data-raw/include.mk.opt @@ -0,0 +1,38 @@ +#Makefile include include.mk.opt.ubuntu +############################################################################ + +# Define make (gnu make works best). +MAKE=/usr/bin/make + +# libraries. +BASE=$(ED_ROOT)/build/ + +# HDF 5 Libraries +HDF5_INCS=-I/usr/include/hdf5/openmpi +HDF5_LIBS= -L/usr/lib/$(shell uname -m)-linux-gnu/hdf5/openmpi -lhdf5_fortran -lhdf5_hl -lhdf5 -lz -lm +USE_COLLECTIVE_MPIO=0 + +# interface +USE_INTERF=1 + +# gfortran +CMACH=PC_LINUX1 +FC_TYPE=GNU +F_COMP=mpif90.openmpi +F_OPTS=-O3 -ffree-line-length-none -frecursive -fopenmp -static +C_COMP=mpicc.openmpi +C_OPTS=-O0 -DLITTLE -g -static +LOADER=mpif90.openmpi +LOADER_OPTS=-O3 -ffree-line-length-none -frecursive -fopenmp +C_LOADER==mpicc.openmpi +LIBS= +MOD_EXT=mod + +# using MPI libraries: +MPI_PATH= +PAR_INCS= +PAR_LIBS= +PAR_DEFS= + +# For IBM,HP,SGI,ALPHA,LINUX use these: +ARCHIVE=ar rs diff --git a/models/maespa/Dockerfile b/models/maespa/Dockerfile index 0b5fbd1be54..22243dc3c1b 100644 --- a/models/maespa/Dockerfile +++ b/models/maespa/Dockerfile @@ -4,7 +4,7 @@ ARG IMAGE_VERSION="latest" # ---------------------------------------------------------------------- # BUILD MODEL BINARY # ---------------------------------------------------------------------- -FROM pecan/models:${IMAGE_VERSION} as model-binary +FROM pecan/models:${IMAGE_VERSION} AS model-binary # download, unzip and build ed2 WORKDIR /src diff --git a/models/sipnet/Dockerfile b/models/sipnet/Dockerfile index 1b0fc75725e..d69e845dd4d 100644 --- a/models/sipnet/Dockerfile +++ b/models/sipnet/Dockerfile @@ -4,7 +4,7 @@ ARG IMAGE_VERSION="latest" # ---------------------------------------------------------------------- # BUILD SIPNET BINARY # ---------------------------------------------------------------------- -FROM pecan/models:${IMAGE_VERSION} as model-binary +FROM pecan/models:${IMAGE_VERSION} AS model-binary # Some variables that can be used to set control the docker build ARG MODEL_VERSION=git diff --git a/shiny/dbsync/Dockerfile b/shiny/dbsync/Dockerfile index d5a89a6ca0f..a133290de8c 100644 --- a/shiny/dbsync/Dockerfile +++ b/shiny/dbsync/Dockerfile @@ -18,4 +18,4 @@ ADD https://raw.githubusercontent.com/rocker-org/shiny/master/shiny-server.sh /u RUN chmod +x /usr/bin/shiny-server.sh # special script to start shiny server and preserve env variable -CMD /srv/shiny-server/save-env-shiny.sh +CMD ["/srv/shiny-server/save-env-shiny.sh"]