From 959b173497572dcc615baa476b96cb88cc60bc0b Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Wed, 7 Aug 2024 13:23:05 -0500 Subject: [PATCH 01/52] start new docker build --- .github/workflows/gha.yml | 93 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 .github/workflows/gha.yml diff --git a/.github/workflows/gha.yml b/.github/workflows/gha.yml new file mode 100644 index 0000000000..f822b71f87 --- /dev/null +++ b/.github/workflows/gha.yml @@ -0,0 +1,93 @@ +name: Docker + +on: + # push: + # branches: + # - main + # tags: + # - "v*.*.*" + # pull_request: + workflow_dispatch: + inputs: + depends: + description: 'Create depends image' + required: true + default: "no" + r_version: + description: 'R version to use' + required: true + default: "4.1" + +# there are 3 jobs to build different images +jobs: + depends: + runs-on: ubuntu-latest + permissions: + packages: write + + steps: + - uses: actions/checkout@v4 + + # create metadata for image + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + # list of Docker images to use as base name for tags + images: | + #pecan/${{ matrix.IMAGE }} + ghcr.io/${{ github.repository_owner }}/${{ matrix.IMAGE }} + # generate Docker tags based on the following events/attributes + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + # setup docker build + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Inspect Builder + run: | + echo "Name: ${{ steps.buildx.outputs.name }}" + echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" + echo "Status: ${{ steps.buildx.outputs.status }}" + echo "Flags: ${{ steps.buildx.outputs.flags }}" + echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + + # login to registries + # - name: Login to DockerHub + # uses: docker/login-action@v3 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} + + # - name: Login to GitHub Container Registry + # uses: docker/login-action@v3 + # with: + # registry: ghcr.io + # username: ${{ github.actor }} + # password: ${{ secrets.GITHUB_TOKEN }} + + # build the docker images + - name: Build and push depends + uses: docker/build-push-action@v5 + with: + context: docker/depends + #push: ${{ github.event_name != 'pull_request' }} + #push: true + platforms: "linux/amd64,linux/arm64" + # cache-from: type=registry,ref=classtranscribe/frontend-pipeline:buildcache + # cache-to: type=registry,ref=classtranscribe/frontend:buildcache,mode=max + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ steps.meta.outputs.version }} + R_VERSION=${{ github.event.inputs.r_version}} From 4699a27776a63fe83824114a500c624fc79ddf4b Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Wed, 7 Aug 2024 13:27:01 -0500 Subject: [PATCH 02/52] disable most builds --- .github/workflows/book.yml | 18 +++++++++--------- .github/workflows/ci.yml | 22 +++++++++++----------- .github/workflows/gha.yml | 2 +- .github/workflows/prlabeler.yml | 2 +- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index a7551fdf2f..30b92fb160 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -1,14 +1,14 @@ name: renderbook -on: - push: - branches: - - master - - develop - tags: - - '*' - pull_request: - merge_group: +# on: +# push: +# branches: +# - master +# - develop +# tags: +# - '*' +# pull_request: +# merge_group: workflow_dispatch: jobs: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bc37aa5a1..8fb6379f27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,18 +1,18 @@ name: CI on: - push: - branches: - - master - - develop + # push: + # branches: + # - master + # - develop - tags: - - '*' - pull_request: - merge_group: - issue_comment: - types: - - created + # tags: + # - '*' + # pull_request: + # merge_group: + # issue_comment: + # types: + # - created workflow_dispatch: env: diff --git a/.github/workflows/gha.yml b/.github/workflows/gha.yml index f822b71f87..aee73e444b 100644 --- a/.github/workflows/gha.yml +++ b/.github/workflows/gha.yml @@ -1,4 +1,4 @@ -name: Docker +name: Docker GHA on: # push: diff --git a/.github/workflows/prlabeler.yml b/.github/workflows/prlabeler.yml index 921b6fd451..abd26db032 100644 --- a/.github/workflows/prlabeler.yml +++ b/.github/workflows/prlabeler.yml @@ -5,7 +5,7 @@ name: "PR Labeler" on: - - pull_request_target + # - pull_request_target jobs: label: permissions: From 2322ea3516caeed11850a5f9df6c57645b608cbb Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Wed, 7 Aug 2024 13:29:01 -0500 Subject: [PATCH 03/52] move to docker.yaml --- .github/workflows/docker.yml | 340 ++++++++++++++++++++++------------- .github/workflows/gha.yml | 93 ---------- 2 files changed, 217 insertions(+), 216 deletions(-) delete mode 100644 .github/workflows/gha.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fded4f7376..60c6ca8844 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,136 +1,230 @@ -name: Docker - -# initially we would us on: [release] as well, the problem is that -# the code in clowder would not know what branch the code is in, -# and would not set the right version flags. - -# This will run when: -# - when new code is pushed to master/develop to push the tags -# latest and develop -# - when a pull request is created and updated to make sure the -# Dockerfile is still valid. -# To be able to push to dockerhub, this expects the following -# secrets to be set in the project: -# - DOCKERHUB_USERNAME : username that can push to the org -# - DOCKERHUB_PASSWORD : password asscoaited with the username +name: Docker GHA + on: - push: - branches: - - master - - develop - pull_request: - merge_group: - issue_comment: - types: - - created + # push: + # branches: + # - main + # tags: + # - "v*.*.*" + # pull_request: workflow_dispatch: - -# Certain actions will only run when this is the master repo. -env: - MASTER_REPO: PecanProject/pecan - DOCKERHUB_ORG: pecan - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - + inputs: + depends: + description: 'Create depends image' + required: true + default: "no" + r_version: + description: 'R version to use' + required: true + default: "4.1" + +# there are 3 jobs to build different images jobs: - docker: - if: github.event_name != 'issue_comment' || startsWith(github.event.comment.body, '/build') + depends: runs-on: ubuntu-latest + permissions: + packages: write steps: - - name: Work around https://github.com/actions/checkout/issues/766 - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - uses: actions/checkout@v4 + + # create metadata for image + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 with: - set-safe-directory: false + # list of Docker images to use as base name for tags + images: | + #pecan/${{ matrix.IMAGE }} + ghcr.io/${{ github.repository_owner }}/${{ matrix.IMAGE }} + # generate Docker tags based on the following events/attributes + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + # setup docker build + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + id: buildx uses: docker/setup-buildx-action@v3 - with: - driver: docker - buildkitd-flags: --debug - install: true - # calculate some variables that are used later - - name: get version tag + - name: Inspect Builder run: | - BRANCH=${GITHUB_REF##*/} - echo "GITHUB_BRANCH=${BRANCH}" >> $GITHUB_ENV - if [ "$BRANCH" == "master" ]; then - version="$(awk '/Version:/ { print $2 }' base/all/DESCRIPTION)" - tags="latest" - oldversion="" - while [ "${oldversion}" != "${version}" ]; do - oldversion="${version}" - tags="${tags},${version}" - version=${version%.*} - done - echo "PECAN_VERSION=$(awk '/Version:/ { print $2 }' base/all/DESCRIPTION)" >> $GITHUB_ENV - echo "PECAN_TAGS=${tags}" >> $GITHUB_ENV - elif [ "$BRANCH" == "develop" ]; then - echo "PECAN_VERSION=develop" >> $GITHUB_ENV - echo "PECAN_TAGS=develop" >> $GITHUB_ENV - else - echo "PECAN_VERSION=develop" >> $GITHUB_ENV - echo "PECAN_TAGS=develop" >> $GITHUB_ENV - fi - # If any dependencies changed in this PR, try to update depends image - - name: check for changed dependencies - uses: dorny/paths-filter@v3 - id: findchanges + echo "Name: ${{ steps.buildx.outputs.name }}" + echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" + echo "Status: ${{ steps.buildx.outputs.status }}" + echo "Flags: ${{ steps.buildx.outputs.flags }}" + echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + + # login to registries + # - name: Login to DockerHub + # uses: docker/login-action@v3 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} + + # - name: Login to GitHub Container Registry + # uses: docker/login-action@v3 + # with: + # registry: ghcr.io + # username: ${{ github.actor }} + # password: ${{ secrets.GITHUB_TOKEN }} + + # build the docker images + - name: Build and push depends + uses: docker/build-push-action@v5 with: - filters: | - deps: - - docker/depends/** - - if: steps.findchanges.outputs.deps == 'true' - # NB: GITHUB_BASE_REF is only set on pull requests, - # so non-PR builds will find no existing tag and build fresh - run: | - if $(docker manifest inspect pecan/depends:${GITHUB_BASE_REF##*/} > /dev/null 2>&1); then - echo "UPDATE_DEPENDS_FROM_TAG=${GITHUB_BASE_REF##*/}" >> $GITHUB_ENV - else - echo "BUILD_DEPENDS_FRESH=true" >> GITHUB_ENV - fi - env: - GITHUB_BASE_REF: ${{ github.base_ref }} - # use shell script to build, there is some complexity in this - - name: create images - run: ./docker.sh -i github - env: - PECAN_GIT_CHECKSUM: ${{ github.sha }} - PECAN_GIT_BRANCH: ${GITHUB_BRANCH} - VERSION: ${{ env.PECAN_VERSION }} - UPDATE_DEPENDS_FROM_TAG: ${{ env.UPDATE_DEPENDS_FROM_TAG }} - BUILD: ${{ env.BUILD_DEPENDS_FRESH }} - - # push all images to github - - name: Publish to GitHub - if: github.event_name == 'push' && github.repository == env.MASTER_REPO - run: | - echo "${INPUT_PASSWORD}" | docker login -u ${INPUT_USERNAME} --password-stdin ${INPUT_REGISTRY} - repo=$(echo ${{ github.repository_owner }} | tr 'A-Z' 'a-z') - for image in $(docker image ls pecan/*:github --format "{{ .Repository }}"); do - for v in ${PECAN_TAGS}; do - docker tag ${image}:github ${INPUT_REGISTRY}/${repo}/${image#pecan/}:${v} - docker push ${INPUT_REGISTRY}/${repo}/${image#pecan/}:${v} - done - done - docker logout - env: - INPUT_REGISTRY: ghcr.io - INPUT_USERNAME: ${{ secrets.GHCR_USERNAME }} - INPUT_PASSWORD: ${{ secrets.GHCR_PASSWORD }} - - # push all images to dockerhub - - name: Publish to DockerHub - if: github.event_name == 'push' && github.repository == env.MASTER_REPO - run: | - echo "${INPUT_PASSWORD}" | docker login -u ${INPUT_USERNAME} --password-stdin - for image in $(docker image ls pecan/*:github --format "{{ .Repository }}"); do - for v in ${PECAN_TAGS}; do - docker tag ${image}:github ${{ env.DOCKERHUB_ORG }}/${image#pecan/}:${v} - docker push ${{ env.DOCKERHUB_ORG }}/${image#pecan/}:${v} - done - done - docker logout - env: - INPUT_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - INPUT_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + context: docker/depends + #push: ${{ github.event_name != 'pull_request' }} + #push: true + platforms: "linux/amd64,linux/arm64" + # cache-from: type=registry,ref=classtranscribe/frontend-pipeline:buildcache + # cache-to: type=registry,ref=classtranscribe/frontend:buildcache,mode=max + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ steps.meta.outputs.version }} + R_VERSION=${{ github.event.inputs.r_version}} + +# name: Docker + +# # initially we would us on: [release] as well, the problem is that +# # the code in clowder would not know what branch the code is in, +# # and would not set the right version flags. + +# # This will run when: +# # - when new code is pushed to master/develop to push the tags +# # latest and develop +# # - when a pull request is created and updated to make sure the +# # Dockerfile is still valid. +# # To be able to push to dockerhub, this expects the following +# # secrets to be set in the project: +# # - DOCKERHUB_USERNAME : username that can push to the org +# # - DOCKERHUB_PASSWORD : password asscoaited with the username +# on: +# push: +# branches: +# - master +# - develop +# pull_request: +# merge_group: +# issue_comment: +# types: +# - created +# workflow_dispatch: + +# # Certain actions will only run when this is the master repo. +# env: +# MASTER_REPO: PecanProject/pecan +# DOCKERHUB_ORG: pecan +# GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + +# jobs: +# docker: +# if: github.event_name != 'issue_comment' || startsWith(github.event.comment.body, '/build') +# runs-on: ubuntu-latest + +# steps: +# - name: Work around https://github.com/actions/checkout/issues/766 +# run: git config --global --add safe.directory "$GITHUB_WORKSPACE" +# - uses: actions/checkout@v4 +# with: +# set-safe-directory: false +# - name: Set up Docker Buildx +# uses: docker/setup-buildx-action@v3 +# with: +# driver: docker +# buildkitd-flags: --debug +# install: true + +# # calculate some variables that are used later +# - name: get version tag +# run: | +# BRANCH=${GITHUB_REF##*/} +# echo "GITHUB_BRANCH=${BRANCH}" >> $GITHUB_ENV +# if [ "$BRANCH" == "master" ]; then +# version="$(awk '/Version:/ { print $2 }' base/all/DESCRIPTION)" +# tags="latest" +# oldversion="" +# while [ "${oldversion}" != "${version}" ]; do +# oldversion="${version}" +# tags="${tags},${version}" +# version=${version%.*} +# done +# echo "PECAN_VERSION=$(awk '/Version:/ { print $2 }' base/all/DESCRIPTION)" >> $GITHUB_ENV +# echo "PECAN_TAGS=${tags}" >> $GITHUB_ENV +# elif [ "$BRANCH" == "develop" ]; then +# echo "PECAN_VERSION=develop" >> $GITHUB_ENV +# echo "PECAN_TAGS=develop" >> $GITHUB_ENV +# else +# echo "PECAN_VERSION=develop" >> $GITHUB_ENV +# echo "PECAN_TAGS=develop" >> $GITHUB_ENV +# fi +# # If any dependencies changed in this PR, try to update depends image +# - name: check for changed dependencies +# uses: dorny/paths-filter@v3 +# id: findchanges +# with: +# filters: | +# deps: +# - docker/depends/** +# - if: steps.findchanges.outputs.deps == 'true' +# # NB: GITHUB_BASE_REF is only set on pull requests, +# # so non-PR builds will find no existing tag and build fresh +# run: | +# if $(docker manifest inspect pecan/depends:${GITHUB_BASE_REF##*/} > /dev/null 2>&1); then +# echo "UPDATE_DEPENDS_FROM_TAG=${GITHUB_BASE_REF##*/}" >> $GITHUB_ENV +# else +# echo "BUILD_DEPENDS_FRESH=true" >> GITHUB_ENV +# fi +# env: +# GITHUB_BASE_REF: ${{ github.base_ref }} +# # use shell script to build, there is some complexity in this +# - name: create images +# run: ./docker.sh -i github +# env: +# PECAN_GIT_CHECKSUM: ${{ github.sha }} +# PECAN_GIT_BRANCH: ${GITHUB_BRANCH} +# VERSION: ${{ env.PECAN_VERSION }} +# UPDATE_DEPENDS_FROM_TAG: ${{ env.UPDATE_DEPENDS_FROM_TAG }} +# BUILD: ${{ env.BUILD_DEPENDS_FRESH }} + +# # push all images to github +# - name: Publish to GitHub +# if: github.event_name == 'push' && github.repository == env.MASTER_REPO +# run: | +# echo "${INPUT_PASSWORD}" | docker login -u ${INPUT_USERNAME} --password-stdin ${INPUT_REGISTRY} +# repo=$(echo ${{ github.repository_owner }} | tr 'A-Z' 'a-z') +# for image in $(docker image ls pecan/*:github --format "{{ .Repository }}"); do +# for v in ${PECAN_TAGS}; do +# docker tag ${image}:github ${INPUT_REGISTRY}/${repo}/${image#pecan/}:${v} +# docker push ${INPUT_REGISTRY}/${repo}/${image#pecan/}:${v} +# done +# done +# docker logout +# env: +# INPUT_REGISTRY: ghcr.io +# INPUT_USERNAME: ${{ secrets.GHCR_USERNAME }} +# INPUT_PASSWORD: ${{ secrets.GHCR_PASSWORD }} + +# # push all images to dockerhub +# - name: Publish to DockerHub +# if: github.event_name == 'push' && github.repository == env.MASTER_REPO +# run: | +# echo "${INPUT_PASSWORD}" | docker login -u ${INPUT_USERNAME} --password-stdin +# for image in $(docker image ls pecan/*:github --format "{{ .Repository }}"); do +# for v in ${PECAN_TAGS}; do +# docker tag ${image}:github ${{ env.DOCKERHUB_ORG }}/${image#pecan/}:${v} +# docker push ${{ env.DOCKERHUB_ORG }}/${image#pecan/}:${v} +# done +# done +# docker logout +# env: +# INPUT_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} +# INPUT_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} diff --git a/.github/workflows/gha.yml b/.github/workflows/gha.yml deleted file mode 100644 index aee73e444b..0000000000 --- a/.github/workflows/gha.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: Docker GHA - -on: - # push: - # branches: - # - main - # tags: - # - "v*.*.*" - # pull_request: - workflow_dispatch: - inputs: - depends: - description: 'Create depends image' - required: true - default: "no" - r_version: - description: 'R version to use' - required: true - default: "4.1" - -# there are 3 jobs to build different images -jobs: - depends: - runs-on: ubuntu-latest - permissions: - packages: write - - steps: - - uses: actions/checkout@v4 - - # create metadata for image - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - #pecan/${{ matrix.IMAGE }} - ghcr.io/${{ github.repository_owner }}/${{ matrix.IMAGE }} - # generate Docker tags based on the following events/attributes - tags: | - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - - # setup docker build - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - - name: Inspect Builder - run: | - echo "Name: ${{ steps.buildx.outputs.name }}" - echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" - echo "Status: ${{ steps.buildx.outputs.status }}" - echo "Flags: ${{ steps.buildx.outputs.flags }}" - echo "Platforms: ${{ steps.buildx.outputs.platforms }}" - - # login to registries - # - name: Login to DockerHub - # uses: docker/login-action@v3 - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} - - # - name: Login to GitHub Container Registry - # uses: docker/login-action@v3 - # with: - # registry: ghcr.io - # username: ${{ github.actor }} - # password: ${{ secrets.GITHUB_TOKEN }} - - # build the docker images - - name: Build and push depends - uses: docker/build-push-action@v5 - with: - context: docker/depends - #push: ${{ github.event_name != 'pull_request' }} - #push: true - platforms: "linux/amd64,linux/arm64" - # cache-from: type=registry,ref=classtranscribe/frontend-pipeline:buildcache - # cache-to: type=registry,ref=classtranscribe/frontend:buildcache,mode=max - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - VERSION=${{ steps.meta.outputs.version }} - R_VERSION=${{ github.event.inputs.r_version}} From f36a3f1bc6f4e2850983912538fe1ad157bd12d3 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Wed, 7 Aug 2024 13:30:47 -0500 Subject: [PATCH 04/52] on PR --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 60c6ca8844..ef504031df 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -6,7 +6,7 @@ on: # - main # tags: # - "v*.*.*" - # pull_request: + pull_request: workflow_dispatch: inputs: depends: From a060043cb4de27a88ceb74f906cf1db58e4aee52 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Wed, 7 Aug 2024 13:59:02 -0500 Subject: [PATCH 05/52] remove maintainer add next images --- .github/workflows/docker.yml | 106 ++++++++++++++++++++++++++++++++++- docker/docs/Dockerfile | 1 - docker/web/Dockerfile | 1 - 3 files changed, 104 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ef504031df..c5b8c3be14 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -12,11 +12,17 @@ on: depends: description: 'Create depends image' required: true - default: "no" + type: boolean + default: true r_version: description: 'R version to use' required: true + type: choice default: "4.1" + options: + - 4.1 + - 4.2 + - 4.3 # there are 3 jobs to build different images jobs: @@ -36,7 +42,7 @@ jobs: # list of Docker images to use as base name for tags images: | #pecan/${{ matrix.IMAGE }} - ghcr.io/${{ github.repository_owner }}/${{ matrix.IMAGE }} + ghcr.io/${{ github.repository_owner }}/depends # generate Docker tags based on the following events/attributes tags: | type=schedule @@ -88,10 +94,106 @@ jobs: # cache-to: type=registry,ref=classtranscribe/frontend:buildcache,mode=max tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + outputs: type=docker,dest=/tmp/depends.tar build-args: | VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ github.event.inputs.r_version}} + # so depends image can be used in other steps + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: myimage + path: /tmp/depends.tar + + misc: + runs-on: ubuntu-latest + permissions: + packages: write + strategy: + fail-fast: false + matrix: + name: + - web + - dbsync + include: + - name: web + PLATFORM: "linux/amd64,linux/arm64" + DOCKERFILE: docker/web/Dockerfile + IMAGE: web + - name: dbsync + PLATFORM: "linux/amd64,linux/arm64" + DOCKERFILE: docker/web/Dockerfile + IMAGE: shiny-dbsync + + steps: + - uses: actions/checkout@v4 + + # create metadata for image + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + # list of Docker images to use as base name for tags + images: | + #pecan/${{ matrix.IMAGE }} + ghcr.io/${{ github.repository_owner }}/${{ matrix.IMAGE }} + # generate Docker tags based on the following events/attributes + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + # setup docker build + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Inspect Builder + run: | + echo "Name: ${{ steps.buildx.outputs.name }}" + echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" + echo "Status: ${{ steps.buildx.outputs.status }}" + echo "Flags: ${{ steps.buildx.outputs.flags }}" + echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + + # login to registries + # - name: Login to DockerHub + # uses: docker/login-action@v3 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} + + # - name: Login to GitHub Container Registry + # uses: docker/login-action@v3 + # with: + # registry: ghcr.io + # username: ${{ github.actor }} + # password: ${{ secrets.GITHUB_TOKEN }} + + # build the docker images + - name: Build and push depends + uses: docker/build-push-action@v5 + with: + context: . + file: ${{ matrix.DOCKERFILE }} + #push: ${{ github.event_name != 'pull_request' }} + #push: true + platforms: ${{ matrix.PLATFORM }} + # cache-from: type=registry,ref=classtranscribe/frontend-pipeline:buildcache + # cache-to: type=registry,ref=classtranscribe/frontend:buildcache,mode=max + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ steps.meta.outputs.version }} + R_VERSION=${{ github.event.inputs.r_version}} + # depends: # name: Docker # # initially we would us on: [release] as well, the problem is that diff --git a/docker/docs/Dockerfile b/docker/docs/Dockerfile index 2c98d076a0..9393365ce2 100644 --- a/docker/docs/Dockerfile +++ b/docker/docs/Dockerfile @@ -5,7 +5,6 @@ ARG IMAGE_VERSION="latest" # compile bookdown to html # ---------------------------------------------------------------------- FROM pecan/base:${IMAGE_VERSION} AS pecandocs -MAINTAINER Rob Kooper RUN apt-get update \ && apt-get install -y --no-install-recommends pandoc \ diff --git a/docker/web/Dockerfile b/docker/web/Dockerfile index f22315cf3d..9382561e6e 100644 --- a/docker/web/Dockerfile +++ b/docker/web/Dockerfile @@ -1,5 +1,4 @@ FROM php:8-apache -MAINTAINER Rob Kooper # ---------------------------------------------------------------------- # install rabbitmq and postgresql extentions From cccf144792ad7f131e91ea5d2ce50c2756d18194 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Wed, 7 Aug 2024 14:32:48 -0500 Subject: [PATCH 06/52] output type image --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c5b8c3be14..9d2a836c2e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -94,7 +94,7 @@ jobs: # cache-to: type=registry,ref=classtranscribe/frontend:buildcache,mode=max tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - outputs: type=docker,dest=/tmp/depends.tar + outputs: type=image,dest=/tmp/depends.tar build-args: | VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ github.event.inputs.r_version}} From e0281d1749b633555eec76152be1905f8a749d4f Mon Sep 17 00:00:00 2001 From: Chris Black Date: Thu, 8 Aug 2024 01:31:38 -0700 Subject: [PATCH 07/52] typo --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9d2a836c2e..72fb13a92b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -97,7 +97,7 @@ jobs: outputs: type=image,dest=/tmp/depends.tar build-args: | VERSION=${{ steps.meta.outputs.version }} - R_VERSION=${{ github.event.inputs.r_version}} + R_VERSION=${{ github.event.inputs.r_version }} # so depends image can be used in other steps - name: Upload artifact From 3ca4cf3c2c717b90209d3e64d7dfed33795d97fc Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 31 Aug 2024 11:56:47 -0500 Subject: [PATCH 08/52] docker build cleanup --- .github/workflows/docker.yml | 186 ++++++++++++++++++----------------- 1 file changed, 98 insertions(+), 88 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9d2a836c2e..b86cbdcd9d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -30,7 +30,17 @@ jobs: runs-on: ubuntu-latest permissions: packages: write - + strategy: + fail-fast: false + matrix: + name: + - depends + include: + - name: depends + PLATFORM: "linux/amd64,linux/arm64" + DOCKERFILE: docker/web/Dockerfile + IMAGE: depends + steps: - uses: actions/checkout@v4 @@ -42,7 +52,7 @@ jobs: # list of Docker images to use as base name for tags images: | #pecan/${{ matrix.IMAGE }} - ghcr.io/${{ github.repository_owner }}/depends + ghcr.io/${{ github.repository_owner }}/${{ matrix.IMAGE }} # generate Docker tags based on the following events/attributes tags: | type=schedule @@ -84,17 +94,17 @@ jobs: # build the docker images - name: Build and push depends - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: docker/depends #push: ${{ github.event_name != 'pull_request' }} #push: true - platforms: "linux/amd64,linux/arm64" - # cache-from: type=registry,ref=classtranscribe/frontend-pipeline:buildcache - # cache-to: type=registry,ref=classtranscribe/frontend:buildcache,mode=max + platforms: ${{ matrix.PLATFORM }} + cache-from: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache + cache-to: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache,mode=max tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,dest=/tmp/depends.tar + outputs: type=image,dest=/tmp/${{ matrix.IMAGE }}.tar build-args: | VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ github.event.inputs.r_version}} @@ -103,96 +113,96 @@ jobs: - name: Upload artifact uses: actions/upload-artifact@v4 with: - name: myimage - path: /tmp/depends.tar + name: ${{ matrix.IMAGE }} + path: /tmp/${{ matrix.IMAGE }}.tar - misc: - runs-on: ubuntu-latest - permissions: - packages: write - strategy: - fail-fast: false - matrix: - name: - - web - - dbsync - include: - - name: web - PLATFORM: "linux/amd64,linux/arm64" - DOCKERFILE: docker/web/Dockerfile - IMAGE: web - - name: dbsync - PLATFORM: "linux/amd64,linux/arm64" - DOCKERFILE: docker/web/Dockerfile - IMAGE: shiny-dbsync + # misc: + # runs-on: ubuntu-latest + # permissions: + # packages: write + # strategy: + # fail-fast: false + # matrix: + # name: + # - web + # - dbsync + # include: + # - name: web + # PLATFORM: "linux/amd64,linux/arm64" + # DOCKERFILE: docker/web/Dockerfile + # IMAGE: web + # - name: dbsync + # PLATFORM: "linux/amd64,linux/arm64" + # DOCKERFILE: docker/web/Dockerfile + # IMAGE: shiny-dbsync - steps: - - uses: actions/checkout@v4 + # steps: + # - uses: actions/checkout@v4 - # create metadata for image - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - #pecan/${{ matrix.IMAGE }} - ghcr.io/${{ github.repository_owner }}/${{ matrix.IMAGE }} - # generate Docker tags based on the following events/attributes - tags: | - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} + # # create metadata for image + # - name: Docker meta + # id: meta + # uses: docker/metadata-action@v5 + # with: + # # list of Docker images to use as base name for tags + # images: | + # #pecan/${{ matrix.IMAGE }} + # ghcr.io/${{ github.repository_owner }}/${{ matrix.IMAGE }} + # # generate Docker tags based on the following events/attributes + # tags: | + # type=schedule + # type=ref,event=branch + # type=ref,event=pr + # type=semver,pattern={{version}} + # type=semver,pattern={{major}}.{{minor}} + # type=semver,pattern={{major}} - # setup docker build - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + # # setup docker build + # - name: Set up QEMU + # uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 + # - name: Set up Docker Buildx + # id: buildx + # uses: docker/setup-buildx-action@v3 - - name: Inspect Builder - run: | - echo "Name: ${{ steps.buildx.outputs.name }}" - echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" - echo "Status: ${{ steps.buildx.outputs.status }}" - echo "Flags: ${{ steps.buildx.outputs.flags }}" - echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + # - name: Inspect Builder + # run: | + # echo "Name: ${{ steps.buildx.outputs.name }}" + # echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" + # echo "Status: ${{ steps.buildx.outputs.status }}" + # echo "Flags: ${{ steps.buildx.outputs.flags }}" + # echo "Platforms: ${{ steps.buildx.outputs.platforms }}" - # login to registries - # - name: Login to DockerHub - # uses: docker/login-action@v3 - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} + # # login to registries + # # - name: Login to DockerHub + # # uses: docker/login-action@v3 + # # with: + # # username: ${{ secrets.DOCKERHUB_USERNAME }} + # # password: ${{ secrets.DOCKERHUB_TOKEN }} - # - name: Login to GitHub Container Registry - # uses: docker/login-action@v3 - # with: - # registry: ghcr.io - # username: ${{ github.actor }} - # password: ${{ secrets.GITHUB_TOKEN }} + # # - name: Login to GitHub Container Registry + # # uses: docker/login-action@v3 + # # with: + # # registry: ghcr.io + # # username: ${{ github.actor }} + # # password: ${{ secrets.GITHUB_TOKEN }} - # build the docker images - - name: Build and push depends - uses: docker/build-push-action@v5 - with: - context: . - file: ${{ matrix.DOCKERFILE }} - #push: ${{ github.event_name != 'pull_request' }} - #push: true - platforms: ${{ matrix.PLATFORM }} - # cache-from: type=registry,ref=classtranscribe/frontend-pipeline:buildcache - # cache-to: type=registry,ref=classtranscribe/frontend:buildcache,mode=max - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - VERSION=${{ steps.meta.outputs.version }} - R_VERSION=${{ github.event.inputs.r_version}} + # # build the docker images + # - name: Build and push depends + # uses: docker/build-push-action@v5 + # with: + # context: . + # file: ${{ matrix.DOCKERFILE }} + # #push: ${{ github.event_name != 'pull_request' }} + # #push: true + # platforms: ${{ matrix.PLATFORM }} + # # cache-from: type=registry,ref=classtranscribe/frontend-pipeline:buildcache + # # cache-to: type=registry,ref=classtranscribe/frontend:buildcache,mode=max + # tags: ${{ steps.meta.outputs.tags }} + # labels: ${{ steps.meta.outputs.labels }} + # build-args: | + # VERSION=${{ steps.meta.outputs.version }} + # R_VERSION=${{ github.event.inputs.r_version}} # depends: # name: Docker From c26d6e2d0fe64c6a3e36b0cc9ef7f7bbdc262f61 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 31 Aug 2024 12:00:16 -0500 Subject: [PATCH 09/52] missing space --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index edcd73a372..b86cbdcd9d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-latest permissions: packages: write - strategy: + strategy: fail-fast: false matrix: name: From 9a0634f6859a748ac14a6663e2eb1d7a756609c1 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 31 Aug 2024 12:03:28 -0500 Subject: [PATCH 10/52] remove maintainer --- docker/base/Dockerfile | 1 - docker/data/Dockerfile | 1 - docker/depends/Dockerfile | 2 -- docker/docs/Dockerfile | 1 - docker/executor/Dockerfile | 1 - docker/models/Dockerfile | 1 - docker/rstudio-nginx/Dockerfile | 1 - 7 files changed, 8 deletions(-) diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index 8ec9f3347e..a1fe2231f9 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -2,7 +2,6 @@ ARG IMAGE_VERSION="latest" ARG FROM_IMAGE="depends" FROM pecan/${FROM_IMAGE}:${IMAGE_VERSION} -MAINTAINER Rob Kooper # ---------------------------------------------------------------------- # PEcAn version information diff --git a/docker/data/Dockerfile b/docker/data/Dockerfile index 2ac60cd7e8..e663819fac 100644 --- a/docker/data/Dockerfile +++ b/docker/data/Dockerfile @@ -1,5 +1,4 @@ FROM alpine -MAINTAINER Rob Kooper # name to use in the machines table FQDN when registering the data files ENV FQDN="" \ diff --git a/docker/depends/Dockerfile b/docker/depends/Dockerfile index 8afa79b5ad..6b9dc4eeef 100644 --- a/docker/depends/Dockerfile +++ b/docker/depends/Dockerfile @@ -5,8 +5,6 @@ ARG FROM_IMAGE="rocker/tidyverse" # PECAN FOR MODEL BASE IMAGE # ---------------------------------------------------------------------- FROM ${FROM_IMAGE}:${R_VERSION} -MAINTAINER Rob Kooper - # ---------------------------------------------------------------------- # INSTALL BINARY/LIBRARY DEPENDENCIES diff --git a/docker/docs/Dockerfile b/docker/docs/Dockerfile index 9393365ce2..dcc7955b18 100644 --- a/docker/docs/Dockerfile +++ b/docker/docs/Dockerfile @@ -27,7 +27,6 @@ RUN make build # copy html pages to container # ---------------------------------------------------------------------- FROM httpd -MAINTAINER Rob Kooper COPY docker/docs/index.html /usr/local/apache2/htdocs/ COPY --from=pecandocs /src/book_source/_book/ /usr/local/apache2/htdocs/docs/pecan/ diff --git a/docker/executor/Dockerfile b/docker/executor/Dockerfile index 3472278390..f1e4c22230 100644 --- a/docker/executor/Dockerfile +++ b/docker/executor/Dockerfile @@ -5,7 +5,6 @@ ARG IMAGE_VERSION="latest" # PECAN FOR MODEL BASE IMAGE # ---------------------------------------------------------------------- FROM pecan/base:${IMAGE_VERSION} -MAINTAINER Rob Kooper # ---------------------------------------------------------------------- # SETUP FOR PYTHON CODE diff --git a/docker/models/Dockerfile b/docker/models/Dockerfile index 1870fba9a0..6819ae48b9 100644 --- a/docker/models/Dockerfile +++ b/docker/models/Dockerfile @@ -5,7 +5,6 @@ ARG IMAGE_VERSION="latest" # PECAN FOR MODEL BASE IMAGE # ---------------------------------------------------------------------- FROM pecan/base:${IMAGE_VERSION} -MAINTAINER Rob Kooper # ---------------------------------------------------------------------- # SETUP FOR PYTHON CODE diff --git a/docker/rstudio-nginx/Dockerfile b/docker/rstudio-nginx/Dockerfile index 600c1da648..af33c2c3d7 100644 --- a/docker/rstudio-nginx/Dockerfile +++ b/docker/rstudio-nginx/Dockerfile @@ -1,4 +1,3 @@ FROM nginx:alpine -MAINTAINER Rob Kooper COPY nginx.conf /etc/nginx/conf.d/default.conf From 19fadd47bcbdd4f40c300e62b61ead4c6132597f Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 31 Aug 2024 12:16:16 -0500 Subject: [PATCH 11/52] fix r version --- .github/workflows/docker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b86cbdcd9d..4737a83cfa 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -28,6 +28,8 @@ on: jobs: depends: runs-on: ubuntu-latest + env: + R_VERSION: ${{ github.event.inputs.r_version || '4.1' }} permissions: packages: write strategy: @@ -107,7 +109,7 @@ jobs: outputs: type=image,dest=/tmp/${{ matrix.IMAGE }}.tar build-args: | VERSION=${{ steps.meta.outputs.version }} - R_VERSION=${{ github.event.inputs.r_version}} + R_VERSION=${{ env.R_VERSION }} # so depends image can be used in other steps - name: Upload artifact From 9a4c8ac564d282c98b2ff450e5babb332dc868c4 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 31 Aug 2024 12:21:07 -0500 Subject: [PATCH 12/52] no arm images --- .github/workflows/docker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4737a83cfa..24235c95f6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -39,7 +39,9 @@ jobs: - depends include: - name: depends - PLATFORM: "linux/amd64,linux/arm64" + # no arm images for tidyverse, see + # https://github.com/rocker-org/rocker-versioned2/issues/830 + PLATFORM: "linux/amd64" DOCKERFILE: docker/web/Dockerfile IMAGE: depends From aabf9b9d9c43a2dd6d62a6a84b7b46b842a73ddf Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 31 Aug 2024 13:22:32 -0500 Subject: [PATCH 13/52] add web/dbsync --- .github/workflows/docker.yml | 136 ++++++++--------------------------- 1 file changed, 31 insertions(+), 105 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 24235c95f6..e20b52b2ec 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -23,13 +23,15 @@ on: - 4.1 - 4.2 - 4.3 - + +# set up the environment, either input or default +env: + R_VERSION: ${{ github.event.inputs.r_version || '4.1' }} + # there are 3 jobs to build different images jobs: depends: runs-on: ubuntu-latest - env: - R_VERSION: ${{ github.event.inputs.r_version || '4.1' }} permissions: packages: write strategy: @@ -37,6 +39,8 @@ jobs: matrix: name: - depends + - web + - dbsync include: - name: depends # no arm images for tidyverse, see @@ -44,7 +48,15 @@ jobs: PLATFORM: "linux/amd64" DOCKERFILE: docker/web/Dockerfile IMAGE: depends - + - name: web + PLATFORM: "linux/amd64,linux/arm64" + DOCKERFILE: docker/web/Dockerfile + IMAGE: web + - name: dbsync + PLATFORM: "linux/amd64,linux/arm64" + DOCKERFILE: docker/web/Dockerfile + IMAGE: shiny-dbsync + steps: - uses: actions/checkout@v4 @@ -55,7 +67,7 @@ jobs: with: # list of Docker images to use as base name for tags images: | - #pecan/${{ matrix.IMAGE }} + pecan/${{ matrix.IMAGE }} ghcr.io/${{ github.repository_owner }}/${{ matrix.IMAGE }} # generate Docker tags based on the following events/attributes tags: | @@ -83,18 +95,18 @@ jobs: echo "Platforms: ${{ steps.buildx.outputs.platforms }}" # login to registries - # - name: Login to DockerHub - # uses: docker/login-action@v3 - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - # - name: Login to GitHub Container Registry - # uses: docker/login-action@v3 - # with: - # registry: ghcr.io - # username: ${{ github.actor }} - # password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} # build the docker images - name: Build and push depends @@ -113,101 +125,15 @@ jobs: VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ env.R_VERSION }} - # so depends image can be used in other steps + # upload depends image to be used in other steps - name: Upload artifact + if: ${{ matrix.IMAGE == 'depends' }} uses: actions/upload-artifact@v4 with: name: ${{ matrix.IMAGE }} path: /tmp/${{ matrix.IMAGE }}.tar - # misc: - # runs-on: ubuntu-latest - # permissions: - # packages: write - # strategy: - # fail-fast: false - # matrix: - # name: - # - web - # - dbsync - # include: - # - name: web - # PLATFORM: "linux/amd64,linux/arm64" - # DOCKERFILE: docker/web/Dockerfile - # IMAGE: web - # - name: dbsync - # PLATFORM: "linux/amd64,linux/arm64" - # DOCKERFILE: docker/web/Dockerfile - # IMAGE: shiny-dbsync - - # steps: - # - uses: actions/checkout@v4 - - # # create metadata for image - # - name: Docker meta - # id: meta - # uses: docker/metadata-action@v5 - # with: - # # list of Docker images to use as base name for tags - # images: | - # #pecan/${{ matrix.IMAGE }} - # ghcr.io/${{ github.repository_owner }}/${{ matrix.IMAGE }} - # # generate Docker tags based on the following events/attributes - # tags: | - # type=schedule - # type=ref,event=branch - # type=ref,event=pr - # type=semver,pattern={{version}} - # type=semver,pattern={{major}}.{{minor}} - # type=semver,pattern={{major}} - - # # setup docker build - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v3 - - # - name: Set up Docker Buildx - # id: buildx - # uses: docker/setup-buildx-action@v3 - - # - name: Inspect Builder - # run: | - # echo "Name: ${{ steps.buildx.outputs.name }}" - # echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" - # echo "Status: ${{ steps.buildx.outputs.status }}" - # echo "Flags: ${{ steps.buildx.outputs.flags }}" - # echo "Platforms: ${{ steps.buildx.outputs.platforms }}" - - # # login to registries - # # - name: Login to DockerHub - # # uses: docker/login-action@v3 - # # with: - # # username: ${{ secrets.DOCKERHUB_USERNAME }} - # # password: ${{ secrets.DOCKERHUB_TOKEN }} - - # # - name: Login to GitHub Container Registry - # # uses: docker/login-action@v3 - # # with: - # # registry: ghcr.io - # # username: ${{ github.actor }} - # # password: ${{ secrets.GITHUB_TOKEN }} - - # # build the docker images - # - name: Build and push depends - # uses: docker/build-push-action@v5 - # with: - # context: . - # file: ${{ matrix.DOCKERFILE }} - # #push: ${{ github.event_name != 'pull_request' }} - # #push: true - # platforms: ${{ matrix.PLATFORM }} - # # cache-from: type=registry,ref=classtranscribe/frontend-pipeline:buildcache - # # cache-to: type=registry,ref=classtranscribe/frontend:buildcache,mode=max - # tags: ${{ steps.meta.outputs.tags }} - # labels: ${{ steps.meta.outputs.labels }} - # build-args: | - # VERSION=${{ steps.meta.outputs.version }} - # R_VERSION=${{ github.event.inputs.r_version}} - # depends: +# depends: # name: Docker # # initially we would us on: [release] as well, the problem is that From 54cd1931485db1367733c46ffe8d4288d4d51f7d Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 31 Aug 2024 13:25:47 -0500 Subject: [PATCH 14/52] use password not token --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e20b52b2ec..b00a816ae9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -99,7 +99,7 @@ jobs: uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Login to GitHub Container Registry uses: docker/login-action@v3 From b6146de33790f2ba94fc6fc379eb8dfa6db33120 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 31 Aug 2024 13:34:07 -0500 Subject: [PATCH 15/52] seperate depend and web --- .github/workflows/docker.yml | 107 ++++++++++++++++++++++++++++++----- 1 file changed, 94 insertions(+), 13 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b00a816ae9..2e85498dd0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -39,23 +39,14 @@ jobs: matrix: name: - depends - - web - - dbsync include: - name: depends + CONTEXT: docker/depends + DOCKERFILE: Dockerfile # no arm images for tidyverse, see # https://github.com/rocker-org/rocker-versioned2/issues/830 PLATFORM: "linux/amd64" - DOCKERFILE: docker/web/Dockerfile IMAGE: depends - - name: web - PLATFORM: "linux/amd64,linux/arm64" - DOCKERFILE: docker/web/Dockerfile - IMAGE: web - - name: dbsync - PLATFORM: "linux/amd64,linux/arm64" - DOCKERFILE: docker/web/Dockerfile - IMAGE: shiny-dbsync steps: - uses: actions/checkout@v4 @@ -112,7 +103,8 @@ jobs: - name: Build and push depends uses: docker/build-push-action@v6 with: - context: docker/depends + context: ${{ matrix.CONTEXT }} + file: ${{ matrix.DOCKERFILE }} #push: ${{ github.event_name != 'pull_request' }} #push: true platforms: ${{ matrix.PLATFORM }} @@ -127,12 +119,101 @@ jobs: # upload depends image to be used in other steps - name: Upload artifact - if: ${{ matrix.IMAGE == 'depends' }} uses: actions/upload-artifact@v4 with: name: ${{ matrix.IMAGE }} path: /tmp/${{ matrix.IMAGE }}.tar + web: + runs-on: ubuntu-latest + permissions: + packages: write + strategy: + fail-fast: false + matrix: + name: + - web + - dbsync + include: + - name: web + CONTEXT: . + DOCKERFILE: docker/web/Dockerfile + PLATFORM: "linux/amd64,linux/arm64" + IMAGE: web + - name: dbsync + CONTEXT: . + DOCKERFILE: shiny/dbsync/Dockerfile + PLATFORM: "linux/amd64,linux/arm64" + IMAGE: shiny-dbsync + + steps: + - uses: actions/checkout@v4 + + # create metadata for image + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + # list of Docker images to use as base name for tags + images: | + pecan/${{ matrix.IMAGE }} + ghcr.io/${{ github.repository_owner }}/${{ matrix.IMAGE }} + # generate Docker tags based on the following events/attributes + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + # setup docker build + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Inspect Builder + run: | + echo "Name: ${{ steps.buildx.outputs.name }}" + echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" + echo "Status: ${{ steps.buildx.outputs.status }}" + echo "Flags: ${{ steps.buildx.outputs.flags }}" + echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + + # login to registries + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # build the docker images + - name: Build and push depends + uses: docker/build-push-action@v6 + with: + context: ${{ matrix.CONTEXT }} + file: ${{ matrix.DOCKERFILE }} + #push: ${{ github.event_name != 'pull_request' }} + #push: true + platforms: ${{ matrix.PLATFORM }} + cache-from: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache + cache-to: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache,mode=max + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ steps.meta.outputs.version }} + R_VERSION=${{ env.R_VERSION }} + # depends: # name: Docker From ce9445c451d46833bc7654a2a7c8a92515aa8a5f Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 31 Aug 2024 13:43:14 -0500 Subject: [PATCH 16/52] more containers --- .github/workflows/docker.yml | 53 ++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2e85498dd0..23380ad2a3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -34,19 +34,6 @@ jobs: runs-on: ubuntu-latest permissions: packages: write - strategy: - fail-fast: false - matrix: - name: - - depends - include: - - name: depends - CONTEXT: docker/depends - DOCKERFILE: Dockerfile - # no arm images for tidyverse, see - # https://github.com/rocker-org/rocker-versioned2/issues/830 - PLATFORM: "linux/amd64" - IMAGE: depends steps: - uses: actions/checkout@v4 @@ -58,8 +45,8 @@ jobs: with: # list of Docker images to use as base name for tags images: | - pecan/${{ matrix.IMAGE }} - ghcr.io/${{ github.repository_owner }}/${{ matrix.IMAGE }} + pecan/depends + ghcr.io/${{ github.repository_owner }}/depends # generate Docker tags based on the following events/attributes tags: | type=schedule @@ -103,16 +90,16 @@ jobs: - name: Build and push depends uses: docker/build-push-action@v6 with: - context: ${{ matrix.CONTEXT }} - file: ${{ matrix.DOCKERFILE }} + context: docker/depends + file: Dockerfile #push: ${{ github.event_name != 'pull_request' }} #push: true - platforms: ${{ matrix.PLATFORM }} - cache-from: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache - cache-to: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache,mode=max + platforms: "linux/amd64" + cache-from: type=registry,ref=pecan/depends:buildcache + cache-to: type=registry,ref=pecan/depends:buildcache,mode=max tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,dest=/tmp/${{ matrix.IMAGE }}.tar + outputs: type=image,dest=/tmp/depends.tar build-args: | VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ env.R_VERSION }} @@ -121,10 +108,10 @@ jobs: - name: Upload artifact uses: actions/upload-artifact@v4 with: - name: ${{ matrix.IMAGE }} - path: /tmp/${{ matrix.IMAGE }}.tar + name: depends + path: /tmp/depends.tar - web: + extras: runs-on: ubuntu-latest permissions: packages: write @@ -134,6 +121,9 @@ jobs: name: - web - dbsync + - data + - monitor + - rstudio-nginx include: - name: web CONTEXT: . @@ -145,6 +135,21 @@ jobs: DOCKERFILE: shiny/dbsync/Dockerfile PLATFORM: "linux/amd64,linux/arm64" IMAGE: shiny-dbsync + - name: data + CONTEXT: docker/data + DOCKERFILE: Dockerfile + PLATFORM: "linux/amd64,linux/arm64" + IMAGE: data + - name: monitor + CONTEXT: docker/monitor + DOCKERFILE: Dockerfile + PLATFORM: "linux/amd64,linux/arm64" + IMAGE: monitor + - name: rstudio-nginx + CONTEXT: docker/rstudio-nginx + DOCKERFILE: Dockerfile + PLATFORM: "linux/amd64,linux/arm64" + IMAGE: rstudio-nginx steps: - uses: actions/checkout@v4 From 14165459d89e9163018cf2e76785bda6f2e4fb5d Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 31 Aug 2024 13:52:31 -0500 Subject: [PATCH 17/52] fix path --- .github/workflows/docker.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 23380ad2a3..ff3b219b0e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -91,7 +91,7 @@ jobs: uses: docker/build-push-action@v6 with: context: docker/depends - file: Dockerfile + file: docker/depends/Dockerfile #push: ${{ github.event_name != 'pull_request' }} #push: true platforms: "linux/amd64" @@ -133,21 +133,21 @@ jobs: - name: dbsync CONTEXT: . DOCKERFILE: shiny/dbsync/Dockerfile - PLATFORM: "linux/amd64,linux/arm64" + PLATFORM: "linux/amd64" IMAGE: shiny-dbsync - name: data CONTEXT: docker/data - DOCKERFILE: Dockerfile + DOCKERFILE: docker/data/Dockerfile PLATFORM: "linux/amd64,linux/arm64" IMAGE: data - name: monitor CONTEXT: docker/monitor - DOCKERFILE: Dockerfile + DOCKERFILE: docker/monitor/Dockerfile PLATFORM: "linux/amd64,linux/arm64" IMAGE: monitor - name: rstudio-nginx CONTEXT: docker/rstudio-nginx - DOCKERFILE: Dockerfile + DOCKERFILE: docker/rstudio-nginx/Dockerfile PLATFORM: "linux/amd64,linux/arm64" IMAGE: rstudio-nginx From 5f055ac2b95ee2332d67d82812f9ef13852c217f Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 31 Aug 2024 14:05:05 -0500 Subject: [PATCH 18/52] build base image --- .github/workflows/docker.yml | 93 +++++++++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ff3b219b0e..c07a0185d5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -111,6 +111,97 @@ jobs: name: depends path: /tmp/depends.tar + base: + runs-on: ubuntu-latest + needs: depends + permissions: + packages: write + + steps: + - uses: actions/checkout@v4 + + # load cached image + - name: Retrieve compiled binary + uses: actions/download-artifact@v4 + with: + name: depends + path: /tmp + - name: Load image + run: docker load --input /tmp/depends.tar + + # create metadata for image + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + # list of Docker images to use as base name for tags + images: | + pecan/base + ghcr.io/${{ github.repository_owner }}/base + # generate Docker tags based on the following events/attributes + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + # setup docker build + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Inspect Builder + run: | + echo "Name: ${{ steps.buildx.outputs.name }}" + echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" + echo "Status: ${{ steps.buildx.outputs.status }}" + echo "Flags: ${{ steps.buildx.outputs.flags }}" + echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + + # login to registries + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # build the docker images + - name: Build and push base + uses: docker/build-base-action@v6 + with: + context: . + file: docker/base/Dockerfile + #push: ${{ github.event_name != 'pull_request' }} + #push: true + platforms: "linux/amd64" + cache-from: type=registry,ref=pecan/base:buildcache + cache-to: type=registry,ref=pecan/base:buildcache,mode=max + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,dest=/tmp/base.tar + build-args: | + VERSION=${{ steps.meta.outputs.version }} + R_VERSION=${{ env.R_VERSION }} + + # upload base image to be used in other steps + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: base + path: /tmp/base.tar + extras: runs-on: ubuntu-latest permissions: @@ -203,7 +294,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} # build the docker images - - name: Build and push depends + - name: Build and push ${{ matrix.name }} uses: docker/build-push-action@v6 with: context: ${{ matrix.CONTEXT }} From 03732f690cbb6885c14c773911c79de124880c70 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 31 Aug 2024 14:10:00 -0500 Subject: [PATCH 19/52] wrong replace --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c07a0185d5..b8e327624d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -179,7 +179,7 @@ jobs: # build the docker images - name: Build and push base - uses: docker/build-base-action@v6 + uses: docker/build-push-action@v6 with: context: . file: docker/base/Dockerfile From e48be5b632242508063ba26884c127ab46c4a183 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 31 Aug 2024 14:23:05 -0500 Subject: [PATCH 20/52] fix output --- .github/workflows/docker.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b8e327624d..5a55602a5b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -190,7 +190,7 @@ jobs: cache-to: type=registry,ref=pecan/base:buildcache,mode=max tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,dest=/tmp/base.tar + outputs: type=docker,dest=/tmp/base.tar build-args: | VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ env.R_VERSION }} @@ -199,6 +199,7 @@ jobs: - name: Upload artifact uses: actions/upload-artifact@v4 with: + if-no-files-found: error name: base path: /tmp/base.tar From f1b0040854bde28e90410837d89fb865e4cb074f Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 31 Aug 2024 15:36:57 -0500 Subject: [PATCH 21/52] changed base not depends --- .github/workflows/docker.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5a55602a5b..84d46382b4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -99,7 +99,7 @@ jobs: cache-to: type=registry,ref=pecan/depends:buildcache,mode=max tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,dest=/tmp/depends.tar + outputs: type=docker,dest=/tmp/depends.tar build-args: | VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ env.R_VERSION }} @@ -108,6 +108,7 @@ jobs: - name: Upload artifact uses: actions/upload-artifact@v4 with: + if-no-files-found: error name: depends path: /tmp/depends.tar From 119a63bf68c93cbf438231d84d6748e76dd0190f Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 31 Aug 2024 15:54:05 -0500 Subject: [PATCH 22/52] push and variables --- .github/workflows/docker.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 84d46382b4..e64ee4d2d9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -30,6 +30,9 @@ env: # there are 3 jobs to build different images jobs: + # ---------------------------------------------------------------------- + # depends image has all the dependencies installed + # ---------------------------------------------------------------------- depends: runs-on: ubuntu-latest permissions: @@ -93,7 +96,7 @@ jobs: context: docker/depends file: docker/depends/Dockerfile #push: ${{ github.event_name != 'pull_request' }} - #push: true + push: true platforms: "linux/amd64" cache-from: type=registry,ref=pecan/depends:buildcache cache-to: type=registry,ref=pecan/depends:buildcache,mode=max @@ -103,6 +106,7 @@ jobs: build-args: | VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ env.R_VERSION }} + GITHUB_PAT=${{ secrets.GITHUB_TOKEN }} # upload depends image to be used in other steps - name: Upload artifact @@ -112,6 +116,9 @@ jobs: name: depends path: /tmp/depends.tar + # ---------------------------------------------------------------------- + # base image has PEcAn compiled and installed, and depends on depends + # ---------------------------------------------------------------------- base: runs-on: ubuntu-latest needs: depends @@ -185,7 +192,7 @@ jobs: context: . file: docker/base/Dockerfile #push: ${{ github.event_name != 'pull_request' }} - #push: true + push: true platforms: "linux/amd64" cache-from: type=registry,ref=pecan/base:buildcache cache-to: type=registry,ref=pecan/base:buildcache,mode=max @@ -195,7 +202,12 @@ jobs: build-args: | VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ env.R_VERSION }} - + GITHUB_PAT=${{ secrets.GITHUB_TOKEN }} + PECAN_VERSION=${{ steps.meta.outputs.version }} + PECAN_GIT_BRANCH= ${{ github.head_ref || github.ref_name }} + PECAN_GIT_CHECKSUM=${{ github.sha }} + PECAN_GIT_DATE=${{ github.event.repository.updated_at }} + # upload base image to be used in other steps - name: Upload artifact uses: actions/upload-artifact@v4 @@ -204,6 +216,9 @@ jobs: name: base path: /tmp/base.tar + # ---------------------------------------------------------------------- + # Next are images that do not depend on either depends or base image + # ---------------------------------------------------------------------- extras: runs-on: ubuntu-latest permissions: From 02a29381965f9cceb11e24f6a32dd829f0892ae3 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 31 Aug 2024 15:58:35 -0500 Subject: [PATCH 23/52] add repository --- .github/workflows/docker.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e64ee4d2d9..e7abf7559c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -49,7 +49,7 @@ jobs: # list of Docker images to use as base name for tags images: | pecan/depends - ghcr.io/${{ github.repository_owner }}/depends + ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/depends # generate Docker tags based on the following events/attributes tags: | type=schedule @@ -145,7 +145,7 @@ jobs: # list of Docker images to use as base name for tags images: | pecan/base - ghcr.io/${{ github.repository_owner }}/base + ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/base # generate Docker tags based on the following events/attributes tags: | type=schedule @@ -270,7 +270,7 @@ jobs: # list of Docker images to use as base name for tags images: | pecan/${{ matrix.IMAGE }} - ghcr.io/${{ github.repository_owner }}/${{ matrix.IMAGE }} + ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/${{ matrix.IMAGE }} # generate Docker tags based on the following events/attributes tags: | type=schedule From cf4ee5cb42bfa96abb2aa7988d97a415cb6ae6b4 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 31 Aug 2024 16:05:41 -0500 Subject: [PATCH 24/52] update base dockerfile --- docker/base/Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index a1fe2231f9..abbffd47c9 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -25,9 +25,7 @@ COPY models /pecan/models/ # install all PEcAn packages # `make clean` is to remove artifacts copied in from host system # (e.g. basgra.so) -RUN --mount=type=secret,id=github_token \ - export GITHUB_PAT=`cat /run/secrets/github_token` \ - && cd /pecan \ +RUN cd /pecan \ && make clean \ && make \ && rm -rf /tmp/downloaded_packages @@ -37,7 +35,7 @@ WORKDIR /work COPY web/workflow.R docker/base/rstudio.sh /work/ # COMMAND TO RUN -CMD Rscript --vanilla workflow.R | tee workflow.Rout +CMD ["bash", "-c", "Rscript --vanilla workflow.R | tee workflow.Rout"] # variables to store in docker image ENV PECAN_VERSION=${PECAN_VERSION} \ From c250e0167ce7fd64467809ffb9f466d15f806e82 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 31 Aug 2024 16:15:35 -0500 Subject: [PATCH 25/52] specify image info --- .github/workflows/docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e7abf7559c..38d0fbd861 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -202,6 +202,8 @@ jobs: build-args: | VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ env.R_VERSION }} + FROM_IMAGE=pecan/depends + IMAGE_VERSION=${{ steps.meta.outputs.version }} GITHUB_PAT=${{ secrets.GITHUB_TOKEN }} PECAN_VERSION=${{ steps.meta.outputs.version }} PECAN_GIT_BRANCH= ${{ github.head_ref || github.ref_name }} From 592e25fbed1422fab432f67db84818d312fc96f5 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 31 Aug 2024 16:20:08 -0500 Subject: [PATCH 26/52] just depends --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 38d0fbd861..8d20ab87fa 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -202,7 +202,7 @@ jobs: build-args: | VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ env.R_VERSION }} - FROM_IMAGE=pecan/depends + FROM_IMAGE=depends IMAGE_VERSION=${{ steps.meta.outputs.version }} GITHUB_PAT=${{ secrets.GITHUB_TOKEN }} PECAN_VERSION=${{ steps.meta.outputs.version }} From c7a4fb53448302cc6bb331a6bc7ca51f3addf9f4 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sun, 1 Sep 2024 16:29:46 -0500 Subject: [PATCH 27/52] Don't install suggested sub-packages. --- scripts/confirm_deps.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/confirm_deps.R b/scripts/confirm_deps.R index d7968292bc..98d19b22b5 100755 --- a/scripts/confirm_deps.R +++ b/scripts/confirm_deps.R @@ -78,7 +78,7 @@ confirm_deps <- function(pkg, return( remotes::install_deps( pkg = pkg, - dependencies = dependencies, + dependencies = NA, ... ) ) From 513fc1970ff4c52544ee727776b268af7fd1a339 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sun, 1 Sep 2024 16:45:52 -0500 Subject: [PATCH 28/52] docs/models/executor --- .github/workflows/docker.yml | 109 +++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8d20ab87fa..e486f58502 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -218,6 +218,115 @@ jobs: name: base path: /tmp/base.tar + # ---------------------------------------------------------------------- + # Next are images that depend on base image + # ---------------------------------------------------------------------- + baseplus: + runs-on: ubuntu-latest + needs: base + permissions: + packages: write + strategy: + fail-fast: false + matrix: + name: + - docs + - models + - executor + include: + - name: docs + CONTEXT: . + DOCKERFILE: docker/docs/Dockerfile + PLATFORM: "linux/amd64" + IMAGE: docs + - name: models + CONTEXT: docker/models + DOCKERFILE: docker/models/Dockerfile + PLATFORM: "linux/amd64" + IMAGE: models + - name: executor + CONTEXT: docker/executor + DOCKERFILE: docker/executor/Dockerfile + PLATFORM: "linux/amd64" + IMAGE: executor + + steps: + - uses: actions/checkout@v4 + + # load cached image + - name: Retrieve compiled binary + uses: actions/download-artifact@v4 + with: + name: base + path: /tmp + - name: Load image + run: docker load --input /tmp/base.tar + + # create metadata for image + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + # list of Docker images to use as base name for tags + images: | + pecan/${{ matrix.IMAGE }} + ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/${{ matrix.IMAGE }} + # generate Docker tags based on the following events/attributes + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + # setup docker build + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Inspect Builder + run: | + echo "Name: ${{ steps.buildx.outputs.name }}" + echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" + echo "Status: ${{ steps.buildx.outputs.status }}" + echo "Flags: ${{ steps.buildx.outputs.flags }}" + echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + + # login to registries + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # build the docker images + - name: Build and push ${{ matrix.name }} + uses: docker/build-push-action@v6 + with: + context: ${{ matrix.CONTEXT }} + file: ${{ matrix.DOCKERFILE }} + #push: ${{ github.event_name != 'pull_request' }} + #push: true + platforms: ${{ matrix.PLATFORM }} + cache-from: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache + cache-to: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache,mode=max + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ steps.meta.outputs.version }} + R_VERSION=${{ env.R_VERSION }} + # ---------------------------------------------------------------------- # Next are images that do not depend on either depends or base image # ---------------------------------------------------------------------- From 73a1e9400440cc3d50adb2c5e8e338c9300d6e6d Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sun, 1 Sep 2024 17:03:19 -0500 Subject: [PATCH 29/52] build models --- .github/workflows/docker.yml | 123 +++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e486f58502..67d5d07917 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -327,6 +327,129 @@ jobs: VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ env.R_VERSION }} +# ---------------------------------------------------------------------- +# Next are images that have models installed +# ---------------------------------------------------------------------- + models: + runs-on: ubuntu-latest + needs: base + permissions: + packages: write + strategy: + fail-fast: false + matrix: + include: + - CONTEXT: models/basgra + DOCKERFILE: models/basgra/Dockerfile + PLATFORM: "linux/amd64" + MODEL: basgra + VERSION: BASGRA_N_v1 + - CONTEXT: models/biocro + DOCKERFILE: models/biocro/Dockerfile + PLATFORM: "linux/amd64" + MODEL: biocro + VERSION: "0.95" + - CONTEXT: models/ed2 + DOCKERFILE: models/ed2/Dockerfile + PLATFORM: "linux/amd64" + MODEL: ed2 + VERSION: "2.2.0" + - CONTEXT: models/ed2 + DOCKERFILE: models/ed2/Dockerfile + PLATFORM: "linux/amd64" + MODEL: ed2 + VERSION: "git" + - CONTEXT: models/maespa + DOCKERFILE: models/maespa/Dockerfile + PLATFORM: "linux/amd64" + MODEL: maespa + VERSION: "git" + - CONTEXT: models/sipnet + DOCKERFILE: models/sipnet/Dockerfile + PLATFORM: "linux/amd64" + MODEL: sipnet + VERSION: "git" + + steps: + - uses: actions/checkout@v4 + + # load cached image + - name: Retrieve compiled binary + uses: actions/download-artifact@v4 + with: + name: base + path: /tmp + - name: Load image + run: docker load --input /tmp/base.tar + + # create metadata for image + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + # list of Docker images to use as base name for tags + images: | + pecan/model-${{ matrix.MODEL }}-${{ matrix.VERSION }} + ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/model-${{ matrix.MODEL }}-${{ matrix.VERSION }} + # generate Docker tags based on the following events/attributes + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + # setup docker build + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Inspect Builder + run: | + echo "Name: ${{ steps.buildx.outputs.name }}" + echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" + echo "Status: ${{ steps.buildx.outputs.status }}" + echo "Flags: ${{ steps.buildx.outputs.flags }}" + echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + + # login to registries + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # build the docker images + - name: Build and push ${{ matrix.MODEL }} ${{ matrix.VERSION }} + uses: docker/build-push-action@v6 + with: + context: ${{ matrix.CONTEXT }} + file: ${{ matrix.DOCKERFILE }} + #push: ${{ github.event_name != 'pull_request' }} + #push: true + platforms: ${{ matrix.PLATFORM }} + cache-from: type=registry,ref=pecan/model-${{ matrix.MODEL }}-${{ matrix.VERSION }}:buildcache + cache-to: type=registry,ref=pecan/model-${{ matrix.MODEL }}-${{ matrix.VERSION }}:buildcache,mode=max + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ steps.meta.outputs.version }} + R_VERSION=${{ env.R_VERSION }} + MODEL_VERSION=${{ matrix.VERSION }} + IMAGE_VERSION=${{ steps.meta.outputs.version }} + BINARY_VERSION="2.2" + # ---------------------------------------------------------------------- # Next are images that do not depend on either depends or base image # ---------------------------------------------------------------------- From 8ab28a9705dfc98bb8f51790cfa8ce19dd21d701 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sun, 1 Sep 2024 17:05:27 -0500 Subject: [PATCH 30/52] depends on models --- .github/workflows/docker.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 67d5d07917..4e5c688b1e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -323,16 +323,26 @@ jobs: cache-to: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache,mode=max tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + outputs: type=docker,dest=/tmp/${{ matrix.name }}.tar build-args: | VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ env.R_VERSION }} + # upload base image to be used in other steps + - name: Upload artifact + if: ${{ matrix.name == 'models' }} + uses: actions/upload-artifact@v4 + with: + if-no-files-found: error + name: models + path: /tmp/models.tar + # ---------------------------------------------------------------------- # Next are images that have models installed # ---------------------------------------------------------------------- models: runs-on: ubuntu-latest - needs: base + needs: baseplus permissions: packages: write strategy: @@ -377,10 +387,10 @@ jobs: - name: Retrieve compiled binary uses: actions/download-artifact@v4 with: - name: base + name: models path: /tmp - name: Load image - run: docker load --input /tmp/base.tar + run: docker load --input /tmp/models.tar # create metadata for image - name: Docker meta From de4c42a5e52f2fc4ab26f522807a5ee8de7ae23f Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sun, 1 Sep 2024 18:06:25 -0500 Subject: [PATCH 31/52] lower case version --- .github/workflows/docker.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4e5c688b1e..d35832b8c5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -354,6 +354,7 @@ jobs: PLATFORM: "linux/amd64" MODEL: basgra VERSION: BASGRA_N_v1 + VERSION_LOWER: basgra_n_v1 - CONTEXT: models/biocro DOCKERFILE: models/biocro/Dockerfile PLATFORM: "linux/amd64" @@ -400,7 +401,7 @@ jobs: # list of Docker images to use as base name for tags images: | pecan/model-${{ matrix.MODEL }}-${{ matrix.VERSION }} - ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/model-${{ matrix.MODEL }}-${{ matrix.VERSION }} + ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/model-${{ matrix.MODEL }}-${{ matrix.VERSION_LOWER || matrix.VERSION }} # generate Docker tags based on the following events/attributes tags: | type=schedule @@ -440,6 +441,9 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: docker image ls + run: docker image ls + # build the docker images - name: Build and push ${{ matrix.MODEL }} ${{ matrix.VERSION }} uses: docker/build-push-action@v6 From 13417f018fe403dd5f9b42a9386c980c6430b5ff Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sun, 1 Sep 2024 18:26:38 -0500 Subject: [PATCH 32/52] fix ed2 image, always push --- .github/workflows/docker.yml | 157 ++--------------------------------- 1 file changed, 7 insertions(+), 150 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d35832b8c5..0217b7c99d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -95,7 +95,6 @@ jobs: with: context: docker/depends file: docker/depends/Dockerfile - #push: ${{ github.event_name != 'pull_request' }} push: true platforms: "linux/amd64" cache-from: type=registry,ref=pecan/depends:buildcache @@ -191,7 +190,6 @@ jobs: with: context: . file: docker/base/Dockerfile - #push: ${{ github.event_name != 'pull_request' }} push: true platforms: "linux/amd64" cache-from: type=registry,ref=pecan/base:buildcache @@ -316,8 +314,7 @@ jobs: with: context: ${{ matrix.CONTEXT }} file: ${{ matrix.DOCKERFILE }} - #push: ${{ github.event_name != 'pull_request' }} - #push: true + push: true platforms: ${{ matrix.PLATFORM }} cache-from: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache cache-to: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache,mode=max @@ -360,13 +357,13 @@ jobs: PLATFORM: "linux/amd64" MODEL: biocro VERSION: "0.95" - - CONTEXT: models/ed2 - DOCKERFILE: models/ed2/Dockerfile + - CONTEXT: models/ed + DOCKERFILE: models/ed/Dockerfile PLATFORM: "linux/amd64" MODEL: ed2 VERSION: "2.2.0" - - CONTEXT: models/ed2 - DOCKERFILE: models/ed2/Dockerfile + - CONTEXT: models/ed + DOCKERFILE: models/ed/Dockerfile PLATFORM: "linux/amd64" MODEL: ed2 VERSION: "git" @@ -450,8 +447,7 @@ jobs: with: context: ${{ matrix.CONTEXT }} file: ${{ matrix.DOCKERFILE }} - #push: ${{ github.event_name != 'pull_request' }} - #push: true + push: true platforms: ${{ matrix.PLATFORM }} cache-from: type=registry,ref=pecan/model-${{ matrix.MODEL }}-${{ matrix.VERSION }}:buildcache cache-to: type=registry,ref=pecan/model-${{ matrix.MODEL }}-${{ matrix.VERSION }}:buildcache,mode=max @@ -564,8 +560,7 @@ jobs: with: context: ${{ matrix.CONTEXT }} file: ${{ matrix.DOCKERFILE }} - #push: ${{ github.event_name != 'pull_request' }} - #push: true + push: true platforms: ${{ matrix.PLATFORM }} cache-from: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache cache-to: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache,mode=max @@ -574,141 +569,3 @@ jobs: build-args: | VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ env.R_VERSION }} - -# depends: -# name: Docker - -# # initially we would us on: [release] as well, the problem is that -# # the code in clowder would not know what branch the code is in, -# # and would not set the right version flags. - -# # This will run when: -# # - when new code is pushed to master/develop to push the tags -# # latest and develop -# # - when a pull request is created and updated to make sure the -# # Dockerfile is still valid. -# # To be able to push to dockerhub, this expects the following -# # secrets to be set in the project: -# # - DOCKERHUB_USERNAME : username that can push to the org -# # - DOCKERHUB_PASSWORD : password asscoaited with the username -# on: -# push: -# branches: -# - master -# - develop -# pull_request: -# merge_group: -# issue_comment: -# types: -# - created -# workflow_dispatch: - -# # Certain actions will only run when this is the master repo. -# env: -# MASTER_REPO: PecanProject/pecan -# DOCKERHUB_ORG: pecan -# GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - -# jobs: -# docker: -# if: github.event_name != 'issue_comment' || startsWith(github.event.comment.body, '/build') -# runs-on: ubuntu-latest - -# steps: -# - name: Work around https://github.com/actions/checkout/issues/766 -# run: git config --global --add safe.directory "$GITHUB_WORKSPACE" -# - uses: actions/checkout@v4 -# with: -# set-safe-directory: false -# - name: Set up Docker Buildx -# uses: docker/setup-buildx-action@v3 -# with: -# driver: docker -# buildkitd-flags: --debug -# install: true - -# # calculate some variables that are used later -# - name: get version tag -# run: | -# BRANCH=${GITHUB_REF##*/} -# echo "GITHUB_BRANCH=${BRANCH}" >> $GITHUB_ENV -# if [ "$BRANCH" == "master" ]; then -# version="$(awk '/Version:/ { print $2 }' base/all/DESCRIPTION)" -# tags="latest" -# oldversion="" -# while [ "${oldversion}" != "${version}" ]; do -# oldversion="${version}" -# tags="${tags},${version}" -# version=${version%.*} -# done -# echo "PECAN_VERSION=$(awk '/Version:/ { print $2 }' base/all/DESCRIPTION)" >> $GITHUB_ENV -# echo "PECAN_TAGS=${tags}" >> $GITHUB_ENV -# elif [ "$BRANCH" == "develop" ]; then -# echo "PECAN_VERSION=develop" >> $GITHUB_ENV -# echo "PECAN_TAGS=develop" >> $GITHUB_ENV -# else -# echo "PECAN_VERSION=develop" >> $GITHUB_ENV -# echo "PECAN_TAGS=develop" >> $GITHUB_ENV -# fi -# # If any dependencies changed in this PR, try to update depends image -# - name: check for changed dependencies -# uses: dorny/paths-filter@v3 -# id: findchanges -# with: -# filters: | -# deps: -# - docker/depends/** -# - if: steps.findchanges.outputs.deps == 'true' -# # NB: GITHUB_BASE_REF is only set on pull requests, -# # so non-PR builds will find no existing tag and build fresh -# run: | -# if $(docker manifest inspect pecan/depends:${GITHUB_BASE_REF##*/} > /dev/null 2>&1); then -# echo "UPDATE_DEPENDS_FROM_TAG=${GITHUB_BASE_REF##*/}" >> $GITHUB_ENV -# else -# echo "BUILD_DEPENDS_FRESH=true" >> GITHUB_ENV -# fi -# env: -# GITHUB_BASE_REF: ${{ github.base_ref }} -# # use shell script to build, there is some complexity in this -# - name: create images -# run: ./docker.sh -i github -# env: -# PECAN_GIT_CHECKSUM: ${{ github.sha }} -# PECAN_GIT_BRANCH: ${GITHUB_BRANCH} -# VERSION: ${{ env.PECAN_VERSION }} -# UPDATE_DEPENDS_FROM_TAG: ${{ env.UPDATE_DEPENDS_FROM_TAG }} -# BUILD: ${{ env.BUILD_DEPENDS_FRESH }} - -# # push all images to github -# - name: Publish to GitHub -# if: github.event_name == 'push' && github.repository == env.MASTER_REPO -# run: | -# echo "${INPUT_PASSWORD}" | docker login -u ${INPUT_USERNAME} --password-stdin ${INPUT_REGISTRY} -# repo=$(echo ${{ github.repository_owner }} | tr 'A-Z' 'a-z') -# for image in $(docker image ls pecan/*:github --format "{{ .Repository }}"); do -# for v in ${PECAN_TAGS}; do -# docker tag ${image}:github ${INPUT_REGISTRY}/${repo}/${image#pecan/}:${v} -# docker push ${INPUT_REGISTRY}/${repo}/${image#pecan/}:${v} -# done -# done -# docker logout -# env: -# INPUT_REGISTRY: ghcr.io -# INPUT_USERNAME: ${{ secrets.GHCR_USERNAME }} -# INPUT_PASSWORD: ${{ secrets.GHCR_PASSWORD }} - -# # push all images to dockerhub -# - name: Publish to DockerHub -# if: github.event_name == 'push' && github.repository == env.MASTER_REPO -# run: | -# echo "${INPUT_PASSWORD}" | docker login -u ${INPUT_USERNAME} --password-stdin -# for image in $(docker image ls pecan/*:github --format "{{ .Repository }}"); do -# for v in ${PECAN_TAGS}; do -# docker tag ${image}:github ${{ env.DOCKERHUB_ORG }}/${image#pecan/}:${v} -# docker push ${{ env.DOCKERHUB_ORG }}/${image#pecan/}:${v} -# done -# done -# docker logout -# env: -# INPUT_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} -# INPUT_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} From 41aa5146b95700870c88dd5f446b448811d2962a Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sun, 1 Sep 2024 18:47:10 -0500 Subject: [PATCH 33/52] pull models out --- .github/workflows/docker.yml | 185 ++++++++++++++++++++++++++--------- 1 file changed, 140 insertions(+), 45 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0217b7c99d..2bf907ce98 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -216,37 +216,14 @@ jobs: name: base path: /tmp/base.tar - # ---------------------------------------------------------------------- - # Next are images that depend on base image - # ---------------------------------------------------------------------- - baseplus: +# ---------------------------------------------------------------------- +# models image has some python installed to run models, depends on base +# ---------------------------------------------------------------------- + models: runs-on: ubuntu-latest needs: base permissions: packages: write - strategy: - fail-fast: false - matrix: - name: - - docs - - models - - executor - include: - - name: docs - CONTEXT: . - DOCKERFILE: docker/docs/Dockerfile - PLATFORM: "linux/amd64" - IMAGE: docs - - name: models - CONTEXT: docker/models - DOCKERFILE: docker/models/Dockerfile - PLATFORM: "linux/amd64" - IMAGE: models - - name: executor - CONTEXT: docker/executor - DOCKERFILE: docker/executor/Dockerfile - PLATFORM: "linux/amd64" - IMAGE: executor steps: - uses: actions/checkout@v4 @@ -259,16 +236,16 @@ jobs: path: /tmp - name: Load image run: docker load --input /tmp/base.tar - + # create metadata for image - name: Docker meta id: meta uses: docker/metadata-action@v5 with: - # list of Docker images to use as base name for tags + # list of Docker images to use as models name for tags images: | - pecan/${{ matrix.IMAGE }} - ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/${{ matrix.IMAGE }} + pecan/models + ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/models # generate Docker tags based on the following events/attributes tags: | type=schedule @@ -309,37 +286,43 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} # build the docker images - - name: Build and push ${{ matrix.name }} + - name: Build and push models uses: docker/build-push-action@v6 with: - context: ${{ matrix.CONTEXT }} - file: ${{ matrix.DOCKERFILE }} + context: docker/models + file: docker/models/Dockerfile push: true - platforms: ${{ matrix.PLATFORM }} - cache-from: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache - cache-to: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache,mode=max + platforms: "linux/amd64" + cache-from: type=registry,ref=pecan/models:buildcache + cache-to: type=registry,ref=pecan/models:buildcache,mode=max tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - outputs: type=docker,dest=/tmp/${{ matrix.name }}.tar + outputs: type=docker,dest=/tmp/models.tar build-args: | VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ env.R_VERSION }} - - # upload base image to be used in other steps + FROM_IMAGE=depends + IMAGE_VERSION=${{ steps.meta.outputs.version }} + GITHUB_PAT=${{ secrets.GITHUB_TOKEN }} + PECAN_VERSION=${{ steps.meta.outputs.version }} + PECAN_GIT_BRANCH= ${{ github.head_ref || github.ref_name }} + PECAN_GIT_CHECKSUM=${{ github.sha }} + PECAN_GIT_DATE=${{ github.event.repository.updated_at }} + + # upload models image to be used in other steps - name: Upload artifact - if: ${{ matrix.name == 'models' }} uses: actions/upload-artifact@v4 with: if-no-files-found: error name: models path: /tmp/models.tar - + # ---------------------------------------------------------------------- # Next are images that have models installed # ---------------------------------------------------------------------- - models: + modelsbinary: runs-on: ubuntu-latest - needs: baseplus + needs: models permissions: packages: write strategy: @@ -397,7 +380,7 @@ jobs: with: # list of Docker images to use as base name for tags images: | - pecan/model-${{ matrix.MODEL }}-${{ matrix.VERSION }} + pecan/model-${{ matrix.MODEL }}-${{ matrix.VERSION_LOWER || matrix.VERSION }} ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/model-${{ matrix.MODEL }}-${{ matrix.VERSION_LOWER || matrix.VERSION }} # generate Docker tags based on the following events/attributes tags: | @@ -459,7 +442,119 @@ jobs: MODEL_VERSION=${{ matrix.VERSION }} IMAGE_VERSION=${{ steps.meta.outputs.version }} BINARY_VERSION="2.2" + + # ---------------------------------------------------------------------- + # Next are images that depend on base image + # ---------------------------------------------------------------------- + baseplus: + runs-on: ubuntu-latest + needs: base + permissions: + packages: write + strategy: + fail-fast: false + matrix: + name: + - docs + - executor + include: + - name: docs + CONTEXT: . + DOCKERFILE: docker/docs/Dockerfile + PLATFORM: "linux/amd64" + IMAGE: docs + - name: executor + CONTEXT: docker/executor + DOCKERFILE: docker/executor/Dockerfile + PLATFORM: "linux/amd64" + IMAGE: executor + + steps: + - uses: actions/checkout@v4 + + # load cached image + - name: Retrieve compiled binary + uses: actions/download-artifact@v4 + with: + name: base + path: /tmp + - name: Load image + run: docker load --input /tmp/base.tar + # create metadata for image + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + # list of Docker images to use as base name for tags + images: | + pecan/${{ matrix.IMAGE }} + ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/${{ matrix.IMAGE }} + # generate Docker tags based on the following events/attributes + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + # setup docker build + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Inspect Builder + run: | + echo "Name: ${{ steps.buildx.outputs.name }}" + echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" + echo "Status: ${{ steps.buildx.outputs.status }}" + echo "Flags: ${{ steps.buildx.outputs.flags }}" + echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + + # login to registries + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # build the docker images + - name: Build and push ${{ matrix.name }} + uses: docker/build-push-action@v6 + with: + context: ${{ matrix.CONTEXT }} + file: ${{ matrix.DOCKERFILE }} + push: true + platforms: ${{ matrix.PLATFORM }} + cache-from: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache + cache-to: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache,mode=max + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=docker,dest=/tmp/${{ matrix.name }}.tar + build-args: | + VERSION=${{ steps.meta.outputs.version }} + R_VERSION=${{ env.R_VERSION }} + + # upload base image to be used in other steps + - name: Upload artifact + if: ${{ matrix.name == 'models' }} + uses: actions/upload-artifact@v4 + with: + if-no-files-found: error + name: models + path: /tmp/models.tar + # ---------------------------------------------------------------------- # Next are images that do not depend on either depends or base image # ---------------------------------------------------------------------- From 1870b607bcd29e70d056ea18fbb857f43af348f3 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sun, 1 Sep 2024 22:11:28 -0500 Subject: [PATCH 34/52] lower case, docker push/pull --- .github/workflows/docker.yml | 161 ++++++++++++++++++----------------- 1 file changed, 84 insertions(+), 77 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2bf907ce98..602cec6555 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -101,19 +101,19 @@ jobs: cache-to: type=registry,ref=pecan/depends:buildcache,mode=max tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - outputs: type=docker,dest=/tmp/depends.tar + # outputs: type=docker,dest=/tmp/depends.tar build-args: | VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ env.R_VERSION }} GITHUB_PAT=${{ secrets.GITHUB_TOKEN }} - # upload depends image to be used in other steps - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - if-no-files-found: error - name: depends - path: /tmp/depends.tar + # # upload depends image to be used in other steps + # - name: Upload artifact + # uses: actions/upload-artifact@v4 + # with: + # if-no-files-found: error + # name: depends + # path: /tmp/depends.tar # ---------------------------------------------------------------------- # base image has PEcAn compiled and installed, and depends on depends @@ -127,14 +127,14 @@ jobs: steps: - uses: actions/checkout@v4 - # load cached image - - name: Retrieve compiled binary - uses: actions/download-artifact@v4 - with: - name: depends - path: /tmp - - name: Load image - run: docker load --input /tmp/depends.tar + # # load cached image + # - name: Retrieve compiled binary + # uses: actions/download-artifact@v4 + # with: + # name: depends + # path: /tmp + # - name: Load image + # run: docker load --input /tmp/depends.tar # create metadata for image - name: Docker meta @@ -196,7 +196,7 @@ jobs: cache-to: type=registry,ref=pecan/base:buildcache,mode=max tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - outputs: type=docker,dest=/tmp/base.tar + # outputs: type=docker,dest=/tmp/base.tar build-args: | VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ env.R_VERSION }} @@ -208,13 +208,13 @@ jobs: PECAN_GIT_CHECKSUM=${{ github.sha }} PECAN_GIT_DATE=${{ github.event.repository.updated_at }} - # upload base image to be used in other steps - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - if-no-files-found: error - name: base - path: /tmp/base.tar + # # upload base image to be used in other steps + # - name: Upload artifact + # uses: actions/upload-artifact@v4 + # with: + # if-no-files-found: error + # name: base + # path: /tmp/base.tar # ---------------------------------------------------------------------- # models image has some python installed to run models, depends on base @@ -228,14 +228,14 @@ jobs: steps: - uses: actions/checkout@v4 - # load cached image - - name: Retrieve compiled binary - uses: actions/download-artifact@v4 - with: - name: base - path: /tmp - - name: Load image - run: docker load --input /tmp/base.tar + # # load cached image + # - name: Retrieve compiled binary + # uses: actions/download-artifact@v4 + # with: + # name: base + # path: /tmp + # - name: Load image + # run: docker load --input /tmp/base.tar # create metadata for image - name: Docker meta @@ -297,7 +297,7 @@ jobs: cache-to: type=registry,ref=pecan/models:buildcache,mode=max tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - outputs: type=docker,dest=/tmp/models.tar + # outputs: type=docker,dest=/tmp/models.tar build-args: | VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ env.R_VERSION }} @@ -309,13 +309,13 @@ jobs: PECAN_GIT_CHECKSUM=${{ github.sha }} PECAN_GIT_DATE=${{ github.event.repository.updated_at }} - # upload models image to be used in other steps - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - if-no-files-found: error - name: models - path: /tmp/models.tar + # # upload models image to be used in other steps + # - name: Upload artifact + # uses: actions/upload-artifact@v4 + # with: + # if-no-files-found: error + # name: models + # path: /tmp/models.tar # ---------------------------------------------------------------------- # Next are images that have models installed @@ -328,34 +328,46 @@ jobs: strategy: fail-fast: false matrix: + name: + - basgra + - biocro + - ed2_2.2.0 + - ed2_git + - maespa_git + - sipnet_git include: - - CONTEXT: models/basgra + - name: basgra + CONTEXT: models/basgra DOCKERFILE: models/basgra/Dockerfile PLATFORM: "linux/amd64" MODEL: basgra VERSION: BASGRA_N_v1 - VERSION_LOWER: basgra_n_v1 - - CONTEXT: models/biocro + - name: biocro + CONTEXT: models/biocro DOCKERFILE: models/biocro/Dockerfile PLATFORM: "linux/amd64" MODEL: biocro VERSION: "0.95" - - CONTEXT: models/ed + - name: ed2_2.2.0 + CONTEXT: models/ed DOCKERFILE: models/ed/Dockerfile PLATFORM: "linux/amd64" MODEL: ed2 VERSION: "2.2.0" - - CONTEXT: models/ed + - name: ed2_git + CONTEXT: models/ed DOCKERFILE: models/ed/Dockerfile PLATFORM: "linux/amd64" MODEL: ed2 VERSION: "git" - - CONTEXT: models/maespa + - name: maespa + CONTEXT: models/maespa DOCKERFILE: models/maespa/Dockerfile PLATFORM: "linux/amd64" MODEL: maespa VERSION: "git" - - CONTEXT: models/sipnet + - name: sipnet + CONTEXT: models/sipnet DOCKERFILE: models/sipnet/Dockerfile PLATFORM: "linux/amd64" MODEL: sipnet @@ -364,14 +376,19 @@ jobs: steps: - uses: actions/checkout@v4 - # load cached image - - name: Retrieve compiled binary - uses: actions/download-artifact@v4 - with: - name: models - path: /tmp - - name: Load image - run: docker load --input /tmp/models.tar + # # load cached image + # - name: Retrieve compiled binary + # uses: actions/download-artifact@v4 + # with: + # name: models + # path: /tmp + # - name: Load image + # run: docker load --input /tmp/models.tar + + # lower case name for docker + - name: docker image name + id: lower + run: echo "image_name=$(echo model-${{ matrix.MODEL }}-${{ matrix.VERSION }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT # create metadata for image - name: Docker meta @@ -380,8 +397,8 @@ jobs: with: # list of Docker images to use as base name for tags images: | - pecan/model-${{ matrix.MODEL }}-${{ matrix.VERSION_LOWER || matrix.VERSION }} - ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/model-${{ matrix.MODEL }}-${{ matrix.VERSION_LOWER || matrix.VERSION }} + pecan/${{ steps.lower.outputs.image_name }} + ghcr.io/${{ github.repository_owner }}/${{ github.repository }}/${{ steps.lower.outputs.image_name }} # generate Docker tags based on the following events/attributes tags: | type=schedule @@ -432,8 +449,8 @@ jobs: file: ${{ matrix.DOCKERFILE }} push: true platforms: ${{ matrix.PLATFORM }} - cache-from: type=registry,ref=pecan/model-${{ matrix.MODEL }}-${{ matrix.VERSION }}:buildcache - cache-to: type=registry,ref=pecan/model-${{ matrix.MODEL }}-${{ matrix.VERSION }}:buildcache,mode=max + cache-from: type=registry,ref=pecan/${{ steps.lower.outputs.image_name }}:buildcache + cache-to: type=registry,ref=pecan/${{ steps.lower.outputs.image_name }}:buildcache,mode=max tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | @@ -472,14 +489,14 @@ jobs: steps: - uses: actions/checkout@v4 - # load cached image - - name: Retrieve compiled binary - uses: actions/download-artifact@v4 - with: - name: base - path: /tmp - - name: Load image - run: docker load --input /tmp/base.tar + # # load cached image + # - name: Retrieve compiled binary + # uses: actions/download-artifact@v4 + # with: + # name: base + # path: /tmp + # - name: Load image + # run: docker load --input /tmp/base.tar # create metadata for image - name: Docker meta @@ -541,20 +558,10 @@ jobs: cache-to: type=registry,ref=pecan/${{ matrix.IMAGE }}:buildcache,mode=max tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - outputs: type=docker,dest=/tmp/${{ matrix.name }}.tar build-args: | VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ env.R_VERSION }} - # upload base image to be used in other steps - - name: Upload artifact - if: ${{ matrix.name == 'models' }} - uses: actions/upload-artifact@v4 - with: - if-no-files-found: error - name: models - path: /tmp/models.tar - # ---------------------------------------------------------------------- # Next are images that do not depend on either depends or base image # ---------------------------------------------------------------------- From f79dd141479461bb2fdaee5a09b5add08a5acd6d Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sun, 1 Sep 2024 22:25:25 -0500 Subject: [PATCH 35/52] fix maepa, sipnet --- .github/workflows/docker.yml | 79 ++++-------------------------------- 1 file changed, 8 insertions(+), 71 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 602cec6555..01096746f0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -101,20 +101,11 @@ jobs: cache-to: type=registry,ref=pecan/depends:buildcache,mode=max tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - # outputs: type=docker,dest=/tmp/depends.tar build-args: | VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ env.R_VERSION }} GITHUB_PAT=${{ secrets.GITHUB_TOKEN }} - # # upload depends image to be used in other steps - # - name: Upload artifact - # uses: actions/upload-artifact@v4 - # with: - # if-no-files-found: error - # name: depends - # path: /tmp/depends.tar - # ---------------------------------------------------------------------- # base image has PEcAn compiled and installed, and depends on depends # ---------------------------------------------------------------------- @@ -127,15 +118,6 @@ jobs: steps: - uses: actions/checkout@v4 - # # load cached image - # - name: Retrieve compiled binary - # uses: actions/download-artifact@v4 - # with: - # name: depends - # path: /tmp - # - name: Load image - # run: docker load --input /tmp/depends.tar - # create metadata for image - name: Docker meta id: meta @@ -196,7 +178,6 @@ jobs: cache-to: type=registry,ref=pecan/base:buildcache,mode=max tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - # outputs: type=docker,dest=/tmp/base.tar build-args: | VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ env.R_VERSION }} @@ -204,17 +185,9 @@ jobs: IMAGE_VERSION=${{ steps.meta.outputs.version }} GITHUB_PAT=${{ secrets.GITHUB_TOKEN }} PECAN_VERSION=${{ steps.meta.outputs.version }} - PECAN_GIT_BRANCH= ${{ github.head_ref || github.ref_name }} + PECAN_GIT_BRANCH= ${{ github.head_ref || github.ref_name }} PECAN_GIT_CHECKSUM=${{ github.sha }} PECAN_GIT_DATE=${{ github.event.repository.updated_at }} - - # # upload base image to be used in other steps - # - name: Upload artifact - # uses: actions/upload-artifact@v4 - # with: - # if-no-files-found: error - # name: base - # path: /tmp/base.tar # ---------------------------------------------------------------------- # models image has some python installed to run models, depends on base @@ -228,15 +201,6 @@ jobs: steps: - uses: actions/checkout@v4 - # # load cached image - # - name: Retrieve compiled binary - # uses: actions/download-artifact@v4 - # with: - # name: base - # path: /tmp - # - name: Load image - # run: docker load --input /tmp/base.tar - # create metadata for image - name: Docker meta id: meta @@ -297,7 +261,6 @@ jobs: cache-to: type=registry,ref=pecan/models:buildcache,mode=max tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - # outputs: type=docker,dest=/tmp/models.tar build-args: | VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ env.R_VERSION }} @@ -305,18 +268,10 @@ jobs: IMAGE_VERSION=${{ steps.meta.outputs.version }} GITHUB_PAT=${{ secrets.GITHUB_TOKEN }} PECAN_VERSION=${{ steps.meta.outputs.version }} - PECAN_GIT_BRANCH= ${{ github.head_ref || github.ref_name }} + PECAN_GIT_BRANCH= ${{ github.head_ref || github.ref_name }} PECAN_GIT_CHECKSUM=${{ github.sha }} PECAN_GIT_DATE=${{ github.event.repository.updated_at }} - - # # upload models image to be used in other steps - # - name: Upload artifact - # uses: actions/upload-artifact@v4 - # with: - # if-no-files-found: error - # name: models - # path: /tmp/models.tar - + # ---------------------------------------------------------------------- # Next are images that have models installed # ---------------------------------------------------------------------- @@ -333,8 +288,8 @@ jobs: - biocro - ed2_2.2.0 - ed2_git - - maespa_git - - sipnet_git + - maespa + - sipnet include: - name: basgra CONTEXT: models/basgra @@ -376,20 +331,11 @@ jobs: steps: - uses: actions/checkout@v4 - # # load cached image - # - name: Retrieve compiled binary - # uses: actions/download-artifact@v4 - # with: - # name: models - # path: /tmp - # - name: Load image - # run: docker load --input /tmp/models.tar - # lower case name for docker - name: docker image name id: lower run: echo "image_name=$(echo model-${{ matrix.MODEL }}-${{ matrix.VERSION }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT - + # create metadata for image - name: Docker meta id: meta @@ -489,15 +435,6 @@ jobs: steps: - uses: actions/checkout@v4 - # # load cached image - # - name: Retrieve compiled binary - # uses: actions/download-artifact@v4 - # with: - # name: base - # path: /tmp - # - name: Load image - # run: docker load --input /tmp/base.tar - # create metadata for image - name: Docker meta id: meta @@ -575,8 +512,8 @@ jobs: name: - web - dbsync - - data - - monitor + - data + - monitor - rstudio-nginx include: - name: web From a903d43368622bd0a93159481a52a38a421204f1 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sun, 1 Sep 2024 22:32:41 -0500 Subject: [PATCH 36/52] leverage binary ed build --- .github/workflows/docker.yml | 14 ++++---- models/ed/Dockerfile | 66 ++++++++++++++++++++---------------- 2 files changed, 43 insertions(+), 37 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 01096746f0..eef80be8e9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -286,7 +286,7 @@ jobs: name: - basgra - biocro - - ed2_2.2.0 + # - ed2_2.2.0 - ed2_git - maespa - sipnet @@ -303,12 +303,12 @@ jobs: PLATFORM: "linux/amd64" MODEL: biocro VERSION: "0.95" - - name: ed2_2.2.0 - CONTEXT: models/ed - DOCKERFILE: models/ed/Dockerfile - PLATFORM: "linux/amd64" - MODEL: ed2 - VERSION: "2.2.0" + # - name: ed2_2.2.0 + # CONTEXT: models/ed + # DOCKERFILE: models/ed/Dockerfile + # PLATFORM: "linux/amd64" + # MODEL: ed2 + # VERSION: "2.2.0" - name: ed2_git CONTEXT: models/ed DOCKERFILE: models/ed/Dockerfile diff --git a/models/ed/Dockerfile b/models/ed/Dockerfile index 282a5b5d39..f8edf578f4 100644 --- a/models/ed/Dockerfile +++ b/models/ed/Dockerfile @@ -1,40 +1,45 @@ -# this needs to be at the top, what version are we building -ARG IMAGE_VERSION="latest" +# # this needs to be at the top, what version are we building +# ARG IMAGE_VERSION="latest" -# ---------------------------------------------------------------------- -# BUILD MODEL BINARY -# ---------------------------------------------------------------------- -FROM pecan/models:${IMAGE_VERSION} as model-binary +# # ---------------------------------------------------------------------- +# # BUILD MODEL BINARY +# # ---------------------------------------------------------------------- +# FROM pecan/models:${IMAGE_VERSION} as model-binary -# Some variables that can be used to set control the docker build -ARG MODEL_VERSION="2.2.0" -ARG BINARY_VERSION="2.2" +# # Some variables that can be used to set control the docker build +# ARG MODEL_VERSION="2.2.0" +# ARG BINARY_VERSION="2.2" -# specify fortran compiler -ENV FC_TYPE=GNU +# # specify fortran compiler +# ENV FC_TYPE=GNU -# install dependencies -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - build-essential \ - curl \ - gfortran \ - git \ - libhdf5-dev \ - libopenmpi-dev \ - && rm -rf /var/lib/apt/lists/* +# # install dependencies +# RUN apt-get update \ +# && apt-get install -y --no-install-recommends \ +# build-essential \ +# curl \ +# gfortran \ +# git \ +# libhdf5-dev \ +# libopenmpi-dev \ +# && rm -rf /var/lib/apt/lists/* -# download, unzip and build ed2 -WORKDIR /src -RUN git -c http.sslVerify=false clone https://github.com/EDmodel/ED2.git \ - && cd ED2/ED/build \ - && curl -o make/include.mk.VM http://isda.ncsa.illinois.edu/~kooper/EBI/include.mk.opt.`uname -s` \ - && if [ "${MODEL_VERSION}" != "git" ]; then git checkout "v.${MODEL_VERSION}"; fi \ - && ./install.sh -g -p VM \ - && mv /src/ED2/ED/build/ed_${BINARY_VERSION}-opt /src/ED2/ED/build/ed +# # download, unzip and build ed2 +# WORKDIR /src +# RUN git -c http.sslVerify=false clone https://github.com/EDmodel/ED2.git \ +# && cd ED2/ED/build \ +# && curl -o make/include.mk.VM http://isda.ncsa.illinois.edu/~kooper/EBI/include.mk.opt.`uname -s` \ +# && if [ "${MODEL_VERSION}" != "git" ]; then git checkout "v.${MODEL_VERSION}"; fi \ +# && ./install.sh -g -p VM \ +# && mv /src/ED2/ED/build/ed_${BINARY_VERSION}-opt /src/ED2/ED/build/ed ######################################################################## +# ---------------------------------------------------------------------- +# ED MODEL +# ---------------------------------------------------------------------- +FROM edmodel/ed2:gnu AS model-binary + # ---------------------------------------------------------------------- # BUILD PECAN FOR MODEL # ---------------------------------------------------------------------- @@ -66,4 +71,5 @@ RUN sed -i -e "s/@VERSION@/${MODEL_VERSION}/g" \ -e "s#@BINARY@#/usr/local/bin/ed.${MODEL_VERSION}#g" /work/model.json # COPY model binary -COPY --from=model-binary /src/ED2/ED/build/ed /usr/local/bin/ed.${MODEL_VERSION} +# COPY --from=model-binary /src/ED2/ED/build/ed /usr/local/bin/ed.${MODEL_VERSION} +COPY --from=model-binary /usr/bin/ed2 /usr/local/bin/ed.${MODEL_VERSION} \ No newline at end of file From f5f44429311bf010a7df29cbd764a63291af412f Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sun, 1 Sep 2024 23:13:45 -0500 Subject: [PATCH 37/52] fix ed --- models/ed/Dockerfile | 58 ++++++++++++++----------------- models/ed/data-raw/include.mk.opt | 38 ++++++++++++++++++++ 2 files changed, 65 insertions(+), 31 deletions(-) create mode 100644 models/ed/data-raw/include.mk.opt diff --git a/models/ed/Dockerfile b/models/ed/Dockerfile index f8edf578f4..395c6fb664 100644 --- a/models/ed/Dockerfile +++ b/models/ed/Dockerfile @@ -1,44 +1,40 @@ # # this needs to be at the top, what version are we building -# ARG IMAGE_VERSION="latest" +ARG IMAGE_VERSION="latest" -# # ---------------------------------------------------------------------- -# # BUILD MODEL BINARY -# # ---------------------------------------------------------------------- -# FROM pecan/models:${IMAGE_VERSION} as model-binary - -# # Some variables that can be used to set control the docker build -# ARG MODEL_VERSION="2.2.0" -# ARG BINARY_VERSION="2.2" +# ---------------------------------------------------------------------- +# BUILD MODEL BINARY +# ---------------------------------------------------------------------- +FROM pecan/models:${IMAGE_VERSION} as model-binary -# # specify fortran compiler -# ENV FC_TYPE=GNU +# Some variables that can be used to set control the docker build +ARG MODEL_VERSION="2.2.0" +ARG BINARY_VERSION="2.2" -# # install dependencies -# RUN apt-get update \ -# && apt-get install -y --no-install-recommends \ -# build-essential \ -# curl \ -# gfortran \ -# git \ -# libhdf5-dev \ -# libopenmpi-dev \ -# && rm -rf /var/lib/apt/lists/* +# install dependencies +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + gfortran \ + git \ + libhdf5-openmpi-dev \ + && rm -rf /var/lib/apt/lists/* -# # download, unzip and build ed2 -# WORKDIR /src -# RUN git -c http.sslVerify=false clone https://github.com/EDmodel/ED2.git \ -# && cd ED2/ED/build \ -# && curl -o make/include.mk.VM http://isda.ncsa.illinois.edu/~kooper/EBI/include.mk.opt.`uname -s` \ -# && if [ "${MODEL_VERSION}" != "git" ]; then git checkout "v.${MODEL_VERSION}"; fi \ -# && ./install.sh -g -p VM \ -# && mv /src/ED2/ED/build/ed_${BINARY_VERSION}-opt /src/ED2/ED/build/ed +# download, unzip and build ed2 +WORKDIR /src +RUN git -c http.sslVerify=false clone https://github.com/EDmodel/ED2.git +COPY data-raw/include.mk.opt /src/ED2/ED/build/make/include.mk.pecan +RUN cd ED2/ED/build \ + && if [ "${MODEL_VERSION}" != "git" ]; then git checkout "v.${MODEL_VERSION}"; fi \ + && ./install.sh -g -p pecan \ + && mv /src/ED2/ED/build/ed_${BINARY_VERSION}-opt /src/ED2/ED/build/ed ######################################################################## # ---------------------------------------------------------------------- # ED MODEL # ---------------------------------------------------------------------- -FROM edmodel/ed2:gnu AS model-binary +#FROM edmodel/ed2:gnu AS model-binary # ---------------------------------------------------------------------- # BUILD PECAN FOR MODEL @@ -51,7 +47,7 @@ FROM pecan/models:${IMAGE_VERSION} RUN apt-get update \ && apt-get install -y --no-install-recommends \ - libopenmpi3 \ + libhdf5-openmpi \ && rm -rf /var/lib/apt/lists/* # INSTALL PEcAn.ED2 diff --git a/models/ed/data-raw/include.mk.opt b/models/ed/data-raw/include.mk.opt new file mode 100644 index 0000000000..14c2a7032a --- /dev/null +++ b/models/ed/data-raw/include.mk.opt @@ -0,0 +1,38 @@ +#Makefile include include.mk.opt.ubuntu +############################################################################ + +# Define make (gnu make works best). +MAKE=/usr/bin/make + +# libraries. +BASE=$(ED_ROOT)/build/ + +# HDF 5 Libraries +HDF5_INCS=-I/usr/include/hdf5/openmpi +HDF5_LIBS= -L/usr/lib/$(shell uname -m)-linux-gnu/hdf5/openmpi -lhdf5_fortran -lhdf5_hl -lhdf5 -lz -lm +USE_COLLECTIVE_MPIO=0 + +# interface +USE_INTERF=1 + +# gfortran +CMACH=PC_LINUX1 +FC_TYPE=GNU +F_COMP=mpif90.openmpi +F_OPTS=-O3 -ffree-line-length-none -frecursive -fopenmp -static +C_COMP=mpicc.openmpi +C_OPTS=-O0 -DLITTLE -g -static +LOADER=mpif90.openmpi +LOADER_OPTS=-O3 -ffree-line-length-none -frecursive -fopenmp +C_LOADER==mpicc.openmpi +LIBS= +MOD_EXT=mod + +# using MPI libraries: +MPI_PATH= +PAR_INCS= +PAR_LIBS= +PAR_DEFS=-DRAMS_MPI + +# For IBM,HP,SGI,ALPHA,LINUX use these: +ARCHIVE=ar rs From d965274ede63192888dbdaa7b371e61638e345a0 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sun, 1 Sep 2024 23:25:35 -0500 Subject: [PATCH 38/52] wrong package names --- models/ed/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/ed/Dockerfile b/models/ed/Dockerfile index 395c6fb664..e1e93b06bd 100644 --- a/models/ed/Dockerfile +++ b/models/ed/Dockerfile @@ -47,7 +47,8 @@ FROM pecan/models:${IMAGE_VERSION} RUN apt-get update \ && apt-get install -y --no-install-recommends \ - libhdf5-openmpi \ + libhdf5-openmpi-103 \ + libgomp1 \ && rm -rf /var/lib/apt/lists/* # INSTALL PEcAn.ED2 From cad3ee592f1830d4b4a1bb11ff94d56ce75d2d83 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Mon, 2 Sep 2024 15:22:32 -0500 Subject: [PATCH 39/52] fix ed build --- models/ed/Dockerfile | 8 +------- models/ed/data-raw/include.mk.opt | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/models/ed/Dockerfile b/models/ed/Dockerfile index e1e93b06bd..de47d18f23 100644 --- a/models/ed/Dockerfile +++ b/models/ed/Dockerfile @@ -31,11 +31,6 @@ RUN cd ED2/ED/build \ ######################################################################## -# ---------------------------------------------------------------------- -# ED MODEL -# ---------------------------------------------------------------------- -#FROM edmodel/ed2:gnu AS model-binary - # ---------------------------------------------------------------------- # BUILD PECAN FOR MODEL # ---------------------------------------------------------------------- @@ -68,5 +63,4 @@ RUN sed -i -e "s/@VERSION@/${MODEL_VERSION}/g" \ -e "s#@BINARY@#/usr/local/bin/ed.${MODEL_VERSION}#g" /work/model.json # COPY model binary -# COPY --from=model-binary /src/ED2/ED/build/ed /usr/local/bin/ed.${MODEL_VERSION} -COPY --from=model-binary /usr/bin/ed2 /usr/local/bin/ed.${MODEL_VERSION} \ No newline at end of file +COPY --from=model-binary /src/ED2/ED/build/ed /usr/local/bin/ed.${MODEL_VERSION} diff --git a/models/ed/data-raw/include.mk.opt b/models/ed/data-raw/include.mk.opt index 14c2a7032a..da68d2f145 100644 --- a/models/ed/data-raw/include.mk.opt +++ b/models/ed/data-raw/include.mk.opt @@ -32,7 +32,7 @@ MOD_EXT=mod MPI_PATH= PAR_INCS= PAR_LIBS= -PAR_DEFS=-DRAMS_MPI +PAR_DEFS= # For IBM,HP,SGI,ALPHA,LINUX use these: ARCHIVE=ar rs From df235cac423d6f8a9c51ca7ccab499e57b5db1f5 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Mon, 2 Sep 2024 15:27:41 -0500 Subject: [PATCH 40/52] cleanup / undo --- .github/workflows/book.yml | 18 +++++++++--------- .github/workflows/ci.yml | 22 +++++++++++----------- .github/workflows/prlabeler.yml | 2 +- docker/data/Dockerfile | 2 +- docker/executor/Dockerfile | 2 +- docker/models/Dockerfile | 2 +- docker/monitor/Dockerfile | 2 +- models/maespa/Dockerfile | 2 +- models/sipnet/Dockerfile | 2 +- shiny/dbsync/Dockerfile | 2 +- 10 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index 30b92fb160..a7551fdf2f 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -1,14 +1,14 @@ name: renderbook -# on: -# push: -# branches: -# - master -# - develop -# tags: -# - '*' -# pull_request: -# merge_group: +on: + push: + branches: + - master + - develop + tags: + - '*' + pull_request: + merge_group: workflow_dispatch: jobs: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8fb6379f27..3bc37aa5a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,18 +1,18 @@ name: CI on: - # push: - # branches: - # - master - # - develop + push: + branches: + - master + - develop - # tags: - # - '*' - # pull_request: - # merge_group: - # issue_comment: - # types: - # - created + tags: + - '*' + pull_request: + merge_group: + issue_comment: + types: + - created workflow_dispatch: env: diff --git a/.github/workflows/prlabeler.yml b/.github/workflows/prlabeler.yml index abd26db032..921b6fd451 100644 --- a/.github/workflows/prlabeler.yml +++ b/.github/workflows/prlabeler.yml @@ -5,7 +5,7 @@ name: "PR Labeler" on: - # - pull_request_target + - pull_request_target jobs: label: permissions: diff --git a/docker/data/Dockerfile b/docker/data/Dockerfile index e663819fac..b058138603 100644 --- a/docker/data/Dockerfile +++ b/docker/data/Dockerfile @@ -18,4 +18,4 @@ RUN apk --no-cache add bash curl rsync postgresql-client unzip \ && curl -s -o create_met_driver.tar.gz http://isda.ncsa.illinois.edu/~kooper/EBI/create_met_driver.tar.gz COPY add-data.sh add.util.sh /work/ -CMD bash /work/add-data.sh +CMD ["bash", "-c", "/work/add-data.sh"] diff --git a/docker/executor/Dockerfile b/docker/executor/Dockerfile index f1e4c22230..19cecced3c 100644 --- a/docker/executor/Dockerfile +++ b/docker/executor/Dockerfile @@ -23,4 +23,4 @@ ENV RABBITMQ_URI="amqp://guest:guest@rabbitmq/%2F" \ # actual application that will be executed COPY executor.py /work/ -CMD python3 /work/executor.py +CMD ["python3", "/work/executor.py"] diff --git a/docker/models/Dockerfile b/docker/models/Dockerfile index 6819ae48b9..f76f51382e 100644 --- a/docker/models/Dockerfile +++ b/docker/models/Dockerfile @@ -22,4 +22,4 @@ ENV RABBITMQ_URI="amqp://guest:guest@rabbitmq/%2F" \ # actual application that will be executed COPY model.py /work/ -CMD python3 /work/model.py +CMD ["python3", "/work/model.py"] diff --git a/docker/monitor/Dockerfile b/docker/monitor/Dockerfile index 24232352fc..ec479cc245 100644 --- a/docker/monitor/Dockerfile +++ b/docker/monitor/Dockerfile @@ -17,4 +17,4 @@ COPY requirements.txt /src/ RUN pip3 install -r /src/requirements.txt COPY . /src/ -CMD python3 monitor.py +CMD ["python3", "./monitor.py"] diff --git a/models/maespa/Dockerfile b/models/maespa/Dockerfile index 0b5fbd1be5..22243dc3c1 100644 --- a/models/maespa/Dockerfile +++ b/models/maespa/Dockerfile @@ -4,7 +4,7 @@ ARG IMAGE_VERSION="latest" # ---------------------------------------------------------------------- # BUILD MODEL BINARY # ---------------------------------------------------------------------- -FROM pecan/models:${IMAGE_VERSION} as model-binary +FROM pecan/models:${IMAGE_VERSION} AS model-binary # download, unzip and build ed2 WORKDIR /src diff --git a/models/sipnet/Dockerfile b/models/sipnet/Dockerfile index 1b0fc75725..d69e845dd4 100644 --- a/models/sipnet/Dockerfile +++ b/models/sipnet/Dockerfile @@ -4,7 +4,7 @@ ARG IMAGE_VERSION="latest" # ---------------------------------------------------------------------- # BUILD SIPNET BINARY # ---------------------------------------------------------------------- -FROM pecan/models:${IMAGE_VERSION} as model-binary +FROM pecan/models:${IMAGE_VERSION} AS model-binary # Some variables that can be used to set control the docker build ARG MODEL_VERSION=git diff --git a/shiny/dbsync/Dockerfile b/shiny/dbsync/Dockerfile index d5a89a6ca0..a133290de8 100644 --- a/shiny/dbsync/Dockerfile +++ b/shiny/dbsync/Dockerfile @@ -18,4 +18,4 @@ ADD https://raw.githubusercontent.com/rocker-org/shiny/master/shiny-server.sh /u RUN chmod +x /usr/bin/shiny-server.sh # special script to start shiny server and preserve env variable -CMD /srv/shiny-server/save-env-shiny.sh +CMD ["/srv/shiny-server/save-env-shiny.sh"] From 43c536223cb2308e464a290bfd9fb297f9e7b7b8 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Mon, 2 Sep 2024 15:31:13 -0500 Subject: [PATCH 41/52] merge/cleanup --- models/ed/Dockerfile | 2 +- scripts/confirm_deps.R | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/ed/Dockerfile b/models/ed/Dockerfile index de47d18f23..a80e0ac5c4 100644 --- a/models/ed/Dockerfile +++ b/models/ed/Dockerfile @@ -1,4 +1,4 @@ -# # this needs to be at the top, what version are we building +# this needs to be at the top, what version are we building ARG IMAGE_VERSION="latest" # ---------------------------------------------------------------------- diff --git a/scripts/confirm_deps.R b/scripts/confirm_deps.R index 33c8643764..fd272c1c29 100755 --- a/scripts/confirm_deps.R +++ b/scripts/confirm_deps.R @@ -85,7 +85,7 @@ confirm_deps <- function(pkg, return( remotes::install_deps( pkg = pkg, - dependencies = NA, + dependencies = dependencies, ... ) ) From 9f698c42df1b1ab38e67e2d5b21996cbfabed9ba Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Mon, 2 Sep 2024 15:51:51 -0500 Subject: [PATCH 42/52] don't set BINARY_VERSION --- .github/workflows/docker.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index eef80be8e9..6574eba8c9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -404,7 +404,6 @@ jobs: R_VERSION=${{ env.R_VERSION }} MODEL_VERSION=${{ matrix.VERSION }} IMAGE_VERSION=${{ steps.meta.outputs.version }} - BINARY_VERSION="2.2" # ---------------------------------------------------------------------- # Next are images that depend on base image From 91f90889f73bfefaff04085cd7e0ad827f7622f2 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Mon, 2 Sep 2024 15:56:17 -0500 Subject: [PATCH 43/52] add healthchecks --- docker-compose.yml | 90 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 74 insertions(+), 16 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 92af1a03a9..c5c77c37da 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,6 +52,11 @@ services: - "traefik.http.routers.rabbitmq.rule=Host(`rabbitmq.pecan.localhost`)" volumes: - rabbitmq:/var/lib/rabbitmq + healthcheck: + test: rabbitmqctl ping + interval: 10s + timeout: 5s + retries: 5 # ---------------------------------------------------------------------- # Database to hold the data from PEcAn and BETY. @@ -65,6 +70,11 @@ services: - pecan volumes: - postgres:/var/lib/postgresql/data + healthcheck: + test: pg_isready -U postgres + interval: 10s + timeout: 5s + retries: 5 # ---------------------------------------------------------------------- # BETY rails frontend to the database @@ -82,11 +92,17 @@ services: volumes: - bety:/home/bety/log depends_on: - - postgres + postgres: + condition: service_healthy labels: - "traefik.enable=true" - "traefik.http.services.bety.loadbalancer.server.port=8000" - "traefik.http.routers.bety.rule=Host(`${TRAEFIK_HOST:-pecan.localhost}`) && PathPrefix(`/bety/`)" + healthcheck: + test: "curl --silent --fail http://localhost:8000/${RAILS_RELATIVE_URL_ROOT} > /dev/null || exit 1" + interval: 10s + timeout: 5s + retries: 5 # ---------------------------------------------------------------------- # RStudio @@ -98,8 +114,10 @@ services: networks: - pecan depends_on: - - rabbitmq - - postgres + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy environment: - KEEP_ENV=RABBITMQ_URI RABBITMQ_PREFIX RABBITMQ_PORT FQDN NAME - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F} @@ -141,6 +159,11 @@ services: - "traefik.enable=true" - "traefik.http.services.docs.loadbalancer.server.port=80" - "traefik.http.routers.docs.rule=Host(`${TRAEFIK_HOST:-pecan.localhost}`) && PathPrefix(`/`)" + healthcheck: + test: "curl --silent --fail http://localhost/ > /dev/null || exit 1" + interval: 10s + timeout: 5s + retries: 5 # PEcAn web front end, this is just the PHP code pecan: @@ -155,8 +178,10 @@ services: - NAME=${PECAN_NAME:-docker} - SECRET_KEY_BASE=${BETY_SECRET_KEY:-thisisnotasecret} depends_on: - - postgres - - rabbitmq + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy labels: - "traefik.enable=true" - "traefik.http.services.pecan.loadbalancer.server.port=8080" @@ -164,6 +189,11 @@ services: volumes: - pecan:/data - pecan:/var/www/html/pecan/data + healthcheck: + test: "curl --silent --fail http://localhost/pecan/ > /dev/null || exit 1" + interval: 10s + timeout: 5s + retries: 5 # PEcAn model monitor monitor: @@ -176,7 +206,10 @@ services: - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F} - FQDN=${PECAN_FQDN:-docker} depends_on: - - rabbitmq + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy labels: - "traefik.enable=true" - "traefik.http.routers.monitor.rule=Host(`${TRAEFIK_HOST:-pecan.localhost}`) && PathPrefix(`/monitor/`)" @@ -184,6 +217,11 @@ services: - "traefik.http.middlewares.monitor-stripprefix.stripprefix.prefixes=/monitor" volumes: - pecan:/data + healthcheck: + test: "curl --silent --fail http://localhost:9999/monitor/ > /dev/null || exit 1" + interval: 10s + timeout: 5s + retries: 5 # PEcAn executor, executes jobs. Does not the actual models executor: @@ -198,8 +236,10 @@ services: - RABBITMQ_PORT=15672 - FQDN=${PECAN_FQDN:-docker} depends_on: - - postgres - - rabbitmq + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy volumes: - pecan:/data @@ -216,7 +256,8 @@ services: environment: - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F} depends_on: - - rabbitmq + rabbitmq: + condition: service_healthy volumes: - pecan:/data @@ -230,7 +271,8 @@ services: environment: - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F} depends_on: - - rabbitmq + rabbitmq: + condition: service_healthy volumes: - pecan:/data @@ -244,7 +286,8 @@ services: environment: - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F} depends_on: - - rabbitmq + rabbitmq: + condition: service_healthy volumes: - pecan:/data @@ -258,7 +301,8 @@ services: environment: - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F} depends_on: - - rabbitmq + rabbitmq: + condition: service_healthy volumes: - pecan:/data @@ -272,7 +316,8 @@ services: environment: - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F} depends_on: - - rabbitmq + rabbitmq: + condition: service_healthy volumes: - pecan:/data @@ -286,7 +331,8 @@ services: environment: - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F} depends_on: - - rabbitmq + rabbitmq: + condition: service_healthy volumes: - pecan:/data @@ -300,12 +346,18 @@ services: networks: - pecan depends_on: - - postgres + postgres: + condition: service_healthy labels: - "traefik.enable=true" - "traefik.http.routers.dbsync.rule=Host(`${TRAEFIK_HOST:-pecan.localhost}`) && PathPrefix(`/dbsync/`)" - "traefik.http.routers.dbsync.middlewares=dbsync-stripprefix" - "traefik.http.middlewares.dbsync-stripprefix.stripprefix.prefixes=/monitor" + healthcheck: + test: "curl --silent --fail http://localhost/dbsync/ > /dev/null || exit 1" + interval: 10s + timeout: 5s + retries: 5 # ---------------------------------------------------------------------- # PEcAn API @@ -329,9 +381,15 @@ services: - "traefik.http.routers.api.rule=Host(`${TRAEFIK_HOST:-pecan.localhost}`) && PathPrefix(`/api/`)" - "traefik.http.services.api.loadbalancer.server.port=8000" depends_on: - - postgres + postgres: + condition: service_healthy volumes: - pecan:/data/ + healthcheck: + test: "curl --silent --fail http://localhost/api/ping > /dev/null || exit 1" + interval: 10s + timeout: 5s + retries: 5 # ---------------------------------------------------------------------- # Name of network to be used by all containers From 465f4e8c17c3be902882f7804ab5b25c59317caf Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Mon, 2 Sep 2024 16:00:38 -0500 Subject: [PATCH 44/52] missing api app --- .github/workflows/docker.yml | 6 ++++++ apps/api/Dockerfile | 10 +++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6574eba8c9..a02c09ebc6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -419,6 +419,7 @@ jobs: name: - docs - executor + - api include: - name: docs CONTEXT: . @@ -430,6 +431,11 @@ jobs: DOCKERFILE: docker/executor/Dockerfile PLATFORM: "linux/amd64" IMAGE: executor + - name: api + CONTEXT: apps/api + DOCKERFILE: apps/api/Dockerfile + PLATFORM: "linux/amd64" + IMAGE: api steps: - uses: actions/checkout@v4 diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 4aa5a6a2d9..059e14f0c0 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -6,7 +6,6 @@ ARG IMAGE_VERSION="latest" # PECAN FOR MODEL BASE IMAGE # -------------------------------------------------------------------------- FROM pecan/base:${IMAGE_VERSION} -LABEL maintainer="Tezan Sahu " EXPOSE 8000 @@ -15,9 +14,7 @@ EXPOSE 8000 # -------------------------------------------------------------------------- # COMMAND TO RUN -RUN --mount=type=secret,id=github_token \ - export GITHUB_PAT=`cat /run/secrets/github_token` \ - && apt-get update \ +RUN apt-get update \ && apt-get install libsodium-dev -y \ && rm -rf /var/lib/apt/lists/* \ && Rscript -e "devtools::install_version('promises', '1.1.0')" \ @@ -35,7 +32,6 @@ ENV AUTH_REQ="TRUE" \ SECRET_KEY_BASE="thisisnotasecret" WORKDIR /api/R - -CMD Rscript entrypoint.R - COPY ./ /api + +CMD ["Rscript", "entrypoint.R"] From 4903ee87f4eea7d557eeb53c1e2edbd58772e3b5 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Tue, 3 Sep 2024 18:22:51 -0500 Subject: [PATCH 45/52] install version of sodium that is newer. --- apps/api/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 059e14f0c0..2fb7a530a0 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -19,6 +19,7 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && Rscript -e "devtools::install_version('promises', '1.1.0')" \ && Rscript -e "devtools::install_version('webutils', '1.1')" \ + && Rscript -e "devtools::install_version('sodium', '>= 1.2.0')" \ && Rscript -e "install.packages('pool')" \ && Rscript -e "devtools::install_github('rstudio/swagger')" \ && Rscript -e "devtools::install_github('rstudio/plumber')" From 331e3e11897f6239223f07f04d117cab80aa0dca Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Tue, 3 Sep 2024 19:10:31 -0500 Subject: [PATCH 46/52] set repo --- apps/api/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 2fb7a530a0..5c5ce46baa 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -19,7 +19,7 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && Rscript -e "devtools::install_version('promises', '1.1.0')" \ && Rscript -e "devtools::install_version('webutils', '1.1')" \ - && Rscript -e "devtools::install_version('sodium', '>= 1.2.0')" \ + && Rscript -e "devtools::install_version('sodium', '>= 1.2.0', repos='https://p3m.dev/')" \ && Rscript -e "install.packages('pool')" \ && Rscript -e "devtools::install_github('rstudio/swagger')" \ && Rscript -e "devtools::install_github('rstudio/plumber')" From 78e649066acf80b4a021ad015290d8039e9a8c4d Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Tue, 3 Sep 2024 21:50:01 -0500 Subject: [PATCH 47/52] specify image version --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a02c09ebc6..1c1894257d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -503,6 +503,7 @@ jobs: build-args: | VERSION=${{ steps.meta.outputs.version }} R_VERSION=${{ env.R_VERSION }} + IMAGE_VERSION=${{ steps.meta.outputs.version }} # ---------------------------------------------------------------------- # Next are images that do not depend on either depends or base image From 64364fe38c07adb676763f4c58763ba2a5be3dff Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Tue, 3 Sep 2024 22:00:01 -0500 Subject: [PATCH 48/52] remove sodium install, had wrong base image --- apps/api/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 5c5ce46baa..059e14f0c0 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -19,7 +19,6 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && Rscript -e "devtools::install_version('promises', '1.1.0')" \ && Rscript -e "devtools::install_version('webutils', '1.1')" \ - && Rscript -e "devtools::install_version('sodium', '>= 1.2.0', repos='https://p3m.dev/')" \ && Rscript -e "install.packages('pool')" \ && Rscript -e "devtools::install_github('rstudio/swagger')" \ && Rscript -e "devtools::install_github('rstudio/plumber')" From 52b06291d6c707f95b552161f4256017efd120e4 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Tue, 3 Sep 2024 22:12:34 -0500 Subject: [PATCH 49/52] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 366366cfe6..13502e217e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ For more information about this file see also [Keep a Changelog](http://keepacha ### Added ### Fixed +- updated github action to build docker images ### Changed - The following components have changed their licensing. With approval of all their contributors, we now provide them under a BSD 3-clause license rather than the previously used NCSA Open Source license. As a reminder, we intend to relicense the entire system and this list will expand as we gather permission from the relevant copyright owners. From 6b03e4faa41d2e364be469255f22efd0e49ffca4 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sun, 8 Sep 2024 08:01:02 -0500 Subject: [PATCH 50/52] fixes from Chris --- .github/workflows/docker.yml | 2 ++ docker/depends/Dockerfile | 4 +--- models/ed/Dockerfile | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1c1894257d..0e116b9221 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -23,6 +23,8 @@ on: - 4.1 - 4.2 - 4.3 + - 4.4 + - devel # set up the environment, either input or default env: diff --git a/docker/depends/Dockerfile b/docker/depends/Dockerfile index 6b9dc4eeef..b784bcd3bf 100644 --- a/docker/depends/Dockerfile +++ b/docker/depends/Dockerfile @@ -31,9 +31,7 @@ RUN apt-get update \ # INSTALL DEPENDENCIES # ---------------------------------------------------------------------- COPY pecan.depends.R pecan_package_dependencies.csv pecan_deps_from_github.txt / -RUN --mount=type=secret,id=github_token \ - export GITHUB_PAT=`cat /run/secrets/github_token` \ - && Rscript -e "install.packages(c('desc', 'remotes'))" \ +RUN Rscript -e "install.packages(c('desc', 'remotes'))" \ && R_LIBS_USER='/usr/local/lib/R/site-library' Rscript /pecan.depends.R \ && rm -rf /tmp/* diff --git a/models/ed/Dockerfile b/models/ed/Dockerfile index a80e0ac5c4..a3873e896a 100644 --- a/models/ed/Dockerfile +++ b/models/ed/Dockerfile @@ -4,7 +4,7 @@ ARG IMAGE_VERSION="latest" # ---------------------------------------------------------------------- # BUILD MODEL BINARY # ---------------------------------------------------------------------- -FROM pecan/models:${IMAGE_VERSION} as model-binary +FROM pecan/models:${IMAGE_VERSION} AS model-binary # Some variables that can be used to set control the docker build ARG MODEL_VERSION="2.2.0" From c9c22db561e9c09858b8264515ab4a0201f78e7b Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sun, 8 Sep 2024 09:29:56 -0500 Subject: [PATCH 51/52] forgot to re-enabled actions --- .github/workflows/docker.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0e116b9221..ffa43813cb 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,11 +1,11 @@ name: Docker GHA on: - # push: - # branches: - # - main - # tags: - # - "v*.*.*" + push: + branches: + - main + tags: + - "v*.*.*" pull_request: workflow_dispatch: inputs: From 9effaf122327f480caf7bf2b1f3d6adca65c40ac Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sun, 8 Sep 2024 09:34:02 -0500 Subject: [PATCH 52/52] build develop/master remove option --- .github/workflows/docker.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ffa43813cb..46ac384c5b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,16 +4,13 @@ on: push: branches: - main + - master + - develop tags: - "v*.*.*" pull_request: workflow_dispatch: inputs: - depends: - description: 'Create depends image' - required: true - type: boolean - default: true r_version: description: 'R version to use' required: true