Release flow (#1019)

Co-authored-by: Jaap Langemeijer <[email protected]>

.github/workflows/create-release-artifacts.yml

@@ -0,0 +1,145 @@
+
+name: Create release artifacts
+on:
+  workflow_dispatch:
+  # running this when PR opens we'll do in prep-release manually, see that file for more details
+  push:
+    branches:
+      - 'release/**'
+  workflow_run:
+    workflows: [Create a new release]
+    types:
+      - completed
+
+jobs:
+  pypi:
+    runs-on: ubuntu-latest
+    outputs:
+      url: ${{ steps.upload-pypi-release-artifacts.outputs.artifact-url }}
+    if: ${{ github.event_name == 'push' || github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v5
+        name: Install Python
+        with:
+          python-version: '3.11'
+
+      - name: generate env
+        run: |
+          pip install tomli flit twine
+
+      - name: Build artifacts
+        run: |
+          git clean -xdf
+          git restore -SW .
+          flit build
+          python -m twine check dist/*
+
+      - name: Publish package to TestPyPI
+        uses: pypa/[email protected]
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_TEST_TOKEN }}
+          repository-url: https://test.pypi.org/legacy/
+          verbose: true
+          skip-existing: true
+
+      - uses: actions/upload-artifact@v4
+        id: upload-pypi-release-artifacts
+        with:
+          name: pypi-release-artifacts
+          path: dist
+  docs:
+    runs-on: ubuntu-latest
+    outputs:
+      url: ${{ steps.upload-docs-release-artifacts.outputs.artifact-url }}
+    if: ${{ github.event_name == 'push' || github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: "v0.21.1"
+          environments: full-py311
+
+      - name: Generate docs
+        run: |
+          pixi run --locked -e full-py311 html
+
+      - uses: actions/upload-artifact@v4
+        id: upload-docs-release-artifacts
+        with:
+          name: docs-release-artifacts
+          path: docs/_build
+
+
+  docker:
+    runs-on: ubuntu-latest
+    outputs:
+      url: ${{ steps.upload-docker-release-artifacts.outputs.artifact-url }}
+    if: ${{ github.event_name == 'push' || github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub to check credentials
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and export
+        uses: docker/build-push-action@v6
+        with:
+          outputs: type=docker,dest=/tmp/hydromt-docker-image.tar
+          tags: hydromt
+
+      - name: Run Tests
+        run: |
+          docker load --input /tmp/hydromt-docker-image.tar
+          docker run --env NUMBA_DISABLE_JIT=1 --rm hydromt
+
+      - name: Test Binder integration with repo2docker
+        run: |
+          pip install jupyter-repo2docker
+          repo2docker . echo 'success!'
+
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        id: upload-docker-release-artifacts
+        with:
+          name: hydromt-docker-image
+          path: /tmp/hydromt-docker-image.tar
+
+  notify:
+    needs:
+      - pypi
+      - docs
+      - docker
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Add comment to PR
+        env:
+          GH_TOKEN: ${{ github.token }}
+          pypi_url: ${{needs.pypi.outputs.url}}
+          docker_url: ${{needs.docker.outputs.url}}
+          docs_url: ${{needs.docs.outputs.url}}
+        run: |
+          echo "The new release artifacts have been generated and tested. You can download and inspect them if you want by using the links below: " > comment.txt
+          echo "    - pypi:   $pypi_url" >> comment.txt
+          echo "    - docker: $docker_url" >> comment.txt
+          echo "    - docs:   $docs_url" >> comment.txt
+          export PR_ID=$(gh pr list --state "open" --author "app/github-actions" --search "release"  --json "number" --jq '. | first | .number')
+
+          gh pr comment $PR_ID --body-file comment.txt

.github/workflows/docs.yml

@@ -32,8 +32,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
-      - uses: prefix-dev/setup-pixi@v0.6.0
+        uses: actions/checkout@v4
+      - uses: prefix-dev/setup-pixi@v0.8.1
         with:
           pixi-version: "v0.21.1"
           environments: full-py311
@@ -49,13 +49,13 @@ jobs:
           pixi run --locked -e full-py311 sphinx-build ./docs ./docs/_build -b dummy -W
 
       - name: Generate docs
-        if: ${{ github.event_name != 'pull_request' && !github.event.act }}
+        if: ${{ github.event_name != 'pull_request' }}
         run: |
           pixi run --locked -e full-py311 html
           echo "DOC_VERSION=$(pixi run --locked -e full-py311 python -c 'from hydromt import __version__ as v; print("dev" if "dev" in v else "v"+v.replace(".dev",""))')" >> $GITHUB_ENV
 
       - name: Upload to GitHub Pages
-        if: ${{ github.event_name != 'pull_request' && !github.event.act }}
+        if: ${{ github.event_name != 'pull_request' }}
         uses: peaceiris/[email protected]
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/finish-release.yml

@@ -0,0 +1,128 @@
+
+name: Finalise a new release
+
+on:
+  pull_request:
+    types:
+    - closed
+    branches:
+    - 'release/**'
+
+jobs:
+  publish-docker:
+    if: github.event.pull_request.merged == true
+    name: Finalise docker (${{ matrix.flavour }}) release
+    runs-on: ubuntu-24.04 # latest LTS release
+    strategy:
+      fail-fast: false
+      matrix:
+        flavour: ['min','slim','full']
+    concurrency:
+      group: NEW_RELEASE_DOCKER
+      cancel-in-progress: false # don't abort while we might be creating a PR or something
+    steps:
+      - uses: actions/checkout@v4
+        name: Checkout code
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: hydromt-docker-image
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push ${{ matrix.flavour }} image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          target: ${{ matrix.flavour }}
+          push:  true
+          tags: deltares/hydromt:${{ env.version }}-${{ matrix.flavour }}
+
+      - name: Build and push latest image
+        if: matrix.flavour == 'slim'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          target: ${{ matrix.flavour }}
+          push:  true
+          tags: deltares/hydromt:${{ env.version }}, deltares/hydromt:latest
+
+  publish-pypi:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.merged == true
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: pypi-release-artifacts
+          path: dist
+      - name: Publish package to PyPI
+        uses: pypa/[email protected]
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_TOKEN }}
+          verbose: true
+
+  publish-docs:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.merged == true
+    steps:
+    - uses: actions/download-artifact@v4
+      with:
+          name: docs-release-artifacts
+          path: docs/_build
+    - name: Upload to GitHub Pages
+      uses: peaceiris/[email protected]
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        publish_dir: ./docs/_build/html
+        exclude_assets: .buildinfo,_sources/*,_examples/*.ipynb
+        destination_dir: ./${{ env.DOC_VERSION }}
+        keep_files: false
+        full_commit_message: Deploy Release ${{ env.DOC_VERSION }} docs to GitHub Pages
+
+
+  gh-release:
+    needs:
+      - publish-docker
+      - publish-pypi
+      - publish-docs
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ github.token }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Finalise PR
+        run: |
+            git pull
+            git checkout main
+            export RELEASE_VERSION=$(grep "version" hydromt/__init__.py | cut -d= -f 2 | tr -d "\" ")
+            if [ -z "$RELEASE_VERSION" ]; then
+              echo "could not determine release version"
+              echo "detection result: $RELEASE_VERSION"
+              exit 1 # somthing went very wrong
+            fi
+            ## seting up git
+            git config user.name "GitHub Actions Bot"
+            git config user.email "<>"
+            git tag "v$RELEASE_VERSION"
+            # post release stuff so we don't forget
+            sed -i "s/v$RELEASE_VERSION/Unreleased\n==========\n\nNew\n---\n\nChanged\n-------\n\nFixed\n-----\n\nDeprecated\n----------\n\nv$RELEASE_VERSION/" docs/changelog.rst
+            sed -i 's/__version__.*=.*"\(.*\)".*/__version__ = "\1.dev0"/' hydromt/__init__.py
+
+            git add hydromt/__init__.py docs/changelog.rst
+            git commit -m "Post release v$RELEASE_VERSION"
+            awk '
+              BEGIN { capture = 0; }
+              # Match the first header and start capturing
+              /^v[0-9]+\.[0-9]+\.[0-9]+.*$/ { if (capture == 0) { capture = 1; } else { exit; } }
+              # Capture lines between headers
+              capture { print; }
+            ' docs/changelog.rst > release_notes.rst
+            git push
+            git push origin tag "v$RELEASE_VERSION"
+            gh release create "v$RELEASE_VERSION" -F release_notes.rst -t "v$RELEASE_VERSION"

.github/workflows/linting.yml

@@ -10,6 +10,6 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
       - uses: pre-commit/[email protected]

.github/workflows/pixi_auto_update.yml

@@ -14,7 +14,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
-      - uses: prefix-dev/setup-pixi@v0.6.0
+      - uses: prefix-dev/setup-pixi@v0.8.1
         with:
           pixi-version: "v0.21.1"
           cache: false