Cahce dependencies in CI to speedup workflows (#471)

.github/workflows/docs.yml

@@ -1,7 +1,6 @@
 ---
 name: Build Documentation
 
-# We don't want pushes (or PRs) to gh-pages to kick anything off
 on:
   push:
     branches: [main]
@@ -23,9 +22,7 @@ on:
       - pyproject.toml
 
 jobs:
-  # Build docs on Linux
   Docs:
-    timeout-minutes: 10
     name: linux docs
     runs-on: ubuntu-latest
     env:
@@ -37,22 +34,35 @@ jobs:
       group: ${{ github.workflow }}-${{ github.ref }}
       cancel-in-progress: true
     steps:
-      - name: Checkout source
+      - name: checkout code
         uses: actions/checkout@v3
-      - name: install tomli
-        run: pip install tomli
+
+      - name: Setup Mambaforge
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+            python-version: 3.11
+            miniforge-variant: Mambaforge
+            miniforge-version: latest
+            channels: conda-forge
+            activate-environment: hydromt
+            use-mamba: true
       - name: Generate env spec
         run: python make_env.py doc
 
-      - name: Setup environment
-        uses: conda-incubator/setup-miniconda@v2
+
+      - name: Set cache date
+        run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV
+
+      - name: load from cache if it exists for our environment
+        uses: actions/cache@v2
         with:
-          python-version: '3.10'
-          miniforge-variant: Mambaforge
-          channels: conda-forge, defaults
-          channel-priority: strict
-          environment-file: environment.yml
-          activate-environment: hydromt
+          path: /usr/share/miniconda3/envs/hydromt
+          key: docs-conda-${{ hashFiles('environment.yml') }}-${{ env.DATE }}
+        id: cache
+
+      - name: Update environment & write to cache
+        run: mamba env update -n hydromt -f environment.yml
+        if: steps.cache.outputs.cache-hit != 'true'
 
       - name: Install hydromt
         run: pip install .

.github/workflows/linting.yml

@@ -9,7 +9,6 @@ on:
 jobs:
   pre-commit:
     runs-on: ubuntu-latest
-    timeout-minutes: 10
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v3

.github/workflows/publish-docker.yml

@@ -1,13 +1,10 @@
 ---
 name: Build and publish HydroMT to DockerHub
 
-# on:
-#   release:
-#     types:
-#       - published
-#   push:
-#     tags:
-#       - v*
+on:
+  release:
+    types:
+      - published
 
 jobs:
   publish-docker-image:

.github/workflows/publish-pypi.yml

@@ -77,7 +77,6 @@ jobs:
           echo "hydromt --models"
 
   upload-to-test-pypi:
-    timeout-minutes: 10
     needs: test-built-dist
     if: github.event_name == 'push'
     runs-on: ubuntu-latest

.github/workflows/purge_all_cache.yml

@@ -0,0 +1,17 @@
+name: cleanup all caches
+on:
+  workflow_disptach:
+
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+
+      - name: Cleanup caches
+        run: |
+          gh cache delete -a
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/purge_pr_cache.yml

@@ -0,0 +1,34 @@
+
+name: cleanup caches by a branch
+on:
+  pull_request:
+    types:
+      - closed
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+
+      - name: Cleanup caches
+        run: |
+          gh extension install actions/gh-actions-cache
+
+          REPO=${{ github.repository }}
+          BRANCH="refs/pull/${{ github.event.pull_request.number }}/merge"
+
+          echo "Fetching list of cache key"
+          cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH -L 100 | cut -f 1 )
+
+          ## Setting this to not fail the workflow while deleting cache keys.
+          set +e
+          echo "Deleting caches..."
+          for cacheKey in $cacheKeysForPR
+          do
+              gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm
+          done
+          echo "Done"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/tests.yml

@@ -1,4 +1,3 @@
----
 name: Tests
 
 on:
@@ -10,44 +9,59 @@ on:
       - data/*
       - pyproject.toml
   pull_request:
-    branches: [main]
+    branches:
+      - main
     paths:
       - tests/*
       - hydromt/*
       - data/*
       - pyproject.toml
 
 jobs:
-  Test-matrix:
-    name: ${{ matrix.os }} - py${{ matrix.python-version }}
-    runs-on: ${{ matrix.os }}
-    timeout-minutes: 10
+  build:
     defaults:
       run:
         shell: bash -l {0}
     strategy:
-      fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.9', '3.10', '3.11']
+        python-version: ['3.9','3.10','3.11']
+    name: py ${{ matrix.python-version }}
+    runs-on: ${{ matrix.os }}
+    concurrency:
+      group: ${{ github.workflow }}-${{ matrix.python-version }}-${{ github.ref }}
+      cancel-in-progress: true
     steps:
       - uses: actions/checkout@v3
+
+      - name: Setup Mambaforge
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+            python-version: ${{ matrix.python-version }}
+            miniforge-variant: Mambaforge
+            miniforge-version: latest
+            activate-environment: hydromt
+            use-mamba: true
       - name: Generate env spec
         run: pip install tomli && python make_env.py test
-      - uses: conda-incubator/setup-miniconda@v2
+
+
+      - name: Set cache date
+        run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV
+
+      - name: load from cache if it exists for our environment
+        uses: actions/cache@v2
         with:
-          python-version: ${{ matrix.python-version }}
-          miniforge-variant: Mambaforge
-          channels: conda-forge, defaults
-          channel-priority: strict
-          environment-file: environment.yml
-          activate-environment: hydromt
+          path: /usr/share/miniconda3/envs/hydromt
+          key: ${{ matrix.python-version }}-conda-${{ hashFiles('environment.yml') }}-${{ env.DATE }}
+        id: cache
+
+      - name: Update environment & write to cache
+        run: mamba env update -n hydromt -f environment.yml
+        if: steps.cache.outputs.cache-hit != 'true'
 
       - name: Test
         run: python -m pytest --verbose --cov=hydromt --cov-report xml
 
-      - uses: codecov/codecov-action@v3
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+      - name: Upload code coverage
+        uses: codecov/codecov-action@v3

make_env.py

@@ -62,6 +62,7 @@ def _parse_profile(profile_str: str, opt_deps: dict, project_name: str) -> List[
 channels = install_config.get("channels", ["conda-forge"])
 if args.channels is not None:
     channels.extend(args.channels.split(","))
+    channels = list(set(channels))
 
 # parse environment name
 name = args.name

pyproject.toml

@@ -72,6 +72,7 @@ test = [
     "pytest>=2.7.3", # testing framework
     "pytest-cov",    # test coverage
     "pytest-mock",   # mocking
+    "pytest-timeout", # darn hanging tests
 ]
 doc = [
     "nbsphinx",                     # build notebooks in docs
@@ -136,7 +137,9 @@ include = ["hydromt"]
 exclude = ["docs", "examples", "envs", "tests", "binder", ".github"]
 
 [tool.pytest.ini_options]
+addopts = "--ff --timeout=60"
 testpaths = ["tests"]
+
 filterwarnings = [
     "ignore:distutils Version classes are deprecated:DeprecationWarning",
     "ignore:getargs:DeprecationWarning",                                     # The 'u' format is deprecated. Use 'U' instead.