From 9c8052ab394a8c5eb23dd6533b27e3472f544281 Mon Sep 17 00:00:00 2001 From: Sam Vente Date: Thu, 10 Aug 2023 15:29:01 +0200 Subject: [PATCH] Cahce dependencies in CI to speedup workflows (#471) --- .github/workflows/docs.yml | 38 ++++++++++++------- .github/workflows/linting.yml | 1 - .github/workflows/publish-docker.yml | 11 ++---- .github/workflows/publish-pypi.yml | 1 - .github/workflows/purge_all_cache.yml | 17 +++++++++ .github/workflows/purge_pr_cache.yml | 34 +++++++++++++++++ .github/workflows/tests.yml | 54 +++++++++++++++++---------- make_env.py | 1 + pyproject.toml | 3 ++ 9 files changed, 117 insertions(+), 43 deletions(-) create mode 100644 .github/workflows/purge_all_cache.yml create mode 100644 .github/workflows/purge_pr_cache.yml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c1416e23f..fefdb89a7 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,7 +1,6 @@ --- name: Build Documentation -# We don't want pushes (or PRs) to gh-pages to kick anything off on: push: branches: [main] @@ -23,9 +22,7 @@ on: - pyproject.toml jobs: - # Build docs on Linux Docs: - timeout-minutes: 10 name: linux docs runs-on: ubuntu-latest env: @@ -37,22 +34,35 @@ jobs: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true steps: - - name: Checkout source + - name: checkout code uses: actions/checkout@v3 - - name: install tomli - run: pip install tomli + + - name: Setup Mambaforge + uses: conda-incubator/setup-miniconda@v2 + with: + python-version: 3.11 + miniforge-variant: Mambaforge + miniforge-version: latest + channels: conda-forge + activate-environment: hydromt + use-mamba: true - name: Generate env spec run: python make_env.py doc - - name: Setup environment - uses: conda-incubator/setup-miniconda@v2 + + - name: Set cache date + run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV + + - name: load from cache if it exists for our environment + uses: actions/cache@v2 with: - python-version: '3.10' - miniforge-variant: Mambaforge - channels: conda-forge, defaults - channel-priority: strict - environment-file: environment.yml - activate-environment: hydromt + path: /usr/share/miniconda3/envs/hydromt + key: docs-conda-${{ hashFiles('environment.yml') }}-${{ env.DATE }} + id: cache + + - name: Update environment & write to cache + run: mamba env update -n hydromt -f environment.yml + if: steps.cache.outputs.cache-hit != 'true' - name: Install hydromt run: pip install . diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 9c2e4eaac..e8272ec5e 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -9,7 +9,6 @@ on: jobs: pre-commit: runs-on: ubuntu-latest - timeout-minutes: 10 steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index da48e0b0b..c79365430 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -1,13 +1,10 @@ --- name: Build and publish HydroMT to DockerHub -# on: -# release: -# types: -# - published -# push: -# tags: -# - v* +on: + release: + types: + - published jobs: publish-docker-image: diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index ec5948133..d622b5a9e 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -77,7 +77,6 @@ jobs: echo "hydromt --models" upload-to-test-pypi: - timeout-minutes: 10 needs: test-built-dist if: github.event_name == 'push' runs-on: ubuntu-latest diff --git a/.github/workflows/purge_all_cache.yml b/.github/workflows/purge_all_cache.yml new file mode 100644 index 000000000..40dbd8684 --- /dev/null +++ b/.github/workflows/purge_all_cache.yml @@ -0,0 +1,17 @@ +name: cleanup all caches +on: + workflow_disptach: + + +jobs: + cleanup: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Cleanup caches + run: | + gh cache delete -a + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/purge_pr_cache.yml b/.github/workflows/purge_pr_cache.yml new file mode 100644 index 000000000..fd0584fb6 --- /dev/null +++ b/.github/workflows/purge_pr_cache.yml @@ -0,0 +1,34 @@ + +name: cleanup caches by a branch +on: + pull_request: + types: + - closed + +jobs: + cleanup: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Cleanup caches + run: | + gh extension install actions/gh-actions-cache + + REPO=${{ github.repository }} + BRANCH="refs/pull/${{ github.event.pull_request.number }}/merge" + + echo "Fetching list of cache key" + cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH -L 100 | cut -f 1 ) + + ## Setting this to not fail the workflow while deleting cache keys. + set +e + echo "Deleting caches..." + for cacheKey in $cacheKeysForPR + do + gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm + done + echo "Done" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1ac039b32..f91b1aa6c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,4 +1,3 @@ ---- name: Tests on: @@ -10,7 +9,8 @@ on: - data/* - pyproject.toml pull_request: - branches: [main] + branches: + - main paths: - tests/* - hydromt/* @@ -18,36 +18,50 @@ on: - pyproject.toml jobs: - Test-matrix: - name: ${{ matrix.os }} - py${{ matrix.python-version }} - runs-on: ${{ matrix.os }} - timeout-minutes: 10 + build: defaults: run: shell: bash -l {0} strategy: - fail-fast: false matrix: os: [ubuntu-latest] - python-version: ['3.9', '3.10', '3.11'] + python-version: ['3.9','3.10','3.11'] + name: py ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + concurrency: + group: ${{ github.workflow }}-${{ matrix.python-version }}-${{ github.ref }} + cancel-in-progress: true steps: - uses: actions/checkout@v3 + + - name: Setup Mambaforge + uses: conda-incubator/setup-miniconda@v2 + with: + python-version: ${{ matrix.python-version }} + miniforge-variant: Mambaforge + miniforge-version: latest + activate-environment: hydromt + use-mamba: true - name: Generate env spec run: pip install tomli && python make_env.py test - - uses: conda-incubator/setup-miniconda@v2 + + + - name: Set cache date + run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV + + - name: load from cache if it exists for our environment + uses: actions/cache@v2 with: - python-version: ${{ matrix.python-version }} - miniforge-variant: Mambaforge - channels: conda-forge, defaults - channel-priority: strict - environment-file: environment.yml - activate-environment: hydromt + path: /usr/share/miniconda3/envs/hydromt + key: ${{ matrix.python-version }}-conda-${{ hashFiles('environment.yml') }}-${{ env.DATE }} + id: cache + + - name: Update environment & write to cache + run: mamba env update -n hydromt -f environment.yml + if: steps.cache.outputs.cache-hit != 'true' - name: Test run: python -m pytest --verbose --cov=hydromt --cov-report xml - - uses: codecov/codecov-action@v3 - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + - name: Upload code coverage + uses: codecov/codecov-action@v3 diff --git a/make_env.py b/make_env.py index 34c38ac3e..d73be9234 100644 --- a/make_env.py +++ b/make_env.py @@ -62,6 +62,7 @@ def _parse_profile(profile_str: str, opt_deps: dict, project_name: str) -> List[ channels = install_config.get("channels", ["conda-forge"]) if args.channels is not None: channels.extend(args.channels.split(",")) + channels = list(set(channels)) # parse environment name name = args.name diff --git a/pyproject.toml b/pyproject.toml index e2a45be10..ba73bd39f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,7 @@ test = [ "pytest>=2.7.3", # testing framework "pytest-cov", # test coverage "pytest-mock", # mocking + "pytest-timeout", # darn hanging tests ] doc = [ "nbsphinx", # build notebooks in docs @@ -136,7 +137,9 @@ include = ["hydromt"] exclude = ["docs", "examples", "envs", "tests", "binder", ".github"] [tool.pytest.ini_options] +addopts = "--ff --timeout=60" testpaths = ["tests"] + filterwarnings = [ "ignore:distutils Version classes are deprecated:DeprecationWarning", "ignore:getargs:DeprecationWarning", # The 'u' format is deprecated. Use 'U' instead.