CI #1923
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - '*' | |
| tags: | |
| - v* | |
| schedule: | |
| # daily (`@daily` not supported, see | |
| # https://docs.github.com/en/actions/reference/events-that-trigger-workflows#scheduled-events ) | |
| # | |
| # Runs on default/base branch (see | |
| # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onschedule ) | |
| - cron: '0 0 * * *' | |
| jobs: | |
| build: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| numfocus_nightly: [false] | |
| os: ["ubuntu-latest"] | |
| pyarrow: ["0.17.1", "1.0.1", "2.0.0", "3.0.0", "4.0.1", "5.0.0", "6.0.1", "nightly"] | |
| python: ["3.7", "3.8"] | |
| include: | |
| - numfocus_nightly: true | |
| os: "ubuntu-latest" | |
| pyarrow: "2.0.0" | |
| python: "3.8" | |
| - numfocus_nightly: false | |
| os: "macos-latest" | |
| pyarrow: "0.17.1" | |
| python: "3.8" | |
| continue-on-error: ${{ matrix.numfocus_nightly || matrix.pyarrow == 'nightly' }} | |
| runs-on: ${{ matrix.os }} | |
| defaults: | |
| run: | |
| # see https://github.com/conda-incubator/setup-miniconda/#important | |
| shell: bash -l {0} | |
| env: | |
| IS_MASTER_BUILD: ${{ !matrix.numfocus_nightly && matrix.os == 'ubuntu-latest' && matrix.pyarrow == '0.17.1' && matrix.python == '3.8' }} | |
| IS_TAG: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')}} | |
| steps: | |
| # CI setup | |
| - name: Is Master Build? | |
| run: echo $IS_MASTER_BUILD | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| - name: Cache conda | |
| uses: actions/[email protected] | |
| with: | |
| path: ~/conda_pkgs_dir | |
| key: ${{ runner.os }}-${{ matrix.python }}-${{ matrix.pyarrow }}-${{ hashFiles('conda-requirements.txt', 'conda-test-requirements.txt')}} | |
| - name: Conda Bootstrap | |
| uses: conda-incubator/[email protected] | |
| with: | |
| auto-update-conda: true | |
| channel-priority: strict | |
| channels: conda-forge | |
| python-version: ${{ matrix.python }} | |
| - name: Conda Config | |
| run: | | |
| conda config --set always_yes yes | |
| conda config --set changeps1 no | |
| # Installation into the `test` environment | |
| - name: Conda Install Pyarrow (non-nightly) | |
| run: conda install pyarrow==${{ matrix.pyarrow }} | |
| if: matrix.pyarrow != 'nightly' | |
| - name: Conda Install Pyarrow (nightly) | |
| # Install both arrow-cpp and pyarrow to make sure that we have the | |
| # latest nightly of both packages. It is sadly not guaranteed that the | |
| # nightlies and the latest release would otherwise work together. | |
| run: conda install -c arrow-nightlies arrow-cpp pyarrow | |
| if: matrix.pyarrow == 'nightly' | |
| - name: Conda Install conda-requirements.txt | |
| # run this AFTER pyarrow, since this is less restrictive (so conda doesn't spend too much time downgrading) | |
| run: conda install --file=conda-requirements.txt | |
| - name: Fix Pandas-Pyarrow Interaction | |
| # pandas >=1.2.0 requires newer pyarrow versions, so downgrade pandas | |
| run: conda install 'pandas<1.2' | |
| if: matrix.pyarrow == '0.17.1' && !matrix.numfocus_nightly | |
| - name: Pip Install kartothek | |
| run: pip install --no-deps . | |
| - name: Test import | |
| run: | | |
| python -c "import kartothek" | |
| python -c "import kartothek.api" | |
| python -c "import kartothek.api.consistency" | |
| python -c "import kartothek.api.cube" | |
| python -c "import kartothek.api.dataset" | |
| python -c "import kartothek.api.discover" | |
| python -c "import kartothek.api.serialization" | |
| python -c "import kartothek.cli" | |
| python -c "import kartothek.core" | |
| python -c "import kartothek.io" | |
| python -c "import kartothek.io_components" | |
| python -c "import kartothek.serialization" | |
| python -c "import kartothek.utils" | |
| - name: Conda install test requirements | |
| run: conda install | |
| --file=conda-test-requirements.txt | |
| --freeze-installed | |
| - name: Pip Instal NumFOCUS nightly | |
| # NumFOCUS nightly wheels, contains numpy and pandas | |
| run: pip install --pre --upgrade --timeout=60 --extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple pandas numpy | |
| if: matrix.numfocus_nightly | |
| - name: Conda Export | |
| run: conda list --export | |
| # Tests | |
| - name: Pytest | |
| run: pytest --cov --cov-report xml | |
| - name: Running benchmarks | |
| run: | | |
| asv --config ./asv_bench/asv.conf.json machine --machine github --os unknown --arch unknown --cpu unknown --ram unknown | |
| asv --config ./asv_bench/asv.conf.json dev | sed "/failed$/ s/^/##[error]/" | tee benchmarks.log | |
| if grep "failed" benchmarks.log > /dev/null ; then | |
| exit 1 | |
| fi | |
| if: env.IS_MASTER_BUILD == 'true' | |
| # Builds | |
| - name: Build Wheel | |
| run: python setup.py sdist bdist_wheel | |
| - name: Codecov | |
| uses: codecov/[email protected] | |
| with: | |
| # NOTE: `token` is not required, because the kartothek repo is public | |
| file: ./coverage.xml | |
| name: pytest-numfocus_nightly_${{ matrix.numfocus_nightly }}-os_${{ matrix.os }}-pyarrow_${{ matrix.pyarrow }}-python_${{ matrix.python }} | |
| # Release | |
| - name: Publish to PyPI | |
| if: env.IS_MASTER_BUILD == 'true' && env.IS_TAG == 'true' | |
| uses: pypa/gh-action-pypi-publish@master | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.PYPI_PASSWORD }} | |
| - name: Create Release Notes | |
| if: env.IS_MASTER_BUILD == 'true' && env.IS_TAG == 'true' | |
| run: | | |
| awk 'BEGIN{found=0} {if (match($0, "==============")) {if (found == 1) exit; found=1}; if (found == 1) {print last}; last=$0}' CHANGES.rst > release_notes.rst | |
| pandoc --from=rst --to=markdown -o release_notes.md release_notes.rst | |
| - name: Create GitHub Release | |
| if: env.IS_MASTER_BUILD == 'true' && env.IS_TAG == 'true' | |
| uses: actions/[email protected] | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| tag_name: ${{ github.ref }} | |
| release_name: Kartothek ${{ github.ref }} | |
| body_path: release_notes.md | |
| draft: false | |
| prerelease: false |