CI/CD #5394
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - backports-v0.7.x | |
| - backports-v0.6.x | |
| tags: | |
| - v*.*.* | |
| pull_request: | |
| branches: | |
| - master | |
| - backports-v0.7.x | |
| - backports-v0.6.x | |
| # Run daily at 0:01 UTC | |
| schedule: | |
| - cron: '1 0 * * *' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| pre-commit: | |
| name: pre-commit | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| - uses: pre-commit/[email protected] | |
| with: | |
| extra_args: --hook-stage manual --all-files | |
| test: | |
| runs-on: ${{ matrix.os }} | |
| needs: pre-commit | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, macOS-latest, windows-latest] | |
| java-version: [17] | |
| java-distribution: ["corretto"] | |
| python-version: ["3.9", "3.12"] | |
| name: test coffea (${{ matrix.os }}) - python ${{ matrix.python-version }}, JDK${{ matrix.java-version }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Set up JDK ${{ matrix.java-distribution }}/${{ matrix.java-version }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| distribution: ${{ matrix.java-distribution }} | |
| - name: Set python test settings | |
| run: | | |
| echo "INSTALL_EXTRAS='[dev,parsl,dask,servicex]'" >> $GITHUB_ENV | |
| - name: Install uv | |
| run: python -m pip install --upgrade uv | |
| - name: Install dependencies (Linux) | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| uv pip install --system --upgrade pip setuptools wheel | |
| # mltool installs | |
| # c.f. https://github.com/astral-sh/uv/issues/3437 | |
| python -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu | |
| uv pip install --system xgboost | |
| uv pip install --system 'tritonclient[grpc,http]!=2.41.0' | |
| # install checked out coffea | |
| uv pip install --system -q '.[dev,parsl,dask,spark]' --upgrade | |
| uv pip list --system | |
| java -version | |
| - name: Install dependencies (MacOS) | |
| if: matrix.os == 'macOS-latest' | |
| run: | | |
| uv pip install --system --upgrade pip setuptools wheel | |
| # mltool installs | |
| # c.f. https://github.com/astral-sh/uv/issues/3437 | |
| python -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu | |
| uv pip install --system xgboost | |
| # install checked out coffea | |
| uv pip install --system -q '.[dev,dask,spark]' --upgrade | |
| uv pip list --system | |
| java -version | |
| - name: Install dependencies (Windows) | |
| if: matrix.os == 'windows-latest' | |
| run: | | |
| uv pip install --system --upgrade pip setuptools wheel | |
| # mltool installs | |
| # c.f. https://github.com/astral-sh/uv/issues/3437 | |
| python -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu | |
| uv pip install --system xgboost | |
| # install checked out coffea | |
| uv pip install --system -q '.[dev,dask]' 'numpy<2' --upgrade | |
| uv pip list --system | |
| java -version | |
| - name: Start triton server with example model | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| docker run -d --rm -p 8000:8000 -p 8001:8001 -p 8002:8002 -v ${{ github.workspace }}/tests/samples/triton_models_test:/models nvcr.io/nvidia/tritonserver:23.04-pyt-python-py3 tritonserver --model-repository=/models | |
| - name: Test with pytest | |
| run: | | |
| python -m pytest --cov-report=xml --cov=coffea --deselect=test_taskvine | |
| - name: Upload codecov | |
| if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.12 | |
| uses: codecov/codecov-action@v4 | |
| - name: Install graphviz | |
| if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.12 | |
| uses: ts-graphviz/setup-graphviz@v2 | |
| - name: Install pandoc | |
| if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.12 | |
| uses: r-lib/actions/setup-pandoc@v2 | |
| - name: Build documentation | |
| if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.12 | |
| run: | | |
| cd docs && make html | |
| touch build/html/.nojekyll | |
| - name: Deploy documentation | |
| if: github.event_name == 'push' && matrix.os == 'ubuntu-latest' && matrix.python-version == 3.12 | |
| uses: crazy-max/ghaction-github-pages@v4 | |
| with: | |
| target_branch: gh-pages | |
| build_dir: docs/build/html | |
| env: | |
| GH_PAT: ${{ secrets.GITHUB_OAUTH }} | |
| test-vine: | |
| runs-on: ubuntu-latest | |
| needs: pre-commit | |
| strategy: | |
| matrix: | |
| python-version: ["3.11"] | |
| name: test coffea-taskvine | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Conda | |
| uses: conda-incubator/setup-miniconda@v3 | |
| env: | |
| ACTIONS_ALLOW_UNSECURE_COMMANDS: true | |
| with: | |
| auto-update-conda: true | |
| python-version: ${{ matrix.python-version }} | |
| channels: conda-forge | |
| - name: Construct conda environment | |
| shell: bash -l {0} | |
| run: | | |
| conda create --yes --prefix ./coffea-conda-test-env -c conda-forge python=${{ matrix.python-version }} ndcctools uv | |
| conda activate ./coffea-conda-test-env | |
| uv pip install --system . | |
| uv pip install --system pytest | |
| - name: Test with pytest | |
| run: | | |
| conda run --prefix ./coffea-conda-test-env pytest tests/test_taskvine.py | |
| # testskyhookjob: | |
| # runs-on: ubuntu-latest | |
| # needs: pre-commit | |
| # name: test coffea-skyhook-job | |
| # | |
| # steps: | |
| # - uses: actions/checkout@3 | |
| # - name: Test Coffea Skyhook Bindings | |
| # shell: bash -l {0} | |
| # run: | | |
| # docker build -t coffea-skyhook-test \ | |
| # --file docker/skyhook/Dockerfile \ | |
| # . | |
| # docker run \ | |
| # -v $(pwd):/w \ | |
| # -w /w \ | |
| # -e IS_CI=true \ | |
| # --privileged \ | |
| # coffea-skyhook-test \ | |
| # ./docker/skyhook/script.sh | |
| release: | |
| if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') | |
| runs-on: ubuntu-latest | |
| needs: [test, test-vine] | |
| permissions: | |
| id-token: write | |
| attestations: write | |
| contents: read | |
| strategy: | |
| matrix: | |
| python-version: ["3.12"] | |
| name: deploy release | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Build package for PyPI | |
| run: | | |
| pipx run hatch build -t sdist -t wheel | |
| - name: Verify the distribution | |
| run: pipx run twine check --strict dist/* | |
| - name: Generate artifact attestation for sdist and wheel | |
| uses: actions/attest-build-provenance@1c608d11d69870c2092266b3f9a6f3abbf17002c # v1.4.3 | |
| with: | |
| subject-path: "dist/coffea-*" | |
| - name: Publish package to PyPI | |
| uses: pypa/[email protected] | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.PYPI_TOKEN }} | |
| pass: | |
| needs: [test, test-vine] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - run: echo "All jobs passed" |