ci.yml

name: Continuous Integration

on:
  pull_request:
    branches:
      - '**'
  push:
    branches:
      - '**'
    tags:
      - 'v*'

jobs:

  build-scala:
    runs-on: ubuntu-20.04

    strategy:
      matrix:
        spark_version:
          - "3.2.4"
          - "3.3.2"
          - "3.4.0"

    steps:
      - name: Checkout Repository
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Setup Scala Build Tools
        uses: ./.github/actions/init-scala-env

      - name: Compile Scala Project
        env:
          SPARK_VERSION: ${{ matrix.spark_version }}
        run: make compile-scala

      - name: Test Scala Project
        # python/* branches are not supposed to change scala code, trust them
        if: ${{ !startsWith(github.event.inputs.from_branch, 'python/') }}
        env:
          SPARK_VERSION: ${{ matrix.spark_version }}
        run:
          ulimit -c unlimited
          make test-scala

      - name: Build Spark Assembly
        env:
          SPARK_VERSION: ${{ matrix.spark_version }}
        shell: bash
        run: make build-scala

      - name: Cache Spark Assembly
        uses: actions/cache@v3
        with:
          path: ./dist/*
          key: dist-${{ matrix.spark_version }}-${{ github.sha }}

  build-python:
    # scala/* branches are not supposed to change python code, trust them
    if: ${{ !startsWith(github.event.inputs.from_branch, 'scala/') }}
    runs-on: ubuntu-20.04
    needs: build-scala

    strategy:
      matrix:
        python:
          - "3.8"
          - "3.9"
        spark_version:
          - "3.2.4"
          - "3.3.2"
          - "3.4.0"

    steps:
      - name: Checkout Repository
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - uses: ./.github/actions/init-python-env
        with:
          python_version: ${{ matrix.python }}
          spark_version: ${{ matrix.spark_version }}

      - name: Static checks
        shell: bash
        run: make lint-python

      - uses: actions/cache@v3
        with:
          path: ./dist/*
          key: dist-${{ matrix.spark_version }}-${{ github.sha }}

      - name: Run tests
        env:
          SPARK_VERSION: ${{ matrix.spark_version }}
        shell: bash
        run: make test-python-quick

  publish-scala:
    name: Publish Scala Artifacts
    needs: [ build-scala, build-python ]
    runs-on: ubuntu-20.04
    if: (github.event_name != 'pull_request') && startsWith(github.ref, 'refs/tags/v')

    strategy:
      matrix:
        spark_version:
          - "3.2.4"
          - "3.3.2"
          - "3.4.0"

    steps:
      - name: Checkout Repository
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Setup Scala Build Tools
        uses: ./.github/actions/init-scala-env

      - name: Publish JARs to GitHub Packages
        shell: bash
        env:
          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
          SPARK_VERSION: ${{ matrix.spark_version }}
        run: make publish-scala

      - name: Build Spark Assembly
        env:
          SPARK_VERSION: ${{ matrix.spark_version }}
        shell: bash
        run: make build-scala

      - name: Cache Spark Assembly
        uses: actions/cache@v3
        with:
          path: ./dist/*
          key: dist-${{ matrix.spark_version }}-${{ github.ref }}


  publish-python:
    name: Publish Scala Artifacts
    needs: [ publish-scala ]
    runs-on: ubuntu-20.04
    if: (github.event_name != 'pull_request') && startsWith(github.ref, 'refs/tags/v')

    strategy:
      matrix:
        python:
          - "3.8"
          - "3.9"
        spark_version:
          - "3.2.4"
          - "3.3.2"
          - "3.4.0"

    steps:
      - name: Checkout Repository
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - uses: ./.github/actions/init-python-env
        with:
          python_version: ${{ matrix.python }}
          spark_version: ${{ matrix.spark_version }}

      - uses: actions/cache@v3
        with:
          path: ./dist/*
          key: dist-${{ matrix.spark_version }}-${{ github.ref }}

      - name: Build Python whl
        shell: bash
        run: make build-python

# TODO: Where does this go, do we need it?
#      - name: upload artefacts
#        uses: ./.github/actions/upload_artefacts

# TODO: Where does this go, do we need it?
#      - uses: actions/cache@v3
#        with:
#          path: ./dist/*
#          key: dist-${{ github.sha }}

# TODO: Where does this go?
#      - name: upload wheel
#        working-directory: dist
#        shell: bash
#        run: