integration.yml

name: Integration

on:
  pull_request:
    branches:
      - main
jobs:
  lint-api:
    name: Python - Lint
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        python-version: [3.10.4]
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
      - name: Install ubuntu pre-requisites (api)
        # The python gdal and R component relies on libgdal-dev being installed.
        # cffdrs requires libudunits2-dev
        run: |
          sudo apt-get update
          sudo apt-get -y install libgdal-dev libudunits2-dev
      - name: Setup Python ${{ matrix.python-version }} (api)
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Cache poetry installer
        uses: actions/cache@v4
        id: cache-poetry-installer
        env:
          cache-name: cache-poetry-installer
        with:
          path: "~/poetry_installer"
          key: "poetry-installer-1.3.1"
      - name: Download poetry installer
        if: steps.cache-poetry-installer.outputs.cache-hit != 'true'
        run: |
          echo
          mkdir ~/poetry_installer
          curl -sSL https://install.python-poetry.org > ~/poetry_installer/install-poetry.py
      - name: Install poetry (api)
        run: |
          cd ~/poetry_installer
          python install-poetry.py --version 1.3.1
          poetry config virtualenvs.create true
          poetry config virtualenvs.in-project false
      - name: Cache poetry
        uses: actions/cache@v4
        env:
          cache-name: cache-poetry
        with:
          path: ~/.cache/pypoetry
          key: ${{ runner.os }}-poetry-1.3.1-cache-${{ hashFiles('**/poetry.lock') }}
          restore-keys: |
            ${{ runner.os }}-poetry-1.3.1-cache-
      - name: Cache .venv
        id: cache-venv
        uses: actions/cache@v4
        with:
          path: ~/work/wps/wps/api/.venv
          key: ${{ runner.os }}-venv-poetry-1.3.1-${{ hashFiles('**/poetry.lock') }}
      - name: Install python dependencies using poetry (api)
        if: steps.cache-venv.outputs.cache-hit != 'true'
        working-directory: ./api
        run: |
          poetry run python -m pip install --upgrade pip
          poetry install
          poetry run python -m pip install gdal==$(gdal-config --version)
      - name: Lint (api)
        # We used to be able to do linting before installing gdal, but it's not possible anymore.
        # We can however place it ahead of the R installs.
        working-directory: ./api
        run: |
          poetry run ruff app/*.py app/**/*.py

  test-api:
    name: Python - Test with coverage
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        # Match versions for python + R to Ubuntu 22.04 LTS at the time of writing.
        python-version: [3.10.4]
        R: ["4.1.2"]
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          # For sonar-scanner to work properly we can't use a shallow fetch.
          fetch-depth: 0
      - name: Install ubuntu pre-requisites (api)
        # The python gdal and R component relies on libgdal-dev being installed.
        # cffdrs requires libudunits2-dev
        # The api uses wkhtmltopdf to generate pdf's.
        run: |
          sudo apt-get update
          sudo apt-get -y install libgdal-dev wkhtmltopdf libudunits2-dev
      - name: Setup Python ${{ matrix.python-version }} (api)
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Cache poetry installer
        uses: actions/cache@v4
        id: cache-poetry-installer
        env:
          cache-name: cache-poetry-installer
        with:
          path: "~/poetry_installer"
          key: "poetry-installer-1.3.1"
      - name: Download poetry installer
        if: steps.cache-poetry-installer.outputs.cache-hit != 'true'
        run: |
          echo
          mkdir ~/poetry_installer
          curl -sSL https://install.python-poetry.org > ~/poetry_installer/install-poetry.py
      - name: Install poetry (api)
        run: |
          cd ~/poetry_installer
          python install-poetry.py --version 1.3.1
          poetry config virtualenvs.create true
          poetry config virtualenvs.in-project false
      # poetry cache folder: /home/runner/.cache/pypoetry
      - name: Cache poetry
        uses: actions/cache@v4
        env:
          cache-name: cache-poetry
        with:
          path: ~/.cache/pypoetry
          key: ${{ runner.os }}-poetry-1.3.1-cache-${{ hashFiles('**/poetry.lock') }}
          restore-keys: |
            ${{ runner.os }}-poetry-1.3.1-cache-
      - name: Cache .venv
        id: cache-venv
        uses: actions/cache@v4
        with:
          path: ~/work/wps/wps/api/.venv
          key: ${{ runner.os }}-venv-poetry-1.3.1-${{ hashFiles('**/poetry.lock') }}
      - name: Install python dependencies using poetry (api)
        if: steps.cache-venv.outputs.cache-hit != 'true'
        working-directory: ./api
        run: |
          poetry run python -m pip install --upgrade pip
          poetry install
          poetry run python -m pip install gdal==$(gdal-config --version)
      - uses: r-lib/actions/setup-r@v2
        # r-lib/actions/setup-r@v2 is supposed to install version ${{ matrix.R }}, BUT, it doesn't.
        # When asking for 4.1.2 - it's installing a new version! Or at least reporting a newer
        # version.
        with:
          r-version: ${{ matrix.R }}
      - name: Cache /home/runner/work/_temp/Library
        # When installing cffdrs, a bunch of stuff is downloaded and compiled and placed in
        # /home/runner/work/_temp/Library ; By caching this folder, subsequent calls to
        # install cffdrs run much faster.
        id: cache-r-cffdrs
        uses: actions/cache@v4
        with:
          path: /home/runner/work/_temp/Library
          key: ${{ runner.os }}-r-${{ matrix.R }}-cffdrs
      - name: Install R dependencies (api)
        run: R -e "install.packages('cffdrs')"
      - name: Unit Test with coverage (api)
        working-directory: ./api
        run: |
          export LD_LIBRARY_PATH=$(poetry run python -m rpy2.situation LD_LIBRARY_PATH):${LD_LIBRARY_PATH}
          export CLASSPATH=./libs/REDapp_Lib.jar:./libs/WTime.jar:./libs/hss-java.jar:$CLASSPATH
          export ORIGINS=testorigin
          export SFMS_SECRET=secret
          poetry run coverage run --source=app -m pytest app/tests -x -o log_cli=true --disable-warnings -vvv
      - name: Create coverage report (api)
        working-directory: ./api
        shell: bash
        run: |
          poetry run coverage report
          poetry run coverage xml -o coverage-reports/coverage-report.xml
      - name: Archive coverage report (api)
        uses: actions/upload-artifact@v3
        with:
          name: api-coverage-report
          path: ./api/coverage-reports/coverage-report.xml
  lint-and-test-web:
    name: Web - Lint, Test with coverage
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        node-version: [18.x]
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          # For sonar-scanner to work properly we can't use a shallow fetch.
          fetch-depth: 0
      - name: Setup kernel for react, increase watchers
        run: echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p
      - name: Use Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
      - uses: actions/cache@v4
        with:
          path: |
            **/node_modules
            ~/.cache/Cypress
          key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
      - name: Install node dependencies (web)
        working-directory: ./web
        if: steps.yarn-cache.outputs.cache-hit != 'true'
        run: yarn install
      - name: Lint (web)
        working-directory: ./web
        run: yarn run lint
      # "Error: ENOSPC: System limit for number of file watchers reached" can be addressed
      # with this: https://github.com/guard/listen/wiki/Increasing-the-amount-of-inotify-watchers#the-technical-details
      # It seems unnecessary at the moment because tests pass anyway
      - name: Cypress tests with coverage (web)
        working-directory: ./web
        run: yarn run cypress:ci
      - name: Unit tests (web)
        working-directory: ./web
        run: yarn run coverage:ci
      - name: Merge and finalize test coverage (web)
        working-directory: ./web
        run: yarn run finalizeCoverage

      - name: Archive coverage report
        uses: actions/upload-artifact@v3
        with:
          name: web-coverage-report
          path: ./web/finalCoverage
  upload-code-coverage:
    name: Coverage with CodeCov
    runs-on: ubuntu-22.04
    needs: [lint-and-test-web, test-api]
    steps:
      # we need to checkout, so that we have codecov.yml
      - name: Checkout repo
        uses: actions/checkout@v4
      - name: Download all workflow run artifacts
        uses: actions/download-artifact@v3
      - name: Upload test coverage to Codecov
        uses: codecov/codecov-action@v4
        with:
          token: ${{secrets.CODECOV_TOKEN}}
          # we want to ensure code coverage is good, so we fail on error. (default is false)
          fail_ci_if_error: true
  lint-and-test-prune:
    name: Backup Prune - Lint and Test
    # We have to use an older ubuntu version, because of the older version of python that the
    # backup script uses.
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        python-version: [3.11.2]
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          # For sonar-scanner to work properly we can't use a shallow fetch.
          fetch-depth: 0
      - name: Setup Python ${{ matrix.python-version }} (api)
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Cache poetry installer
        uses: actions/cache@v4
        id: cache-poetry-installer
        env:
          cache-name: cache-poetry-installer
        with:
          path: "~/poetry_installer"
          key: "poetry-installer-1.7.1"
      - name: Download poetry installer
        if: steps.cache-poetry-installer.outputs.cache-hit != 'true'
        run: |
          echo
          mkdir ~/poetry_installer
          curl -sSL https://install.python-poetry.org > ~/poetry_installer/install-poetry.py
      - name: Install poetry (api)
        run: |
          cd ~/poetry_installer
          python install-poetry.py --version 1.7.1
          poetry config virtualenvs.create true
          poetry config virtualenvs.in-project false
      # poetry cache folder: /home/runner/.cache/pypoetry
      - name: Cache poetry
        uses: actions/cache@v4
        env:
          cache-name: cache-poetry
        with:
          path: ~/.cache/pypoetry
          key: ${{ runner.os }}-poetry-prune-cache-${{ hashFiles('**/poetry.lock') }}
          restore-keys: |
            ${{ runner.os }}-poetry-prune-cache-
      - name: Install python dependencies using poetry (s3-backup)
        working-directory: ./openshift/s3-backup/docker
        run: |
          poetry install
      - name: Lint (s3-backup)
        working-directory: ./openshift/s3-backup/docker
        run: poetry run pylint *.py
      - name: Unit tests (s3-backup)
        working-directory: ./openshift/s3-backup/docker
        run: |
          poetry run pytest -v