TPI layout overhaul #11120

Workflow file for this run

.github/workflows/integration.yml at bbfa43a

	name: Integration

	on:
	pull_request:
	branches:
	- main
	jobs:
	lint-api:
	name: Python - Lint
	runs-on: ubuntu-24.04
	strategy:
	matrix:
	python-version: [3.12.3]
	steps:
	- name: Checkout repo
	uses: actions/checkout@v4
	- name: Install ubuntu pre-requisites (api)
	# The python gdal and R component relies on libgdal-dev being installed.
	# cffdrs requires libudunits2-dev
	run: \|
	sudo apt-get update
	sudo apt-get -y install libudunits2-dev r-base-dev libgdal-dev libproj-dev libgeos-dev libsqlite3-dev libtirpc-dev
	- name: Setup Python ${{ matrix.python-version }} (api)
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- name: Cache poetry installer
	uses: actions/cache@v4
	id: cache-poetry-installer
	env:
	cache-name: cache-poetry-installer
	with:
	path: "~/poetry_installer"
	key: "poetry-installer-1.8.3"
	- name: Download poetry installer
	if: steps.cache-poetry-installer.outputs.cache-hit != 'true'
	run: \|
	echo
	mkdir ~/poetry_installer
	curl -sSL https://install.python-poetry.org > ~/poetry_installer/install-poetry.py
	- name: Install poetry (api)
	run: \|
	cd ~/poetry_installer
	python install-poetry.py --version 1.8.3
	poetry config virtualenvs.create true
	poetry config virtualenvs.in-project false
	- name: Cache poetry
	uses: actions/cache@v4
	env:
	cache-name: cache-poetry
	with:
	path: ~/.cache/pypoetry
	key: ${{ runner.os }}-poetry-1.8.3-cache-${{ hashFiles('**/poetry.lock') }}
	restore-keys: \|
	${{ runner.os }}-poetry-1.8.3-cache-
	- name: Cache .venv
	id: cache-venv
	uses: actions/cache@v4
	with:
	path: ~/work/wps/wps/api/.venv
	key: ${{ runner.os }}-venv-poetry-1.8.3-${{ hashFiles('**/poetry.lock') }}
	- name: Install python dependencies using poetry (api)
	if: steps.cache-venv.outputs.cache-hit != 'true'
	working-directory: ./api
	run: \|
	poetry run python -m pip install --upgrade pip
	poetry install
	poetry run python -m pip install -U setuptools wheel
	poetry run python -m pip install --no-build-isolation --no-cache-dir --force-reinstall gdal==$(gdal-config --version)
	- name: Lint (api)
	# We used to be able to do linting before installing gdal, but it's not possible anymore.
	# We can however place it ahead of the R installs.
	working-directory: ./api
	run: \|
	poetry run ruff app/.py app//.py

	test-api:
	name: Python - Test with coverage
	runs-on: ubuntu-24.04
	strategy:
	matrix:
	# Match versions for python to Ubuntu 24.04 LTS at the time of writing. Keep R at same version.
	python-version: [3.12.3]
	R: ["4.1.2"]
	steps:
	- name: Checkout repo
	uses: actions/checkout@v4
	with:
	# For sonar-scanner to work properly we can't use a shallow fetch.
	fetch-depth: 0
	- name: Install ubuntu pre-requisites (api)
	# The python gdal and R component relies on libgdal-dev being installed.
	# cffdrs requires libudunits2-dev
	# The api uses wkhtmltopdf to generate pdf's.
	run: \|
	sudo apt-get update
	sudo apt-get -y install wkhtmltopdf libudunits2-dev r-base-dev build-essential libgdal-dev libproj-dev libgeos-dev libsqlite3-dev libtirpc-dev
	- name: Setup Python ${{ matrix.python-version }} (api)
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- name: Cache poetry installer
	uses: actions/cache@v4
	id: cache-poetry-installer
	env:
	cache-name: cache-poetry-installer
	with:
	path: "~/poetry_installer"
	key: "poetry-installer-1.8.3"
	- name: Download poetry installer
	if: steps.cache-poetry-installer.outputs.cache-hit != 'true'
	run: \|
	echo
	mkdir ~/poetry_installer
	curl -sSL https://install.python-poetry.org > ~/poetry_installer/install-poetry.py
	- name: Install poetry (api)
	run: \|
	cd ~/poetry_installer
	python install-poetry.py --version 1.8.3
	poetry config virtualenvs.create true
	poetry config virtualenvs.in-project false
	# poetry cache folder: /home/runner/.cache/pypoetry
	- name: Cache poetry
	uses: actions/cache@v4
	env:
	cache-name: cache-poetry
	with:
	path: ~/.cache/pypoetry
	key: ${{ runner.os }}-poetry-1.8.3-cache-${{ hashFiles('**/poetry.lock') }}
	restore-keys: \|
	${{ runner.os }}-poetry-1.8.3-cache-
	- name: Cache .venv
	id: cache-venv
	uses: actions/cache@v4
	with:
	path: ~/work/wps/wps/api/.venv
	key: ${{ runner.os }}-venv-poetry-1.8.3-${{ hashFiles('**/poetry.lock') }}
	- name: Install python dependencies using poetry (api)
	if: steps.cache-venv.outputs.cache-hit != 'true'
	working-directory: ./api
	run: \|
	poetry run python -m pip install --upgrade pip
	poetry install
	poetry run python -m pip install -U setuptools wheel
	poetry run python -m pip install --no-build-isolation --no-cache-dir --force-reinstall gdal==$(gdal-config --version)
	- uses: r-lib/actions/setup-r@v2
	# r-lib/actions/setup-r@v2 is supposed to install version ${{ matrix.R }}, BUT, it doesn't.
	# When asking for 4.1.2 - it's installing a new version! Or at least reporting a newer
	# version.
	with:
	r-version: ${{ matrix.R }}
	- name: Cache /home/runner/work/_temp/Library
	# When installing cffdrs, a bunch of stuff is downloaded and compiled and placed in
	# /home/runner/work/_temp/Library ; By caching this folder, subsequent calls to
	# install cffdrs run much faster.
	id: cache-r-cffdrs
	uses: actions/cache@v4
	with:
	path: /home/runner/work/_temp/Library
	key: ${{ runner.os }}-r-${{ matrix.R }}-cffdrs
	- name: Install R dependencies (api)
	run: R -e "install.packages('cffdrs')"
	- name: Unit Test with coverage (api)
	working-directory: ./api
	run: \|
	export LD_LIBRARY_PATH=$(poetry run python -m rpy2.situation LD_LIBRARY_PATH):${LD_LIBRARY_PATH}
	export CLASSPATH=./libs/REDapp_Lib.jar:./libs/WTime.jar:./libs/hss-java.jar:$CLASSPATH
	export ORIGINS=testorigin
	export SFMS_SECRET=secret
	poetry run coverage run --source=app -m pytest app/tests -x -o log_cli=true --disable-warnings -vvv
	- name: Create coverage report (api)
	working-directory: ./api
	shell: bash
	run: \|
	poetry run coverage report
	poetry run coverage xml -o coverage-reports/coverage-report.xml
	- name: Archive coverage report (api)
	uses: actions/upload-artifact@v4
	with:
	name: api-coverage-report
	path: ./api/coverage-reports/coverage-report.xml
	lint-and-test-web:
	name: Web - Lint, Test with coverage
	runs-on: ubuntu-24.04
	strategy:
	matrix:
	node-version: [20.x]
	steps:
	- name: Checkout repo
	uses: actions/checkout@v4
	with:
	# For sonar-scanner to work properly we can't use a shallow fetch.
	fetch-depth: 0
	- name: Use Node.js ${{ matrix.node-version }}
	uses: actions/setup-node@v4
	with:
	node-version: ${{ matrix.node-version }}
	- uses: actions/cache@v4
	with:
	path: \|
	**/node_modules
	~/.cache/Cypress
	key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
	- name: enable corepack
	run: corepack enable
	- name: use new yarn
	run: yarn set version berry
	- name: Install node dependencies (web)
	working-directory: ./web
	if: steps.yarn-cache.outputs.cache-hit != 'true'
	run: yarn install
	- name: Lint (web)
	working-directory: ./web
	run: yarn run lint
	- name: Cypress tests with coverage (web)
	working-directory: ./web
	run: yarn run cypress:ci
	- name: Unit tests (web)
	working-directory: ./web
	run: yarn run coverage:ci
	- name: Merge and finalize test coverage (web)
	working-directory: ./web
	run: yarn run finalizeCoverage
	- name: Archive coverage report
	uses: actions/upload-artifact@v4
	with:
	name: web-coverage-report
	path: ./web/finalCoverage
	upload-code-coverage:
	name: Coverage with CodeCov
	runs-on: ubuntu-24.04
	needs: [lint-and-test-web, test-api]
	steps:
	# we need to checkout, so that we have codecov.yml
	- name: Checkout repo
	uses: actions/checkout@v4
	- name: Download all workflow run artifacts
	uses: actions/download-artifact@v4
	- name: Upload test coverage to Codecov
	uses: codecov/codecov-action@v4
	with:
	token: ${{secrets.CODECOV_TOKEN}}
	# we want to ensure code coverage is good, so we fail on error. (default is false)
	fail_ci_if_error: true
	lint-and-test-prune:
	name: Backup Prune - Lint and Test
	# We have to use an older ubuntu version, because of the older version of python that the
	# backup script uses.
	runs-on: ubuntu-24.04
	strategy:
	matrix:
	python-version: [3.12.3]
	steps:
	- name: Checkout repo
	uses: actions/checkout@v4
	with:
	# For sonar-scanner to work properly we can't use a shallow fetch.
	fetch-depth: 0
	- name: Setup Python ${{ matrix.python-version }} (api)
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- name: Cache poetry installer
	uses: actions/cache@v4
	id: cache-poetry-installer
	env:
	cache-name: cache-poetry-installer
	with:
	path: "~/poetry_installer"
	key: "poetry-installer-1.8.3"
	- name: Download poetry installer
	if: steps.cache-poetry-installer.outputs.cache-hit != 'true'
	run: \|
	echo
	mkdir ~/poetry_installer
	curl -sSL https://install.python-poetry.org > ~/poetry_installer/install-poetry.py
	- name: Install poetry (api)
	run: \|
	cd ~/poetry_installer
	python install-poetry.py --version 1.8.3
	poetry config virtualenvs.create true
	poetry config virtualenvs.in-project false
	# poetry cache folder: /home/runner/.cache/pypoetry
	- name: Cache poetry
	uses: actions/cache@v4
	env:
	cache-name: cache-poetry
	with:
	path: ~/.cache/pypoetry
	key: ${{ runner.os }}-poetry-prune-cache-${{ hashFiles('**/poetry.lock') }}
	restore-keys: \|
	${{ runner.os }}-poetry-prune-cache-
	- name: Install python dependencies using poetry (s3-backup)
	working-directory: ./openshift/s3-backup/docker
	run: \|
	poetry install
	- name: Lint (s3-backup)
	working-directory: ./openshift/s3-backup/docker
	run: poetry run ruff check *.py
	- name: Unit tests (s3-backup)
	working-directory: ./openshift/s3-backup/docker
	run: \|
	poetry run pytest -v

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

TPI layout overhaul #11120

Workflow file

TPI layout overhaul #11120

Jobs

Run details

Workflow file for this run