refactor: Clean up DB sessions consistently #2239
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| pull_request_target: | |
| types: [assigned, opened, synchronize, reopened] | |
| push: | |
| branches: | |
| - main | |
| env: | |
| GCS_PROJECT: qc-oss-mamba-org-quetz-dev | |
| GCS_TOKEN: /home/runner/gcs/token.json | |
| GCS_TOKEN_CONTENT: ${{ secrets.gcs_token_content }} | |
| jobs: | |
| test_quetz: | |
| # timeout for the whole job | |
| timeout-minutes: 15 | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| test_database: ["sqlite", "postgres"] | |
| db_init: ["use-migrations", "create-tables"] | |
| services: | |
| postgres: | |
| image: postgres | |
| env: | |
| POSTGRES_PASSWORD: mysecretpassword | |
| POSTGRES_USER: postgres | |
| POSTGRES_DB: postgres | |
| ports: | |
| - 5432/tcp | |
| # Set health checks to wait until postgres has started | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| redis: | |
| image: redis | |
| options: >- | |
| --health-cmd "redis-cli ping" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 6379:6379 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: write gcs token file | |
| run: | | |
| mkdir $(dirname "${GCS_TOKEN}") | |
| echo "${GCS_TOKEN_CONTENT}" > "${GCS_TOKEN}" | |
| - name: install mamba | |
| uses: mamba-org/setup-micromamba@v1 | |
| with: | |
| environment-file: environment.yml | |
| - name: test quetz | |
| shell: bash -l -eo pipefail {0} | |
| env: | |
| TEST_DB_BACKEND: ${{ matrix.test_database }} | |
| QUETZ_TEST_DBINIT: ${{ matrix.db_init }} | |
| POSTGRES_HOST: localhost | |
| POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} | |
| S3_ACCESS_KEY: ${{ secrets.s3_access_key }} | |
| S3_SECRET_KEY: ${{ secrets.s3_secret_key }} | |
| S3_ENDPOINT: https://s3.sbg.cloud.ovh.net/ | |
| S3_REGION: sbg | |
| S3_BUCKET_NAME: quetz | |
| run: | | |
| # install dev dependencies | |
| pip install -e .[all,dev] | |
| pip install redis rq | |
| pip install pytest-github-actions-annotate-failures | |
| - name: Testing server | |
| shell: bash -l -eo pipefail {0} | |
| # timeout for the step | |
| timeout-minutes: 15 | |
| env: | |
| TEST_DB_BACKEND: ${{ matrix.test_database }} | |
| QUETZ_TEST_DBINIT: ${{ matrix.db_init }} | |
| POSTGRES_HOST: localhost | |
| POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} | |
| S3_ACCESS_KEY: ${{ secrets.s3_access_key }} | |
| S3_SECRET_KEY: ${{ secrets.s3_secret_key }} | |
| S3_ENDPOINT: https://s3.sbg.cloud.ovh.net/ | |
| S3_REGION: sbg | |
| S3_BUCKET_NAME: quetz | |
| run: | | |
| if [ "$TEST_DB_BACKEND" == "postgres" ]; then | |
| export QUETZ_TEST_DATABASE="postgresql://postgres:mysecretpassword@${POSTGRES_HOST}:${POSTGRES_PORT}/postgres" | |
| echo "Running with postgres" | |
| fi | |
| if [ "$QUETZ_TEST_DBINIT" == "use-migrations" ]; then | |
| echo "Using migrations" | |
| fi | |
| export QUETZ_IS_TEST=1 | |
| pytest -v ./quetz/tests/ --cov-config=pyproject.toml --cov=. --cov-report=xml --capture=no | |
| - name: Test the plugins | |
| shell: bash -l -eo pipefail {0} | |
| env: | |
| TEST_DB_BACKEND: ${{ matrix.test_database }} | |
| QUETZ_TEST_DBINIT: ${{ matrix.db_init }} | |
| POSTGRES_HOST: localhost | |
| POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} | |
| S3_ACCESS_KEY: ${{ secrets.s3_access_key }} | |
| S3_SECRET_KEY: ${{ secrets.s3_secret_key }} | |
| S3_ENDPOINT: https://s3.sbg.cloud.ovh.net/ | |
| S3_REGION: sbg | |
| S3_BUCKET_NAME: quetz | |
| run: | | |
| if [ "$TEST_DB_BACKEND" == "postgres" ]; then | |
| export QUETZ_TEST_DATABASE="postgresql://postgres:mysecretpassword@${POSTGRES_HOST}:${POSTGRES_PORT}/postgres" | |
| echo "Running with postgres" | |
| fi | |
| # add micromamba to path | |
| export PATH=$(dirname $MAMBA_EXE):$PATH | |
| echo "adding micromamba to path: $MAMBA_EXE" | |
| pip install --no-deps git+https://[email protected]/regro/libcflib@master | |
| for f in ./plugins/quetz_* | |
| do | |
| echo "::group::Testing plugin ${f}" | |
| quetz plugin install $f | |
| # We want to test all the plugins, regardless of whether one fails | |
| set +e | |
| pytest -v $f | |
| pytest_exit_code=$? | |
| set -e | |
| echo "::endgroup::" | |
| if [ $pytest_exit_code -ne 0 ]; then | |
| echo "::error::Tests for plugin $f failed!" | |
| tests_failed="true" | |
| fi | |
| done | |
| if [ "$tests_failed" == "true" ]; then | |
| echo "::error::Some plugin tests failed!" | |
| exit 1 | |
| fi | |
| - uses: codecov/codecov-action@v1 | |
| with: | |
| verbose: true | |
| file: ./coverage.xml | |
| test_release: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: "3.10" | |
| - name: install missing dependency | |
| run: pip install ipython_genutils | |
| - name: Get branch name | |
| id: branch-name | |
| uses: tj-actions/branch-names@eee8675bd61ec38bcfbfedd504d8473292ba649e | |
| - name: Check Release | |
| uses: jupyter-server/jupyter_releaser/.github/actions/check-release@v2 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} |