test_destinations_slow.yml

name: test destinations bigquery and redshift

on:
  pull_request:
    branches:
      - master
      - devel
  workflow_dispatch:

env:
  ALL_DESTINATIONS: '["bigquery", "redshift"]'
  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
  RUNTIME__LOG_LEVEL: ERROR

jobs:
  get_changed_sources:
    uses: ./.github/workflows/get_changed_sources.yml
    # Tests that require credentials do not run in forks
    if: ${{ !github.event.pull_request.head.repo.fork }}

  run_loader:
    name: test destinations bigquery and redshift
    needs: get_changed_sources
    if: needs.get_changed_sources.outputs.sources_list != ''

    # strategy:
    #   fail-fast: false
    #   matrix:
    #     os: ["ubuntu-latest", "macos-latest", "windows-latest"]
    defaults:
      run:
        shell: bash
    runs-on: "ubuntu-latest"

    # Service containers to run with `container-job`
    services:
      # Label used to access the service container
      postgres:
        # Docker Hub image
        image: postgres
        # Provide the password for postgres
        env:
          POSTGRES_DB: dlt_data
          POSTGRES_USER: loader
          POSTGRES_PASSWORD: loader
        ports:
          - 5432:5432
        # Set health checks to wait until postgres has started
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

    steps:

      - name: Check out
        uses: actions/checkout@master

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8.x"

      - name: Install Poetry
        uses: snok/install-poetry@v1.3.2
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v2
        with:
          # path: ${{ steps.pip-cache.outputs.dir }}
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-destinations

      - name: Install dependencies
        # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --all-extras

      - name: create secrets.toml
        run: pwd && echo "$DLT_SECRETS_TOML" > sources/.dlt/secrets.toml
      # - name: Setup upterm session
      #   uses: lhotari/action-upterm@v1
      - run: |
          sources_list="${{ needs.get_changed_sources.outputs.sources_list }}"
          test_paths=$(echo "$sources_list" | awk '{for(i=1;i<=NF;i++) printf "tests/%s ", $i}')
          poetry run pytest $test_paths
        if: runner.os != 'Windows'
        name: Run tests on Linux/macOS
        env:
          DESTINATION__POSTGRES__CREDENTIALS: postgresql://loader:loader@localhost:5432/dlt_data