separates ci runs on forks

.github/workflows/test_destinations_slow.yml

@@ -16,6 +16,8 @@ env:
 jobs:
   get_changed_sources:
     uses: ./.github/workflows/get_changed_sources.yml
+    # Tests that require credentials do not run in forks
+    if: ${{ !github.event.pull_request.head.repo.fork }}
 
   run_loader:
     name: test destinations bigquery and redshift

.github/workflows/test_destinations.yml → .github/workflows/test_on_local_destinations.yml

@@ -18,6 +18,8 @@ env:
 jobs:
   get_changed_sources:
     uses: ./.github/workflows/get_changed_sources.yml
+    # Tests that require credentials do not run in forks
+    if: ${{ !github.event.pull_request.head.repo.fork }}
 
   run_loader:
     name: test destinations redshift, postgres and duckdb

.github/workflows/test_on_local_destinations_forks.yml

@@ -0,0 +1,73 @@
+
+name: test postgres and duckdb
+
+on:
+  pull_request:
+    branches:
+      - master
+      - devel
+  workflow_dispatch:
+
+env:
+  ALL_DESTINATIONS: '["postgres", "duckdb"]'
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
+  RUNTIME__LOG_LEVEL: ERROR
+
+jobs:
+  get_changed_sources:
+    uses: ./.github/workflows/get_changed_sources.yml
+    # run only on fork
+    # if: ${{ github.event.pull_request.head.repo.fork }}
+
+  run_loader:
+    name: test on local postgres and duckdb
+    needs: get_changed_sources
+    if: needs.get_changed_sources.outputs.sources_list != ''
+    strategy:
+      max-parallel: 1
+      fail-fast: false
+    defaults:
+      run:
+        shell: bash
+    runs-on: "ubuntu-latest"
+
+    steps:
+      - name: Check out
+        uses: actions/checkout@master
+
+      - name: Setup Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: "3.9.x"
+
+      - name: Install Poetry
+        uses: snok/[email protected]
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v2
+        with:
+          # path: ${{ steps.pip-cache.outputs.dir }}
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-destinations-forks
+
+      - name: Install dependencies
+        # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --all-extras
+
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > sources/.dlt/secrets.toml
+      #   run: pwd && echo "$DESTINATIONS_SECRETS" > sources/.dlt/secrets.toml && echo "$SOURCES_SECRETS" >> sources/.dlt/secrets.toml
+      # - name: Setup upterm session
+      #   uses: lhotari/action-upterm@v1
+      - run: |
+          sources_list="${{ needs.get_changed_sources.outputs.sources_list }}"
+          test_paths=$(echo "$sources_list" | awk '{for(i=1;i<=NF;i++) printf "tests/%s ", $i}')
+          poetry run pytest $test_paths
+        name: Run tests on Linux/macOS
+        env:
+          DESTINATION__POSTGRES__CREDENTIALS: postgresql://loader:loader@localhost:5432/dlt_data

pyproject.toml

@@ -12,7 +12,7 @@ packages = [{include = "sources"}]
 
 [tool.poetry.dependencies]
 python = "^3.8.1"
-dlt = {version = "^0.3.8", allow-prereleases = true, extras = ["redshift", "bigquery", "postgres", "duckdb"]}
+dlt = {version = "^0.3.11", allow-prereleases = true, extras = ["redshift", "bigquery", "postgres", "duckdb"]}
 
 [tool.poetry.group.dev.dependencies]
 mypy = "^0.991"