From 3e534485b7e16c9409ba4414a5d3a646089dac28 Mon Sep 17 00:00:00 2001
From: Marcin Rudolf <rudolfix@rudolfix.org>
Date: Mon, 14 Aug 2023 15:14:29 +0200
Subject: [PATCH] separates ci runs on forks

---
 .github/workflows/test_destinations_slow.yml  |  2 +
 ...ons.yml => test_on_local_destinations.yml} |  2 +
 .../test_on_local_destinations_forks.yml      | 73 +++++++++++++++++++
 poetry.lock                                   | 14 ++--
 pyproject.toml                                |  2 +-
 5 files changed, 85 insertions(+), 8 deletions(-)
 rename .github/workflows/{test_destinations.yml => test_on_local_destinations.yml} (95%)
 create mode 100644 .github/workflows/test_on_local_destinations_forks.yml

diff --git a/.github/workflows/test_destinations_slow.yml b/.github/workflows/test_destinations_slow.yml
index 27ff4e0cb..614cff664 100644
--- a/.github/workflows/test_destinations_slow.yml
+++ b/.github/workflows/test_destinations_slow.yml
@@ -16,6 +16,8 @@ env:
 jobs:
   get_changed_sources:
     uses: ./.github/workflows/get_changed_sources.yml
+    # Tests that require credentials do not run in forks
+    if: ${{ !github.event.pull_request.head.repo.fork }}
 
   run_loader:
     name: test destinations bigquery and redshift
diff --git a/.github/workflows/test_destinations.yml b/.github/workflows/test_on_local_destinations.yml
similarity index 95%
rename from .github/workflows/test_destinations.yml
rename to .github/workflows/test_on_local_destinations.yml
index 1f3bd4efc..cdce3edd6 100644
--- a/.github/workflows/test_destinations.yml
+++ b/.github/workflows/test_on_local_destinations.yml
@@ -18,6 +18,8 @@ env:
 jobs:
   get_changed_sources:
     uses: ./.github/workflows/get_changed_sources.yml
+    # Tests that require credentials do not run in forks
+    if: ${{ !github.event.pull_request.head.repo.fork }}
 
   run_loader:
     name: test destinations redshift, postgres and duckdb
diff --git a/.github/workflows/test_on_local_destinations_forks.yml b/.github/workflows/test_on_local_destinations_forks.yml
new file mode 100644
index 000000000..4a205988f
--- /dev/null
+++ b/.github/workflows/test_on_local_destinations_forks.yml
@@ -0,0 +1,73 @@
+
+name: test postgres and duckdb
+
+on:
+  pull_request:
+    branches:
+      - master
+      - devel
+  workflow_dispatch:
+
+env:
+  ALL_DESTINATIONS: '["postgres", "duckdb"]'
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
+  RUNTIME__LOG_LEVEL: ERROR
+
+jobs:
+  get_changed_sources:
+    uses: ./.github/workflows/get_changed_sources.yml
+    # run only on fork
+    # if: ${{ github.event.pull_request.head.repo.fork }}
+
+  run_loader:
+    name: test on local postgres and duckdb
+    needs: get_changed_sources
+    if: needs.get_changed_sources.outputs.sources_list != ''
+    strategy:
+      max-parallel: 1
+      fail-fast: false
+    defaults:
+      run:
+        shell: bash
+    runs-on: "ubuntu-latest"
+
+    steps:
+      - name: Check out
+        uses: actions/checkout@master
+
+      - name: Setup Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: "3.9.x"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1.3.2
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v2
+        with:
+          # path: ${{ steps.pip-cache.outputs.dir }}
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-destinations-forks
+
+      - name: Install dependencies
+        # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --all-extras
+
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > sources/.dlt/secrets.toml
+      #   run: pwd && echo "$DESTINATIONS_SECRETS" > sources/.dlt/secrets.toml && echo "$SOURCES_SECRETS" >> sources/.dlt/secrets.toml
+      # - name: Setup upterm session
+      #   uses: lhotari/action-upterm@v1
+      - run: |
+          sources_list="${{ needs.get_changed_sources.outputs.sources_list }}"
+          test_paths=$(echo "$sources_list" | awk '{for(i=1;i<=NF;i++) printf "tests/%s ", $i}')
+          poetry run pytest $test_paths
+        name: Run tests on Linux/macOS
+        env:
+          DESTINATION__POSTGRES__CREDENTIALS: postgresql://loader:loader@localhost:5432/dlt_data
diff --git a/poetry.lock b/poetry.lock
index a0e91896e..afd3962f8 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -377,7 +377,7 @@ dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"]
 
 [[package]]
 name = "dlt"
-version = "0.3.10"
+version = "0.3.11"
 description = "DLT is an open-source python-native scalable data loading framework that does not require any devops efforts to run."
 category = "main"
 optional = false
@@ -469,7 +469,7 @@ python-versions = ">=3.6"
 
 [[package]]
 name = "exceptiongroup"
-version = "1.1.2"
+version = "1.1.3"
 description = "Backport of PEP 654 (exception groups)"
 category = "dev"
 optional = false
@@ -2636,7 +2636,7 @@ cffi = ["cffi (>=1.11)"]
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.8.1"
-content-hash = "6eb2ea40bca7e151375c97d2f000013da3cec11735526b28a2afaeb00e158430"
+content-hash = "e6552af866c43ff9e01d295881a75f772cffd2d556154daeb0ba3722fc40e4bf"
 
 [metadata.files]
 aiohttp = [
@@ -3075,8 +3075,8 @@ deprecated = [
     {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"},
 ]
 dlt = [
-    {file = "dlt-0.3.10-py3-none-any.whl", hash = "sha256:c1bacd5edc768ac2b92614f1b597afaa286d44fe59fdb3a47b8d71074489b09d"},
-    {file = "dlt-0.3.10.tar.gz", hash = "sha256:61e99d69a0f622d33c8f7fa86bf25cd287fb2c1d6df780bd9e0a1f03bb00fecd"},
+    {file = "dlt-0.3.11-py3-none-any.whl", hash = "sha256:fa3c9e2060faa12defbc6b4ed7b8641d8013f08d79787121584e2a9927cf1e0a"},
+    {file = "dlt-0.3.11.tar.gz", hash = "sha256:c1f5e922d4936d0cffb38f367a9e7ee1d3adb46a236df07aad5e1920c5e770d6"},
 ]
 domdf-python-tools = [
     {file = "domdf_python_tools-3.6.1-py3-none-any.whl", hash = "sha256:e18158460850957f18e740eb94ede56f580ddb0cb162ab9d9834ed8bbb1b6431"},
@@ -3141,8 +3141,8 @@ et-xmlfile = [
     {file = "et_xmlfile-1.1.0.tar.gz", hash = "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c"},
 ]
 exceptiongroup = [
-    {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"},
-    {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"},
+    {file = "exceptiongroup-1.1.3-py3-none-any.whl", hash = "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3"},
+    {file = "exceptiongroup-1.1.3.tar.gz", hash = "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9"},
 ]
 facebook-business = [
     {file = "facebook_business-17.0.3-py3-none-any.whl", hash = "sha256:ec13b66ea13a0bb830f8c9b47f443878904cc1dd24699d8d0f786e9586207aaa"},
diff --git a/pyproject.toml b/pyproject.toml
index 41834ff8c..6a9869c51 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -12,7 +12,7 @@ packages = [{include = "sources"}]
 
 [tool.poetry.dependencies]
 python = "^3.8.1"
-dlt = {version = "^0.3.8", allow-prereleases = true, extras = ["redshift", "bigquery", "postgres", "duckdb"]}
+dlt = {version = "^0.3.11", allow-prereleases = true, extras = ["redshift", "bigquery", "postgres", "duckdb"]}
 
 [tool.poetry.group.dev.dependencies]
 mypy = "^0.991"