update to push to bigquery

dlt-hub · Dec 20, 2023 · 26358a7 · 26358a7
1 parent 003f2b2
commit 26358a7
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 1 deletion.
diff --git a/.github/workflows/run_slack_workflow.yml b/.github/workflows/run_slack_workflow.yml
@@ -9,6 +9,9 @@ name: Run slack pipeline from slack_pipeline.py
 env:
   RUNTIME__DLTHUB_DSN: ${{ secrets.RUNTIME__DLTHUB_DSN }}
   SOURCES__SLACK__ACCESS_TOKEN: ${{ secrets.SOURCES__SLACK__ACCESS_TOKEN }}
+  DESTINATION__BIGQUERY__CREDENTIALS__CLIENT_EMAIL: ${{ DESTINATION__BIGQUERY__CREDENTIALS__CLIENT_EMAIL }}
+  DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY: ${{ DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY }}
+  DESTINATION__BIGQUERY__CREDENTIALS__PROJECT_ID: ${{ DESTINATION__BIGQUERY__CREDENTIALS__PROJECT_ID }}
 jobs:
   maybe_skip:
     runs-on: ubuntu-latest

diff --git a/slack_pipeline.py b/slack_pipeline.py
@@ -11,7 +11,7 @@ def load_all_resources(start_date) -> None:
     """Load all resources from slack without any selection of channels."""
 
     pipeline = dlt.pipeline(
-        pipeline_name="slack", destination='duckdb', dataset_name="slack_data"
+        pipeline_name="slack", destination='bigquery', dataset_name="akela_test_slack_backup"
     )
 
     source = slack_source(