diff --git a/.github/workflows/run_slack_workflow.yml b/.github/workflows/run_slack_workflow.yml index 189df25..ddae773 100644 --- a/.github/workflows/run_slack_workflow.yml +++ b/.github/workflows/run_slack_workflow.yml @@ -9,6 +9,9 @@ name: Run slack pipeline from slack_pipeline.py env: RUNTIME__DLTHUB_DSN: ${{ secrets.RUNTIME__DLTHUB_DSN }} SOURCES__SLACK__ACCESS_TOKEN: ${{ secrets.SOURCES__SLACK__ACCESS_TOKEN }} + DESTINATION__BIGQUERY__CREDENTIALS__CLIENT_EMAIL: ${{ DESTINATION__BIGQUERY__CREDENTIALS__CLIENT_EMAIL }} + DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY: ${{ DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY }} + DESTINATION__BIGQUERY__CREDENTIALS__PROJECT_ID: ${{ DESTINATION__BIGQUERY__CREDENTIALS__PROJECT_ID }} jobs: maybe_skip: runs-on: ubuntu-latest diff --git a/slack_pipeline.py b/slack_pipeline.py index a769cdc..52383e9 100644 --- a/slack_pipeline.py +++ b/slack_pipeline.py @@ -11,7 +11,7 @@ def load_all_resources(start_date) -> None: """Load all resources from slack without any selection of channels.""" pipeline = dlt.pipeline( - pipeline_name="slack", destination='duckdb', dataset_name="slack_data" + pipeline_name="slack", destination='bigquery', dataset_name="akela_test_slack_backup" ) source = slack_source(