Fixed meltano setup (#1249)

.github/workflows/meltano/action.yml

@@ -0,0 +1,38 @@
+# Generic worker executor. This is callable for use with cron scheduling.
+
+# Execute the worker for a specific group
+name: warehouse-meltano-sync
+
+inputs:
+  tap:
+    description: The name of the tap
+    required: true
+  target:
+    description: The name of the target
+    required: true
+  meltano_path:
+    description: The path to meltano
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.10.13' 
+
+    - uses: snok/install-poetry@v1
+      with:
+        version: '1.7.1'
+    - name: Install poetry deps for meltano
+      shell: bash
+      run: |
+        cd ${{ inputs.meltano_path }} && poetry install && meltano install
+
+    - name: Run tap:${{ inputs.tap }} into target:${{ inputs.target }}
+      shell: bash
+      # This is for testing for now
+      run: |
+        env
+      
+    

.github/workflows/warehouse-meltano-ecosystems-ost.yml

@@ -0,0 +1,46 @@
+# This is a test meltano pipeline
+name: meltano-ecosystem-ost
+env:
+  X_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_PASSWORD: ${{ secrets.TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_PASSWORD }}
+  TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_HOST: ${{ secrets.TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_HOST }}
+  TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_DATABASE: ${{ secrets.TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_DATABASE }}
+  TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_PORT: ${{ secrets.TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_PORT }}
+  TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_USERNAME: ${{ secrets.TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_USERNAME }}
+  MELTANO_STATE_BACKEND_URI: ${{ secrets.MELTANO_STATE_BACKEND_URI }}
+
+on:
+  workflow_dispatch:
+
+jobs:
+  meltano-ecosystem-ost:
+    name: meltano-ecosystem-ost
+    environment: indexer
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+
+      - uses: 'google-github-actions/auth@v2'
+        with:
+          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_JSON }}'
+          create_credentials_file: true
+
+      - name: Run meltano
+        uses: ./.github/workflows/meltano
+        with:
+          tap: tap-ecosystems-ost
+          target: target-bigquery
+        env:
+          TARGET_BIGQUERY_CREDENTIALS_PATH: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}
+          TARGET_BIGQUERY_BUCKET: oso-dataset-transfer-bucket
+          TARGET_BIGQUERY_PROJECT: opensource-observer
+          TARGET_BIGQUERY_DATASET: ecosystems-ost

warehouse/meltano-setup/meltano.yml

@@ -5,30 +5,71 @@ environments:
 - name: dev
 - name: staging
 - name: prod
+# state_backend:
+#   uri: gs://oso-playground-dataset-transfer-bucket/meltano/state.db
 plugins:
   extractors:
-  - name: tap-postgres
+  - name: tap-ecosystems-ost
+    inherit_from: tap-postgres
     variant: airbyte
-    pip_url: git+https://github.com/MeltanoLabs/tap-airbyte-wrapper.git
+    pip_url: git+https://github.com/opensource-observer/tap-airbyte-wrapper.git
     config:
+      nullable_generated_fields:
+        - "*._ab_cdc_deleted_at"
+      airbyte_spec:
+        image: airbyte/source-postgres
+        tag: 3.3.26
       airbyte_config:
-        jdbc_url_params: "replication=postgres"
+        jdbc_url_params: replication=postgres
         ssl_mode:
           mode: disable
         schemas:
         - public
         replication_method:
           plugin: pgoutput
           method: CDC
-          publication: publication_name
+          publication: oso_publication
           replication_slot: oso_slot
-          initial_waiting_seconds: 5
+          initial_waiting_seconds: 10
     select:
-    - test_table.id
-    - test_table.data
-    - test_table.created_at
+    - projects.*
+    - issues.*
+    - collections.*
+    - votes.*
+
+  - name: tap-local-test
+    inherit_from: tap-postgres
+    variant: airbyte
+    pip_url: git+https://github.com/opensource-observer/tap-airbyte-wrapper.git
+    config:
+      nullable_generated_fields:
+        - "*._ab_cdc_deleted_at"
+      airbyte_spec:
+        image: airbyte/source-postgres
+        tag: 3.3.26
+      airbyte_config:
+        jdbc_url_params: replication=postgres
+        ssl_mode:
+          mode: disable
+        schemas:
+        - public
+        replication_method:
+          plugin: pgoutput
+          method: CDC
+          publication: oso_publication
+          replication_slot: oso_replication_slot
+          initial_waiting_seconds: 10
+    select:
+    - foo.*
+    - bar.*
 
   loaders:
-  - name: target-jsonl
-    variant: andyh1203
-    pip_url: target-jsonl
+  - name: target-bigquery
+    variant: z3z1ma
+    pip_url: git+https://github.com/z3z1ma/target-bigquery.git
+    config:
+      denormalized: true
+      method: gcs_stage
+
+elt:
+  buffer_size: 1073741824