test_destination_dremio.yml

name: test | dremio

on:
  pull_request:
    branches:
      - master
      - devel
  workflow_dispatch:
  schedule:
    - cron:  '0 2 * * *'

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  # RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752
  RUNTIME__LOG_LEVEL: ERROR
  RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}

  ACTIVE_DESTINATIONS: "[\"dremio\"]"
  ALL_FILESYSTEM_DRIVERS: "[\"memory\"]"

jobs:
  get_docs_changes:
    name: docs changes
    uses: ./.github/workflows/get_docs_changes.yml
    if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}}

  run_loader:
    name: test | dremio tests
    needs: get_docs_changes
    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
    defaults:
      run:
        shell: bash
    runs-on: "ubuntu-latest"

    steps:

      - name: Check out
        uses: actions/checkout@master

      - name: Start dremio
        run: docker compose -f "tests/load/dremio/docker-compose.yml" up -d

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10.x"

      - name: Install Poetry
        uses: snok/install-poetry@v1.3.2
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v3
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp

      - name: Install dependencies
        run: poetry install --no-interaction -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline

      - run: |
          poetry run pytest tests/load --ignore tests/load/sources
        if: runner.os != 'Windows'
        name: Run tests Linux/MAC
        env:
          DESTINATION__DREMIO__CREDENTIALS: grpc://dremio:dremio123@localhost:32010/nas
          DESTINATION__DREMIO__STAGING_DATA_SOURCE: minio
          DESTINATION__MINIO__BUCKET_URL: s3://dlt-ci-test-bucket
          DESTINATION__MINIO__CREDENTIALS__AWS_ACCESS_KEY_ID: minioadmin
          DESTINATION__MINIO__CREDENTIALS__AWS_SECRET_ACCESS_KEY: minioadmin
          DESTINATION__MINIO__CREDENTIALS__ENDPOINT_URL: http://127.0.0.1:9010

      - run: |
          poetry run pytest tests/load --ignore tests/load/sources
        if: runner.os == 'Windows'
        name: Run tests Windows
        shell: cmd

      - name: Stop dremio
        if: always()
        run: docker compose -f "tests/load/dremio/docker-compose.yml" down -v