Regression.yml

name: Regression
on:
  workflow_dispatch:
  repository_dispatch:
  push:
    branches:
      - '**'
      - '!main'
      - '!feature'
    paths-ignore:
      - '**.md'
      - 'tools/**'
      - '!tools/pythonpkg/**'
      - '.github/patches/duckdb-wasm/**'
      - '.github/workflows/**'
      - '!.github/workflows/Regression.yml'

  pull_request:
    types: [opened, reopened, ready_for_review]
    paths-ignore:
      - '**.md'
      - 'tools/**'
      - '!tools/pythonpkg/**'
      - '.github/patches/duckdb-wasm/**'
      - '.github/workflows/**'
      - '!.github/workflows/Regression.yml'


concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
  cancel-in-progress: true

env:
  GH_TOKEN: ${{ secrets.GH_TOKEN }}
  BASE_BRANCH: ${{ github.base_ref || (endsWith(github.ref, '_feature') && 'feature' || 'main') }}

jobs:
  regression-test-benchmark-runner:
    name: Regression Tests
    runs-on: ubuntu-20.04
    env:
      CC: gcc-10
      CXX: g++-10
      GEN: ninja
      BUILD_BENCHMARK: 1
      BUILD_JEMALLOC: 1
      CORE_EXTENSIONS: "json;tpch;tpcds;httpfs;inet"

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install
        shell: bash
        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build && pip install requests

      - name: Setup Ccache
        uses: hendrikmuhs/ccache-action@main
        with:
          key: ${{ github.job }}
          save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}

      - name: Build
        shell: bash
        run: |
          make
          git clone --branch ${{ env.BASE_BRANCH }} https://github.com/duckdb/duckdb.git --depth=1
          cd duckdb
          make
          cd ..

      - name: Set up benchmarks
        shell: bash
        run: |
          cp -r benchmark duckdb/

      - name: Regression Test Micro
        if: always()
        shell: bash
        run: |
          python scripts/regression/test_runner.py --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/micro.csv --verbose --threads 2

      - name: Regression Test Ingestion Perf
        if: always()
        shell: bash
        run: |
          python scripts/regression/test_runner.py --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/ingestion.csv --verbose --threads 2

      - name: Regression Test TPCH
        if: always()
        shell: bash
        run: |
          python scripts/regression/test_runner.py --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/tpch.csv --verbose --threads 2

      - name: Regression Test TPCH-PARQUET
        if: always()
        shell: bash
        run: |
          python scripts/regression/test_runner.py --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/tpch_parquet.csv --verbose --threads 2

      - name: Regression Test TPCDS
        if: always()
        shell: bash
        run: |
          python scripts/regression/test_runner.py --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/tpcds.csv --verbose --threads 2

      - name: Regression Test H2OAI
        if: always()
        shell: bash
        run: |
          python scripts/regression/test_runner.py --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/h2oai.csv --verbose --threads 2

      - name: Regression Test IMDB
        if: always()
        shell: bash
        run: |
          python scripts/regression/test_runner.py --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/imdb.csv --verbose --threads 2

      - name: Regression Test CSV
        if: always()
        shell: bash
        run: |
          python scripts/regression/test_runner.py --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/csv.csv --verbose --threads 2

      - name: Regression Test RealNest 
        if: always()
        shell: bash
        run: |
          mkdir -p duckdb_benchmark_data
          mkdir -p duckdb/duckdb_benchmark_data
          wget https://duckdb-blobs.s3.amazonaws.com/data/realnest/realnest.duckdb --output-document=duckdb_benchmark_data/real_nest.duckdb
          cp duckdb_benchmark_data/real_nest.duckdb duckdb/duckdb_benchmark_data/real_nest.duckdb
          python scripts/regression/test_runner.py --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/realnest.csv --verbose --threads 2
         
  regression-test-storage:
    name: Storage Size Regression Test
    runs-on: ubuntu-20.04
    env:
      CC: gcc-10
      CXX: g++-10
      GEN: ninja
      CORE_EXTENSIONS: "tpch;tpcds"

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install
        shell: bash
        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build && pip install requests

      - name: Setup Ccache
        uses: hendrikmuhs/ccache-action@main
        with:
          key: ${{ github.job }}
          save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}

      - name: Build
        shell: bash
        run: |
          make
          git clone --branch ${{ env.BASE_BRANCH }} https://github.com/duckdb/duckdb.git --depth=1
          cd duckdb
          make
          cd ..

      - name: Regression Test
        shell: bash
        run: |
          python scripts/regression_test_storage_size.py --old duckdb/build/release/duckdb --new build/release/duckdb

      - name: Test for incompatibility
        shell: bash
        run: |
          if (cmp test/sql/storage_version/storage_version.db duckdb/test/sql/storage_version/storage_version.db); then
            echo "storage_changed=false" >> $GITHUB_ENV
          else
            echo "storage_changed=true" >> $GITHUB_ENV
          fi

      - name: Regression Compatibility Test (testing bidirectional compatibility)
        shell: bash
        if: env.storage_changed == 'false'
        run: |
          # Regenerate test/sql/storage_version.db with newer version -> read with older version
          python3 scripts/generate_storage_version.py
          ./duckdb/build/release/duckdb test/sql/storage_version/storage_version.db
          # Regenerate test/sql/storage_version.db with older version -> read with newer version (already performed as part of test.slow)
          cd duckdb
          python3 ../scripts/generate_storage_version.py
          ../build/release/duckdb duckdb/test/sql/storage_version/storage_version.db
          cd ..

      - name: Regression Compatibility Test (testing storage version has been bumped)
        shell: bash
        if: env.storage_changed == 'true'
        run: |
          python3 scripts/generate_storage_version.py
          cd duckdb
          python3 scripts/generate_storage_version.py
          cd ..
          if (cmp -i 8 -n 12 test/sql/storage_version.db duckdb/test/sql/storage_version.db); then
            echo "Expected storage format to be bumped, but this is not the case"
            echo "This might fail spuriously if changes to content of test database / generation script happened"
            exit 1
          else
            echo "Storage bump detected, all good!"
          fi

  regression-test-binary-size:
    name: Regression test binary size
    runs-on: ubuntu-20.04
    env:
      CC: gcc-10
      CXX: g++-10
      GEN: ninja
      CORE_EXTENSIONS: "tpch;tpcds;json;parquet"
      EXTENSION_STATIC_BUILD: 1
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install
        shell: bash
        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build && pip install requests

      - name: Setup Ccache
        uses: hendrikmuhs/ccache-action@main
        with:
          key: ${{ github.job }}
          save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}

      - name: Build
        shell: bash
        run: |
          make
          git clone --branch ${{ env.BASE_BRANCH }} https://github.com/duckdb/duckdb.git --depth=1
          cd duckdb
          make
          cd ..

      - name: Regression Test Extension binary size
        shell: bash
        run: |
          python scripts/regression_test_extension_size.py --old 'duckdb/build/release/extension' --new build/release/extension --expect json,parquet,tpch,tpcds

  regression-test-plan-cost:
    name: Regression Test Join Order Plan Cost
    runs-on: ubuntu-20.04
    env:
      CC: gcc-10
      CXX: g++-10
      GEN: ninja
      CORE_EXTENSIONS: "tpch;httpfs"

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install
        shell: bash
        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build && pip install tqdm

      - name: Setup Ccache
        uses: hendrikmuhs/ccache-action@main
        with:
          key: ${{ github.job }}
          save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}

      - name: Build
        shell: bash
        run: |
          make
          git clone --branch ${{ env.BASE_BRANCH }} https://github.com/duckdb/duckdb.git --depth=1
          cd duckdb
          make
          cd ..

      - name: Set up benchmarks
        shell: bash
        run: |
          cp -r benchmark duckdb/

      - name: Regression Test IMDB
        if: always()
        shell: bash
        run: |
          python scripts/plan_cost_runner.py --old duckdb/build/release/duckdb --new build/release/duckdb --dir=benchmark/imdb_plan_cost

      - name: Regression Test TPCH
        if: always()
        shell: bash
        run: |
          python scripts/plan_cost_runner.py --old duckdb/build/release/duckdb --new build/release/duckdb --dir=benchmark/tpch_plan_cost