differentiate between top-level and nested validity #1600
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Regression | |
| on: | |
| workflow_dispatch: | |
| repository_dispatch: | |
| push: | |
| branches: | |
| - '**' | |
| - '!main' | |
| - '!feature' | |
| paths-ignore: | |
| - '**.md' | |
| - 'tools/**' | |
| - '!tools/pythonpkg/**' | |
| - '.github/patches/duckdb-wasm/**' | |
| - '.github/workflows/**' | |
| - '!.github/workflows/Regression.yml' | |
| pull_request: | |
| types: [opened, reopened, ready_for_review] | |
| paths-ignore: | |
| - '**.md' | |
| - 'tools/**' | |
| - '!tools/pythonpkg/**' | |
| - '.github/patches/duckdb-wasm/**' | |
| - '.github/workflows/**' | |
| - '!.github/workflows/Regression.yml' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} | |
| cancel-in-progress: true | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_TOKEN }} | |
| BASE_BRANCH: ${{ github.base_ref || (endsWith(github.ref, '_feature') && 'feature' || 'main') }} | |
| jobs: | |
| regression-test-benchmark-runner: | |
| name: Regression Tests | |
| runs-on: ubuntu-20.04 | |
| env: | |
| CC: gcc-10 | |
| CXX: g++-10 | |
| GEN: ninja | |
| BUILD_BENCHMARK: 1 | |
| BUILD_JEMALLOC: 1 | |
| CORE_EXTENSIONS: "json;tpch;tpcds;httpfs;inet" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install | |
| shell: bash | |
| run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build && pip install requests | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: | | |
| make | |
| git clone --branch ${{ env.BASE_BRANCH }} https://github.com/duckdb/duckdb.git --depth=1 | |
| cd duckdb | |
| make | |
| cd .. | |
| - name: Set up benchmarks | |
| shell: bash | |
| run: | | |
| cp -r benchmark duckdb/ | |
| - name: Regression Test Micro | |
| if: always() | |
| shell: bash | |
| run: | | |
| python scripts/regression/test_runner.py --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/micro.csv --verbose --threads 2 | |
| - name: Regression Test Ingestion Perf | |
| if: always() | |
| shell: bash | |
| run: | | |
| python scripts/regression/test_runner.py --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/ingestion.csv --verbose --threads 2 | |
| - name: Regression Test TPCH | |
| if: always() | |
| shell: bash | |
| run: | | |
| python scripts/regression/test_runner.py --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/tpch.csv --verbose --threads 2 | |
| - name: Regression Test TPCH-PARQUET | |
| if: always() | |
| shell: bash | |
| run: | | |
| python scripts/regression/test_runner.py --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/tpch_parquet.csv --verbose --threads 2 | |
| - name: Regression Test TPCDS | |
| if: always() | |
| shell: bash | |
| run: | | |
| python scripts/regression/test_runner.py --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/tpcds.csv --verbose --threads 2 | |
| - name: Regression Test H2OAI | |
| if: always() | |
| shell: bash | |
| run: | | |
| python scripts/regression/test_runner.py --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/h2oai.csv --verbose --threads 2 | |
| - name: Regression Test IMDB | |
| if: always() | |
| shell: bash | |
| run: | | |
| python scripts/regression/test_runner.py --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/imdb.csv --verbose --threads 2 | |
| - name: Regression Test CSV | |
| if: always() | |
| shell: bash | |
| run: | | |
| python scripts/regression/test_runner.py --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/csv.csv --verbose --threads 2 | |
| - name: Regression Test RealNest | |
| if: always() | |
| shell: bash | |
| run: | | |
| mkdir -p duckdb_benchmark_data | |
| mkdir -p duckdb/duckdb_benchmark_data | |
| wget https://duckdb-blobs.s3.amazonaws.com/data/realnest/realnest.duckdb --output-document=duckdb_benchmark_data/real_nest.duckdb | |
| cp duckdb_benchmark_data/real_nest.duckdb duckdb/duckdb_benchmark_data/real_nest.duckdb | |
| python scripts/regression/test_runner.py --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/realnest.csv --verbose --threads 2 | |
| regression-test-storage: | |
| name: Storage Size Regression Test | |
| runs-on: ubuntu-20.04 | |
| env: | |
| CC: gcc-10 | |
| CXX: g++-10 | |
| GEN: ninja | |
| CORE_EXTENSIONS: "tpch;tpcds" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install | |
| shell: bash | |
| run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build && pip install requests | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: | | |
| make | |
| git clone --branch ${{ env.BASE_BRANCH }} https://github.com/duckdb/duckdb.git --depth=1 | |
| cd duckdb | |
| make | |
| cd .. | |
| - name: Regression Test | |
| shell: bash | |
| run: | | |
| python scripts/regression_test_storage_size.py --old duckdb/build/release/duckdb --new build/release/duckdb | |
| - name: Test for incompatibility | |
| shell: bash | |
| run: | | |
| if (cmp test/sql/storage_version/storage_version.db duckdb/test/sql/storage_version/storage_version.db); then | |
| echo "storage_changed=false" >> $GITHUB_ENV | |
| else | |
| echo "storage_changed=true" >> $GITHUB_ENV | |
| fi | |
| - name: Regression Compatibility Test (testing bidirectional compatibility) | |
| shell: bash | |
| if: env.storage_changed == 'false' | |
| run: | | |
| # Regenerate test/sql/storage_version.db with newer version -> read with older version | |
| python3 scripts/generate_storage_version.py | |
| ./duckdb/build/release/duckdb test/sql/storage_version/storage_version.db | |
| # Regenerate test/sql/storage_version.db with older version -> read with newer version (already performed as part of test.slow) | |
| cd duckdb | |
| python3 ../scripts/generate_storage_version.py | |
| ../build/release/duckdb duckdb/test/sql/storage_version/storage_version.db | |
| cd .. | |
| - name: Regression Compatibility Test (testing storage version has been bumped) | |
| shell: bash | |
| if: env.storage_changed == 'true' | |
| run: | | |
| python3 scripts/generate_storage_version.py | |
| cd duckdb | |
| python3 scripts/generate_storage_version.py | |
| cd .. | |
| if (cmp -i 8 -n 12 test/sql/storage_version.db duckdb/test/sql/storage_version.db); then | |
| echo "Expected storage format to be bumped, but this is not the case" | |
| echo "This might fail spuriously if changes to content of test database / generation script happened" | |
| exit 1 | |
| else | |
| echo "Storage bump detected, all good!" | |
| fi | |
| regression-test-binary-size: | |
| name: Regression test binary size | |
| runs-on: ubuntu-20.04 | |
| env: | |
| CC: gcc-10 | |
| CXX: g++-10 | |
| GEN: ninja | |
| CORE_EXTENSIONS: "tpch;tpcds;json;parquet" | |
| EXTENSION_STATIC_BUILD: 1 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install | |
| shell: bash | |
| run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build && pip install requests | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: | | |
| make | |
| git clone --branch ${{ env.BASE_BRANCH }} https://github.com/duckdb/duckdb.git --depth=1 | |
| cd duckdb | |
| make | |
| cd .. | |
| - name: Regression Test Extension binary size | |
| shell: bash | |
| run: | | |
| python scripts/regression_test_extension_size.py --old 'duckdb/build/release/extension' --new build/release/extension --expect json,parquet,tpch,tpcds | |
| regression-test-plan-cost: | |
| name: Regression Test Join Order Plan Cost | |
| runs-on: ubuntu-20.04 | |
| env: | |
| CC: gcc-10 | |
| CXX: g++-10 | |
| GEN: ninja | |
| CORE_EXTENSIONS: "tpch;httpfs" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install | |
| shell: bash | |
| run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build && pip install tqdm | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: | | |
| make | |
| git clone --branch ${{ env.BASE_BRANCH }} https://github.com/duckdb/duckdb.git --depth=1 | |
| cd duckdb | |
| make | |
| cd .. | |
| - name: Set up benchmarks | |
| shell: bash | |
| run: | | |
| cp -r benchmark duckdb/ | |
| - name: Regression Test IMDB | |
| if: always() | |
| shell: bash | |
| run: | | |
| python scripts/plan_cost_runner.py --old duckdb/build/release/duckdb --new build/release/duckdb --dir=benchmark/imdb_plan_cost | |
| - name: Regression Test TPCH | |
| if: always() | |
| shell: bash | |
| run: | | |
| python scripts/plan_cost_runner.py --old duckdb/build/release/duckdb --new build/release/duckdb --dir=benchmark/tpch_plan_cost | |