More moves #357
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: LinuxRelease | |
| on: | |
| workflow_dispatch: | |
| repository_dispatch: | |
| push: | |
| branches: | |
| - '**' | |
| - '!master' | |
| - '!feature' | |
| tags: | |
| - '**' | |
| paths-ignore: | |
| - '**.md' | |
| - 'tools/**' | |
| - '.github/workflows/**' | |
| - '!.github/workflows/LinuxRelease.yml' | |
| - '.github/config/uncovered_files.csv' | |
| pull_request: | |
| paths-ignore: | |
| - '**.md' | |
| - 'tools/**' | |
| - '.github/workflows/**' | |
| - '!.github/workflows/LinuxRelease.yml' | |
| - '.github/config/uncovered_files.csv' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/master' || github.sha }} | |
| cancel-in-progress: true | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_TOKEN }} | |
| jobs: | |
| linux-release-64: | |
| name: Linux (64 Bit) | |
| runs-on: ubuntu-latest | |
| container: ubuntu:16.04 | |
| env: | |
| GEN: ninja | |
| BUILD_BENCHMARK: 1 | |
| BUILD_ICU: 1 | |
| BUILD_INET: 1 | |
| BUILD_TPCH: 1 | |
| BUILD_FTS: 1 | |
| BUILD_JSON: 1 | |
| BUILD_EXCEL: 1 | |
| BUILD_ODBC: 1 | |
| BUILD_JEMALLOC: 1 | |
| DEBUG_STACKTRACE: 1 | |
| BUILD_AUTOCOMPLETE: 1 | |
| TREAT_WARNINGS_AS_ERRORS: 1 | |
| FORCE_WARN_UNUSED: 1 | |
| STATIC_LIBCPP: 1 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/ubuntu_16_setup | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: make | |
| - name: Test | |
| shell: bash | |
| run: make allunit | |
| - name: Tools Tests | |
| shell: bash | |
| run: | | |
| python3.7 tools/shell/shell-test.py build/release/duckdb | |
| - name: Examples | |
| shell: bash | |
| run: | | |
| (cd examples/embedded-c; make) | |
| (cd examples/embedded-c++; make) | |
| build/release/benchmark/benchmark_runner benchmark/tpch/sf1/q01.benchmark | |
| build/release/duckdb -c "COPY (SELECT 42) TO '/dev/stdout' (FORMAT PARQUET)" | cat | |
| - name: Deploy | |
| shell: bash | |
| run: | | |
| python3.7 scripts/amalgamation.py | |
| zip -j duckdb_cli-linux-amd64.zip build/release/duckdb | |
| zip -j libduckdb-linux-amd64.zip build/release/src/libduckdb*.so src/amalgamation/duckdb.hpp src/include/duckdb.h | |
| zip -j libduckdb-src.zip src/amalgamation/duckdb.hpp src/amalgamation/duckdb.cpp src/include/duckdb.h | |
| zip -j duckdb_odbc-linux-amd64.zip build/release/tools/odbc/libduckdb_odbc.so tools/odbc/linux_setup/unixodbc_setup.sh | |
| python3.7 scripts/asset-upload-gha.py libduckdb-src.zip libduckdb-linux-amd64.zip duckdb_cli-linux-amd64.zip duckdb_odbc-linux-amd64.zip | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: duckdb-binaries-linux | |
| path: | | |
| libduckdb-linux-amd64.zip | |
| duckdb_cli-linux-amd64.zip | |
| linux-release-aarch64: | |
| name: Linux (AARCH64) | |
| runs-on: ubuntu-latest | |
| needs: linux-release-64 | |
| container: ubuntu:18.04 # cross compiler not available in 16 | |
| env: | |
| GEN: ninja | |
| BUILD_BENCHMARK: 1 | |
| BUILD_ICU: 1 | |
| BUILD_INET: 1 | |
| BUILD_TPCH: 1 | |
| BUILD_FTS: 1 | |
| BUILD_JSON: 1 | |
| BUILD_EXCEL: 1 | |
| BUILD_JEMALLOC: 1 | |
| TREAT_WARNINGS_AS_ERRORS: 1 | |
| FORCE_WARN_UNUSED: 1 | |
| STATIC_LIBCPP: 1 | |
| BUILD_ODBC: 1 | |
| ODBC_CONFIG: ../../build/unixodbc/build/bin/odbc_config | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/ubuntu_16_setup | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' }} | |
| - name: Install Stuff | |
| shell: bash | |
| run: | | |
| apt-get update -y -qq | |
| apt-get install -y -qq gcc-aarch64-linux-gnu g++-aarch64-linux-gnu | |
| - name: Install unixODBC | |
| shell: bash | |
| run: | # we need an x86 odbc_config tool to run cmake. fun. | |
| apt-get remove -y unixodbc unixodbc-dev | |
| CC=gcc ./scripts/install_unixodbc.sh | |
| cp build/unixodbc/build/bin/odbc_config . | |
| CC=aarch64-linux-gnu-gcc ./scripts/install_unixodbc.sh --host aarch64-linux-gnu | |
| cp odbc_config build/unixodbc/build/bin/odbc_config | |
| - name: Build | |
| shell: bash | |
| run: CC=aarch64-linux-gnu-gcc CXX=aarch64-linux-gnu-g++ make | |
| - name: Deploy | |
| shell: bash | |
| run: | | |
| python3.7 scripts/amalgamation.py | |
| zip -j duckdb_cli-linux-aarch64.zip build/release/duckdb | |
| zip -j duckdb_odbc-linux-aarch64.zip build/release/tools/odbc/libduckdb_odbc.so | |
| zip -j libduckdb-linux-aarch64.zip build/release/src/libduckdb*.so src/amalgamation/duckdb.hpp src/include/duckdb.h | |
| python3.7 scripts/asset-upload-gha.py libduckdb-linux-aarch64.zip duckdb_cli-linux-aarch64.zip duckdb_odbc-linux-aarch64.zip | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: duckdb-binaries-linux-aarch64 | |
| path: | | |
| libduckdb-linux-aarch64.zip | |
| duckdb_odbc-linux-aarch64.zip | |
| duckdb_cli-linux-aarch64.zip | |
| linux-extensions-64: | |
| name: Linux Extensions (64 Bit) | |
| runs-on: ubuntu-latest | |
| container: ubuntu:16.04 | |
| needs: linux-release-64 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/ubuntu_16_setup | |
| with: | |
| openssl: 1 | |
| - name: Configure OpenSSL path | |
| shell: bash | |
| run: | | |
| echo "OPENSSL_ROOT_DIR=/usr/local/ssl" >> $GITHUB_ENV | |
| - uses: ./.github/actions/build_extensions | |
| with: | |
| post_install: rm build/release/src/libduckdb* | |
| deploy_as: linux_amd64 | |
| s3_id: ${{ secrets.S3_ID }} | |
| s3_key: ${{ secrets.S3_KEY }} | |
| signing_pk: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }} | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: linux-extensions-64 | |
| path: | | |
| build/release/extension/*/*.duckdb_extension | |
| linux-extensions-64-aarch64: | |
| name: Linux Extensions (AARCH64) | |
| runs-on: ubuntu-latest | |
| container: ubuntu:18.04 # cross compiler not available in 16 | |
| needs: linux-release-64 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/ubuntu_16_setup | |
| with: | |
| openssl: 1 | |
| aarch64_cross_compile: 1 | |
| - name: Configure OpenSSL path | |
| shell: bash | |
| run: | | |
| export OPENSSL_ROOT_DIR=`pwd`/build/openssl/build | |
| echo "OPENSSL_ROOT_DIR=$OPENSSL_ROOT_DIR" >> $GITHUB_ENV | |
| - uses: ./.github/actions/build_extensions | |
| with: | |
| openssl_path: ${{ env.OPENSSL_ROOT_DIR }} | |
| deploy_as: linux_arm64 | |
| treat_warn_as_error: 0 | |
| s3_id: ${{ secrets.S3_ID }} | |
| s3_key: ${{ secrets.S3_KEY }} | |
| signing_pk: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }} | |
| aarch64_cross_compile: 1 | |
| run_tests: 0 # Cannot run tests here due to cross-compiling | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: linux-extensions-64-aarch64 | |
| path: | | |
| build/release/extension/*/*.duckdb_extension | |
| check-load-install-extensions: | |
| name: Checks extension functions | |
| runs-on: ubuntu-20.04 | |
| needs: linux-extensions-64 | |
| env: | |
| CC: gcc-10 | |
| CXX: g++-10 | |
| GEN: ninja | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - name: Install | |
| shell: bash | |
| run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: | | |
| mkdir -p build/release | |
| cd build/release | |
| cmake -G "Ninja" -DCMAKE_BUILD_TYPE=Release ../.. | |
| cmake --build . --config Release | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: linux-extensions-64 | |
| path: /tmp | |
| - name: Check if extension_functions.hpp is up to date | |
| shell: bash | |
| run: | | |
| python scripts/generate_extensions_function.py --validate | |
| linux-release-32: | |
| name: Linux (32 Bit) | |
| runs-on: ubuntu-latest | |
| container: ubuntu:16.04 | |
| needs: linux-release-64 | |
| env: | |
| GEN: ninja | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: ./.github/actions/ubuntu_16_setup | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: | | |
| mkdir -p build/release | |
| (cd build/release && cmake -DSTATIC_LIBCPP=1 -DBUILD_ICU_EXTENSION=1 -DBUILD_PARQUET_EXTENSION=1 -DBUILD_FTS_EXTENSION=1 -DBUILD_JSON_EXTENSION=1 -DBUILD_EXCEL_EXTENSION=1 -DFORCE_32_BIT=1 -DCMAKE_BUILD_TYPE=Release ../.. && cmake --build .) | |
| - name: Test | |
| shell: bash | |
| run: build/release/test/unittest "*" | |
| - name: Deploy | |
| shell: bash | |
| run: | | |
| python3.7 scripts/amalgamation.py | |
| zip -j duckdb_cli-linux-i386.zip build/release/duckdb | |
| zip -j libduckdb-linux-i386.zip build/release/src/libduckdb*.so src/amalgamation/duckdb.hpp src/include/duckdb.h | |
| python3.7 scripts/asset-upload-gha.py libduckdb-linux-i386.zip duckdb_cli-linux-i386.zip | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: duckdb-binaries-linux-32bit | |
| path: | | |
| libduckdb-linux-i386.zip | |
| duckdb_cli-linux-i386.zip | |
| linux-rpi: | |
| name: Linux (Raspberry Pi) | |
| runs-on: ubuntu-20.04 | |
| needs: linux-release-64 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.7' | |
| - name: Install | |
| shell: bash | |
| run: | | |
| sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build | |
| git clone https://github.com/raspberrypi/tools --depth=1 rpi-tools | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: | | |
| export TOOLCHAIN=`pwd`/rpi-tools | |
| mkdir -p build/release | |
| cd build/release | |
| cmake -G Ninja -DBUILD_TPCH_EXTENSION=1 -DBUILD_TPCDS_EXTENSION=1 -DDUCKDB_RPI_TOOLCHAIN_PREFIX=$TOOLCHAIN -DBUILD_UNITTESTS=0 -DCMAKE_TOOLCHAIN_FILE=../../scripts/raspberry-pi-cmake-toolchain.cmake ../../ | |
| cmake --build . | |
| file duckdb | |
| - name: Deploy | |
| shell: bash | |
| run: | | |
| python scripts/amalgamation.py | |
| zip -j duckdb_cli-linux-rpi.zip build/release/duckdb | |
| zip -j libduckdb-linux-rpi.zip build/release/src/libduckdb*.so src/amalgamation/duckdb.hpp src/include/duckdb.h | |
| python scripts/asset-upload-gha.py libduckdb-linux-rpi.zip duckdb_cli-linux-rpi.zip | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: duckdb-binaries-rpi | |
| path: | | |
| libduckdb-linux-rpi.zip | |
| duckdb_cli-linux-rpi.zip | |
| linux-clang: | |
| name: Clang 14 | |
| runs-on: ubuntu-20.04 | |
| needs: linux-release-64 | |
| env: | |
| CC: /home/runner/work/llvm/bin/clang | |
| CPP: /home/runner/work/llvm/bin/clang-cpp | |
| CXX: /home/runner/work/llvm/bin/clang++ | |
| LD: /home/runner/work/llvm/bin/ld.lld | |
| EXTENSION_STATIC_BUILD: 1 | |
| BUILD_JSON: 1 | |
| TREAT_WARNINGS_AS_ERRORS: 1 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install LLVM and Clang | |
| uses: KyleMayes/install-llvm-action@v1 | |
| with: | |
| version: "14.0" | |
| directory: '/home/runner/work/llvm' | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.7' | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: make release | |
| - name: Test | |
| shell: bash | |
| run: make allunit | |
| old-gcc: | |
| name: GCC 4.8 | |
| runs-on: ubuntu-20.04 | |
| container: ubuntu:16.04 | |
| needs: linux-release-64 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/ubuntu_16_setup | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.7' | |
| - name: Install | |
| shell: bash | |
| run: apt-get update -y -qq && apt-get install -y -qq g++-4.8 binutils | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: make release | |
| env: | |
| CC: gcc-4.8 | |
| CXX: g++-4.8 | |
| - name: Test | |
| shell: bash | |
| run: make allunit | |
| env: | |
| CC: gcc-4.8 | |
| CXX: g++-4.8 | |
| release-assert: | |
| name: Release Assertions | |
| runs-on: ubuntu-20.04 | |
| needs: linux-release-64 | |
| env: | |
| CC: gcc-10 | |
| CXX: g++-10 | |
| GEN: ninja | |
| BUILD_ICU: 1 | |
| BUILD_INET: 1 | |
| BUILD_TPCH: 1 | |
| BUILD_TPCDS: 1 | |
| BUILD_FTS: 1 | |
| BUILD_EXCEL: 1 | |
| BUILD_VISUALIZER: 1 | |
| BUILD_JSON: 1 | |
| BUILD_JEMALLOC: 1 | |
| DISABLE_SANITIZER: 1 | |
| CRASH_ON_ASSERT: 1 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install | |
| shell: bash | |
| run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: make relassert | |
| - name: Test | |
| shell: bash | |
| run: | | |
| python3 scripts/run_tests_one_by_one.py build/relassert/test/unittest "*" | |
| vector-sizes: | |
| name: Vector Sizes | |
| runs-on: ubuntu-20.04 | |
| needs: linux-release-64 | |
| env: | |
| CC: gcc-10 | |
| CXX: g++-10 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.7' | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' }} | |
| - name: Test | |
| shell: bash | |
| run: python scripts/test_vector_sizes.py | |
| symbol-leakage: | |
| name: Symbol Leakage | |
| runs-on: ubuntu-20.04 | |
| needs: linux-release-64 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.7' | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: make | |
| - name: Symbol Leakage Test | |
| shell: bash | |
| run: python3.7 scripts/exported_symbols_check.py build/release/src/libduckdb*.so | |
| linux-memory-leaks: | |
| name: Linux Memory Leaks | |
| runs-on: ubuntu-20.04 | |
| needs: linux-release-64 | |
| env: | |
| GEN: ninja | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.7' | |
| - name: Install Ninja | |
| shell: bash | |
| run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: make | |
| - name: Test | |
| shell: bash | |
| run: | | |
| python3 test/memoryleak/test_memory_leaks.py | |
| linux-httpfs: | |
| name: Linux HTTPFS | |
| runs-on: ubuntu-20.04 | |
| needs: linux-release-64 | |
| env: | |
| BUILD_VISUALIZER: 1 | |
| BUILD_HTTPFS: 1 | |
| BUILD_TPCH: 1 | |
| BUILD_TPCDS: 1 | |
| BUILD_PARQUET: 1 | |
| BUILD_JSON: 1 | |
| S3_TEST_SERVER_AVAILABLE: 1 | |
| AWS_DEFAULT_REGION: eu-west-1 | |
| AWS_ACCESS_KEY_ID: minio_duckdb_user | |
| AWS_SECRET_ACCESS_KEY: minio_duckdb_user_password | |
| DUCKDB_S3_ENDPOINT: duckdb-minio.com:9000 | |
| DUCKDB_S3_USE_SSL: false | |
| GEN: ninja | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.7' | |
| - name: Install Ninja | |
| shell: bash | |
| run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: make | |
| - name: Start S3/HTTP test server | |
| shell: bash | |
| run: | | |
| sudo ./scripts/install_s3_test_server.sh | |
| ./scripts/generate_presigned_url.sh | |
| source ./scripts/run_s3_test_server.sh | |
| sleep 60 | |
| - name: Test | |
| shell: bash | |
| run: | | |
| python3 scripts/get_test_list.py --file-contains 'require-env S3_TEST_SERVER_AVAILABLE 1' --list '"*"' > test.list | |
| build/release/test/unittest -f test.list | |
| amalgamation-tests: | |
| name: Amalgamation Tests | |
| needs: linux-release-64 | |
| runs-on: ubuntu-20.04 | |
| env: | |
| CC: clang | |
| CXX: clang++ | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.7' | |
| - name: Install LLVM and Clang | |
| uses: KyleMayes/install-llvm-action@v1 | |
| with: | |
| version: "10.0" | |
| - name: Generate Amalgamation | |
| shell: bash | |
| run: | | |
| python scripts/amalgamation.py --extended | |
| clang++ -std=c++17 -Isrc/amalgamation src/amalgamation/duckdb.cpp -emit-llvm -S -O0 |