generate serialization after changing json #1650
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: LinuxRelease | |
| on: | |
| workflow_call: | |
| inputs: | |
| override_git_describe: | |
| type: string | |
| git_ref: | |
| type: string | |
| skip_tests: | |
| type: string | |
| workflow_dispatch: | |
| inputs: | |
| override_git_describe: | |
| type: string | |
| git_ref: | |
| type: string | |
| skip_tests: | |
| type: string | |
| repository_dispatch: | |
| push: | |
| branches: | |
| - '**' | |
| - '!main' | |
| - '!feature' | |
| paths-ignore: | |
| - '**.md' | |
| - 'tools/**' | |
| - '!tools/shell/**' | |
| - '.github/patches/duckdb-wasm/**' | |
| - '.github/workflows/**' | |
| - '!.github/workflows/LinuxRelease.yml' | |
| pull_request: | |
| types: [opened, reopened, ready_for_review] | |
| paths-ignore: | |
| - '**.md' | |
| - 'tools/**' | |
| - '!tools/shell/**' | |
| - '.github/patches/duckdb-wasm/**' | |
| - '.github/workflows/**' | |
| - '!.github/workflows/LinuxRelease.yml' | |
| concurrency: | |
| group: linuxrelease-${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}-${{ inputs.override_git_describe }} | |
| cancel-in-progress: true | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_TOKEN }} | |
| OVERRIDE_GIT_DESCRIBE: ${{ inputs.override_git_describe }} | |
| jobs: | |
| linux-release-64: | |
| # Builds binaries for linux_amd64_gcc4 | |
| name: Linux (x64) | |
| runs-on: ubuntu-latest | |
| container: quay.io/pypa/manylinux2014_x86_64 | |
| env: | |
| EXTENSION_CONFIGS: '${GITHUB_WORKSPACE}/.github/config/bundled_extensions.cmake' | |
| ENABLE_EXTENSION_AUTOLOADING: 1 | |
| ENABLE_EXTENSION_AUTOINSTALL: 1 | |
| GEN: ninja | |
| BUILD_BENCHMARK: 1 | |
| FORCE_WARN_UNUSED: 1 | |
| DUCKDB_RUN_PARALLEL_CSV_TESTS: 1 | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.git_ref }} | |
| - uses: ./.github/actions/manylinux_2014_setup | |
| with: | |
| ninja-build: 1 | |
| ccache: 0 | |
| python_alias: 1 | |
| aws-cli: 1 | |
| - name: Install pytest | |
| run: | | |
| python3 -m pip install pytest | |
| - name: Build | |
| shell: bash | |
| run: make | |
| - name: Print platform | |
| shell: bash | |
| run: ./build/release/duckdb -c "PRAGMA platform;" | |
| - name: Test | |
| shell: bash | |
| if: ${{ inputs.skip_tests != 'true' }} | |
| run: | | |
| make | |
| python3 scripts/run_tests_one_by_one.py build/release/test/unittest "*" --time_execution | |
| - name: Tools Tests | |
| shell: bash | |
| if: ${{ inputs.skip_tests != 'true' }} | |
| run: | | |
| python3 -m pytest tools/shell/tests --shell-binary build/release/duckdb | |
| - name: Examples | |
| shell: bash | |
| if: ${{ inputs.skip_tests != 'true' }} | |
| run: | | |
| (cd examples/embedded-c; make) | |
| (cd examples/embedded-c++; make) | |
| build/release/benchmark/benchmark_runner benchmark/tpch/sf1/q01.benchmark | |
| build/release/duckdb -c "COPY (SELECT 42) TO '/dev/stdout' (FORMAT PARQUET)" | cat | |
| - name: Deploy | |
| shell: bash | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} | |
| run: | | |
| python3 scripts/amalgamation.py | |
| zip -j duckdb_cli-linux-amd64.zip build/release/duckdb | |
| zip -j libduckdb-linux-amd64.zip build/release/src/libduckdb*.* src/amalgamation/duckdb.hpp src/include/duckdb.h | |
| zip -j libduckdb-src.zip src/amalgamation/duckdb.hpp src/amalgamation/duckdb.cpp src/include/duckdb.h src/include/duckdb_extension.h | |
| ./scripts/upload-assets-to-staging.sh github_release libduckdb-src.zip libduckdb-linux-amd64.zip duckdb_cli-linux-amd64.zip | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: duckdb-binaries-linux | |
| path: | | |
| libduckdb-linux-amd64.zip | |
| duckdb_cli-linux-amd64.zip | |
| linux-release-aarch64: | |
| # Builds binaries for linux_arm64 | |
| name: Linux (aarch64) | |
| if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' | |
| runs-on: ubuntu-latest | |
| needs: linux-release-64 | |
| container: ubuntu:18.04 | |
| env: | |
| EXTENSION_CONFIGS: '${GITHUB_WORKSPACE}/.github/config/bundled_extensions.cmake' | |
| ENABLE_EXTENSION_AUTOLOADING: 1 | |
| ENABLE_EXTENSION_AUTOINSTALL: 1 | |
| GEN: ninja | |
| BUILD_BENCHMARK: 1 | |
| TREAT_WARNINGS_AS_ERRORS: 1 | |
| FORCE_WARN_UNUSED: 1 | |
| DUCKDB_PLATFORM: linux_arm64 | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.git_ref }} | |
| - uses: ./.github/actions/ubuntu_18_setup | |
| with: | |
| ccache: 1 | |
| aarch64_cross_compile: 1 | |
| git_ref: ${{ inputs.git_ref }} | |
| - name: Build | |
| shell: bash | |
| run: CC=aarch64-linux-gnu-gcc CXX=aarch64-linux-gnu-g++ make | |
| - name: Deploy | |
| shell: bash | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} | |
| run: | | |
| python3 scripts/amalgamation.py | |
| zip -j duckdb_cli-linux-aarch64.zip build/release/duckdb | |
| zip -j libduckdb-linux-aarch64.zip build/release/src/libduckdb*.* src/amalgamation/duckdb.hpp src/include/duckdb.h | |
| ./scripts/upload-assets-to-staging.sh github_release libduckdb-linux-aarch64.zip duckdb_cli-linux-aarch64.zip | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: duckdb-binaries-linux-aarch64 | |
| path: | | |
| libduckdb-linux-aarch64.zip | |
| duckdb_cli-linux-aarch64.zip | |
| # Linux extensions for builds that use C++11 ABI, currently these are all linux builds based on ubuntu >= 18 (e.g. NodeJS) | |
| # note that the linux-release-64 is based on the manylinux-based extensions, which are built in .github/workflows/Python.yml | |
| linux-extensions-64: | |
| # Builds extensions for linux_amd64 | |
| name: Linux Extensions (x64) | |
| runs-on: ubuntu-latest | |
| container: ubuntu:18.04 | |
| needs: linux-release-64 | |
| env: | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.git_ref }} | |
| - uses: ./.github/actions/ubuntu_18_setup | |
| with: | |
| vcpkg: 1 | |
| openssl: 1 | |
| ccache: 1 | |
| git_ref: ${{ inputs.git_ref }} | |
| - name: Setup Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Configure OpenSSL for Rust | |
| run: | | |
| echo "OPENSSL_ROOT_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV | |
| echo "OPENSSL_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV | |
| echo "OPENSSL_INCLUDE_DIR=`pwd`/build/release/vcpkg_installed/x64-linux/include" >> $GITHUB_ENV | |
| echo "OPENSSL_USE_STATIC_LIBS=true" >> $GITHUB_ENV | |
| - uses: ./.github/actions/build_extensions | |
| with: | |
| vcpkg_target_triplet: x64-linux | |
| post_install: rm build/release/src/libduckdb* | |
| deploy_as: linux_amd64 | |
| treat_warn_as_error: 0 | |
| run_tests: ${{ inputs.skip_tests != 'true' && 1 || 0 }} | |
| run_autoload_tests: ${{ inputs.skip_tests != 'true' && 1 || 0 }} | |
| s3_id: ${{ secrets.S3_ID }} | |
| s3_key: ${{ secrets.S3_KEY }} | |
| signing_pk: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }} | |
| ninja: 1 | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: linux-extensions-64 | |
| path: | | |
| build/release/extension/*/*.duckdb_extension | |
| linux-extensions-64-aarch64: | |
| # Builds extensions for linux_arm64 | |
| name: Linux Extensions (aarch64) | |
| if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| container: ubuntu:18.04 | |
| needs: linux-release-64 | |
| env: | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.git_ref }} | |
| - uses: ./.github/actions/ubuntu_18_setup | |
| with: | |
| vcpkg: 1 | |
| openssl: 1 | |
| aarch64_cross_compile: 1 | |
| ccache: 1 | |
| git_ref: ${{ inputs.git_ref }} | |
| - name: Setup Rust for cross compilation | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| targets: aarch64-unknown-linux-gnu | |
| - name: Configure OpenSSL for Rust | |
| run: | | |
| echo "OPENSSL_ROOT_DIR=`pwd`/build/release/vcpkg_installed/arm64-linux" >> $GITHUB_ENV | |
| echo "OPENSSL_DIR=`pwd`/build/release/vcpkg_installed/arm64-linux" >> $GITHUB_ENV | |
| echo "OPENSSL_INCLUDE_DIR=`pwd`/build/release/vcpkg_installed/arm64-linux/include" >> $GITHUB_ENV | |
| echo "OPENSSL_USE_STATIC_LIBS=true" >> $GITHUB_ENV | |
| - uses: ./.github/actions/build_extensions | |
| with: | |
| vcpkg_target_triplet: arm64-linux | |
| deploy_as: linux_arm64 | |
| duckdb_arch: linux_arm64 | |
| treat_warn_as_error: 0 | |
| s3_id: ${{ secrets.S3_ID }} | |
| s3_key: ${{ secrets.S3_KEY }} | |
| signing_pk: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }} | |
| aarch64_cross_compile: 1 | |
| run_tests: 0 # Cannot run tests here due to cross-compiling | |
| run_autoload_tests: 0 | |
| ninja: 1 | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: linux-extensions-64-aarch64 | |
| path: | | |
| build/release/extension/*/*.duckdb_extension | |
| check-load-install-extensions: | |
| name: Checks extension entries | |
| if: ${{ inputs.skip_tests != 'true' }} | |
| runs-on: ubuntu-20.04 | |
| needs: linux-extensions-64 | |
| env: | |
| CC: gcc-10 | |
| CXX: g++-10 | |
| GEN: ninja | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.git_ref }} | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| - name: Install | |
| shell: bash | |
| run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| env: | |
| GENERATE_EXTENSION_ENTRIES: 1 | |
| LOCAL_EXTENSION_REPO: build/release/repository_other | |
| run: | | |
| make | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: linux-extensions-64 | |
| path: build/release/repository | |
| - name: Check if extension_entries.hpp is up to date | |
| shell: bash | |
| env: | |
| EXTENSION_CONFIGS: '.github/config/in_tree_extensions.cmake;.github/config/out_of_tree_extensions.cmake' | |
| run: | | |
| make extension_configuration | |
| python scripts/generate_extensions_function.py | |
| pip install "black>=24" | |
| pip install cmake-format | |
| make format-fix | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: extension_entries.hpp | |
| path: | | |
| src/include/duckdb/main/extension_entries.hpp | |
| - name: Check for any difference | |
| run: | | |
| git diff --exit-code src/include/duckdb/main/extension_entries.hpp && echo "No differences found" | |
| - name: Explainer | |
| if: failure() | |
| run: | | |
| echo "There are differences in src/include/duckdb/main/extension_entries.hpp" | |
| echo "Check the uploaded extension_entries.hpp (in the workflow Summary), and check that in instead of src/include/duckdb/main/extension_entries.hpp" | |
| symbol-leakage: | |
| name: Symbol Leakage | |
| if: ${{ inputs.skip_tests != 'true' }} | |
| runs-on: ubuntu-20.04 | |
| needs: linux-release-64 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.git_ref }} | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: make | |
| - name: Symbol Leakage Test | |
| shell: bash | |
| run: python3 scripts/exported_symbols_check.py build/release/src/libduckdb*.so | |
| amalgamation-tests: | |
| name: Amalgamation Tests | |
| if: ${{ inputs.skip_tests != 'true' }} | |
| needs: linux-release-64 | |
| runs-on: ubuntu-20.04 | |
| env: | |
| CC: clang | |
| CXX: clang++ | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.git_ref }} | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install LLVM and Clang | |
| uses: KyleMayes/install-llvm-action@v1 | |
| with: | |
| version: "10.0" | |
| - name: Generate Amalgamation | |
| shell: bash | |
| run: | | |
| python scripts/amalgamation.py --extended | |
| clang++ -std=c++17 -Isrc/amalgamation src/amalgamation/duckdb.cpp -emit-llvm -S -O0 | |
| linux-release-32: | |
| name: Linux (32 Bit) | |
| runs-on: ubuntu-latest | |
| container: quay.io/pypa/manylinux2014_x86_64 | |
| needs: linux-release-64 | |
| env: | |
| CC: /usr/bin/gcc | |
| CXX: /usr/bin/g++ | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/manylinux_2014_setup | |
| with: | |
| ninja-build: 1 | |
| ccache: 1 | |
| glibc32: 1 | |
| gcc_4_8: 1 # Note: we run this job on the older gcc 4.8 toolchain | |
| - name: Build | |
| shell: bash | |
| run: | | |
| mkdir -p build/release | |
| (cd build/release && cmake -G "Ninja" -DSTATIC_LIBCPP=1 -DCORE_EXTENSIONS='icu;parquet;fts;json' -DFORCE_32_BIT=1 -DCMAKE_BUILD_TYPE=Release ../.. && cmake --build .) |