OSX.yml

name: OSX
on:
  workflow_call:
    inputs:
      override_git_describe:
        type: string
      git_ref:
        type: string
      skip_tests:
        type: string
  workflow_dispatch:
    inputs:
      override_git_describe:
        type: string
      git_ref:
        type: string
      skip_tests:
        type: string
  repository_dispatch:
  push:
    branches:
      - '**'
      - '!main'
      - '!feature'
    paths-ignore:
      - '**.md'
      - 'tools/**'
      - '!tools/odbc/**'
      - '!tools/shell/**'
      - '.github/patches/duckdb-wasm/**'
      - '.github/workflows/**'
      - '!.github/workflows/OSX.yml'


concurrency:
  group: osx-${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}-${{ inputs.override_git_describe }}
  cancel-in-progress: true

env:
  GH_TOKEN: ${{ secrets.GH_TOKEN }}
  OVERRIDE_GIT_DESCRIBE: ${{ inputs.override_git_describe }}

jobs:
  xcode-debug:
    name: OSX Debug
    runs-on: macos-14

    env:
      TREAT_WARNINGS_AS_ERRORS: 1

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
        ref: ${{ inputs.git_ref }}

    - uses: actions/setup-python@v5
      with:
        python-version: '3.12'

    - name: Setup Ccache
      uses: hendrikmuhs/ccache-action@main
      with:
        key: ${{ github.job }}
        save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}

    - name: Install ninja
      shell: bash
      run: brew install ninja

    - name: Build
      shell: bash
      run: GEN=ninja make debug

    - name: Set DUCKDB_INSTALL_LIB for ADBC tests
      shell: bash
      run: echo "DUCKDB_INSTALL_LIB=$(find `pwd` -name "libduck*.dylib" | head -n 1)" >> $GITHUB_ENV

    - name: Test DUCKDB_INSTALL_LIB variable
      run: echo $DUCKDB_INSTALL_LIB

    - name: Test
      if: ${{ inputs.skip_tests != 'true' }}
      shell: bash
      run: make unittestci

    - name: Amalgamation
      if: ${{ inputs.skip_tests != 'true' }}
      shell: bash
      run: |
        python scripts/amalgamation.py --extended
        cd src/amalgamation
        clang++ -std=c++11 -O0 -Wall -Werror -emit-llvm -S duckdb.cpp
        clang++ -DNDEBUG -O0 -std=c++11 -Wall -Werror -emit-llvm -S duckdb.cpp
        clang++ -DDEBUG -O0 -std=c++11 -Wall -Werror -emit-llvm -S duckdb.cpp

  xcode-release:
    # Builds binaries for osx_arm64 and osx_amd64
    name: OSX Release
    runs-on: macos-14
    needs: xcode-debug
    env:
      EXTENSION_CONFIGS: '${GITHUB_WORKSPACE}/.github/config/bundled_extensions.cmake'
      ENABLE_EXTENSION_AUTOLOADING: 1
      ENABLE_EXTENSION_AUTOINSTALL: 1
      BUILD_ODBC: 1
      OSX_BUILD_UNIVERSAL: 1
      GEN: ninja
      ODBC_CONFIG: ../../build/unixodbc/build/bin/odbc_config

    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
          ref: ${{ inputs.git_ref }}

      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install Ninja
        run: brew install ninja

      - name: Setup Ccache
        uses: hendrikmuhs/ccache-action@main
        with:
          key: ${{ github.job }}
          save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}

      - name: Install UnixODBC
        shell: bash
        run: |
          CFLAGS="-arch x86_64 -arch arm64" ./scripts/install_unixodbc.sh

      - name: Install pytest
        run: |
          python -m pip install pytest

      - name: Build
        shell: bash
        run: make

      - name: Print platform
        shell: bash
        run: ./build/release/duckdb -c "PRAGMA platform;"

      - name: Unit Test
        shell: bash
        if: ${{ inputs.skip_tests != 'true' }}
        run: make allunit

      - name: Tools Tests
        shell: bash
        if: ${{ inputs.skip_tests != 'true' }}
        run: |
          python -m pytest tools/shell/tests --shell-binary build/release/duckdb

      - name: Examples
        shell: bash
        if: ${{ inputs.skip_tests != 'true' }}
        run: |
          (cd examples/embedded-c; make)
          (cd examples/embedded-c++; make)

        #     from https://docs.github.com/en/actions/deployment/deploying-xcode-applications/installing-an-apple-certificate-on-macos-runners-for-xcode-development
      - name: Sign Binaries
        shell: bash
        env:
          BUILD_CERTIFICATE_BASE64: ${{ secrets.OSX_CODESIGN_BUILD_CERTIFICATE_BASE64 }}
          P12_PASSWORD: ${{ secrets.OSX_CODESIGN_P12_PASSWORD }}
          KEYCHAIN_PASSWORD: ${{ secrets.OSX_CODESIGN_KEYCHAIN_PASSWORD }}
        run: |
          if [[ "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then
            . scripts/osx_import_codesign_certificate.sh

            codesign --all-architectures --force --sign "Developer ID Application: Stichting DuckDB Foundation" build/release/duckdb
            codesign --all-architectures --force --sign "Developer ID Application: Stichting DuckDB Foundation" build/release/src/libduckdb*.dylib
            codesign --all-architectures --force --sign "Developer ID Application: Stichting DuckDB Foundation" build/release/tools/odbc/libduckdb_odbc.dylib
          fi

      - name: Deploy
        shell: bash
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
        run: |
          python scripts/amalgamation.py
          zip -j duckdb_cli-osx-universal.zip build/release/duckdb
          zip -j libduckdb-osx-universal.zip build/release/src/libduckdb*.dylib src/amalgamation/duckdb.hpp src/include/duckdb.h
          zip -j duckdb_odbc-osx-universal.zip build/release/tools/odbc/libduckdb_odbc.dylib
          ./scripts/upload-assets-to-staging.sh github_release libduckdb-osx-universal.zip duckdb_cli-osx-universal.zip duckdb_odbc-osx-universal.zip

      - uses: actions/upload-artifact@v3
        with:
          name: duckdb-binaries-osx
          path: |
            libduckdb-osx-universal.zip
            duckdb_cli-osx-universal.zip
            duckdb_odbc-osx-universal.zip

  xcode-extensions:
    # Builds extensions for osx_arm64 and osx_amd64
    name: OSX Extensions Release
    runs-on: macos-14
    needs: xcode-debug
    strategy:
      matrix:
        label: [ 'x86_64', 'arm64' ]
        include:
          - label: x86_64
            osx_arch: x86_64
            duckdb_arch: osx_amd64
            vcpkg_triplet: x64-osx
            run_autoload_tests: 0
          - label: arm64
            osx_arch: arm64
            duckdb_arch: osx_arm64
            vcpkg_triplet: arm64-osx
            run_autoload_tests: ${{ inputs.skip_tests != 'true' }}

    env:
      DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }}
      AWS_DEFAULT_REGION: us-east-1
      VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }}
      GEN: ninja
      DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }}

    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
          ref: ${{ inputs.git_ref }}

      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install Ninja and Pkg-config
        run: brew install pkg-config ninja

      - name: Setup Ccache
        uses: hendrikmuhs/ccache-action@main
        with:
          key: ${{ github.job }}-${{ matrix.label }}
          save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}

      - uses: ./.github/actions/build_extensions
        with:
          treat_warn_as_error: 0
          run_tests: 0
          osx_arch: ${{ matrix.osx_arch }}
          vcpkg_target_triplet: ${{ matrix.vcpkg_triplet }}
          duckdb_arch: ${{ matrix.duckdb_arch }}
          build_in_tree_extensions: 1
          build_out_of_tree_extensions: 1
          run_autoload_tests: ${{ matrix.run_autoload_tests }}
          ninja: 1

      - name: Test statically linked extensions
        if: ${{ matrix.osx_arch == 'arm64' && inputs.skip_tests != 'true' }}
        shell: bash
        run: |
          python3 scripts/run_tests_one_by_one.py ./build/release/test/unittest

      - name: Sign Extension Binaries
        env:
          BUILD_CERTIFICATE_BASE64: ${{ secrets.OSX_CODESIGN_BUILD_CERTIFICATE_BASE64 }}
          P12_PASSWORD: ${{ secrets.OSX_CODESIGN_P12_PASSWORD }}
          KEYCHAIN_PASSWORD: ${{ secrets.OSX_CODESIGN_KEYCHAIN_PASSWORD }}
        run: |
          if [[ "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then
          . scripts/osx_import_codesign_certificate.sh
          codesign --all-architectures --force --sign "Developer ID Application: Stichting DuckDB Foundation" build/release/extension/*/*.duckdb_extension
          fi

      - name: Deploy
        shell: bash
        env:
          AWS_ACCESS_KEY_ID: ${{secrets.S3_ID}}
          AWS_SECRET_ACCESS_KEY: ${{secrets.S3_KEY}}
          DUCKDB_DEPLOY_SCRIPT_MODE: for_real
        run: |
          if [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ && "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then
            pip install awscli
            ./scripts/extension-upload-all.sh ${{ matrix.duckdb_arch }} ${{ github.ref_name }}
          elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ && "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then
            pip install awscli
            ./scripts/extension-upload-all.sh ${{ matrix.duckdb_arch }} `git log -1 --format=%h`
          else
            echo "would do: ./scripts/extension-upload-all.sh ${{ matrix.duckdb_arch }} ${{ github.ref_name }}"
          fi

      - name: Test loadable extensions
        if: ${{ matrix.osx_arch == 'arm64' &&  inputs.skip_tests != 'true' }}
        shell: bash
        env:
          AWS_ACCESS_KEY_ID: ${{secrets.S3_ID}}
          AWS_SECRET_ACCESS_KEY: ${{secrets.S3_KEY}}
          AWS_DEFAULT_REGION: us-east-1
        run: |
          if [[ "$GITHUB_REF" =~ ^(refs/heads/main|refs/tags/v.+)$ && "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then
            ./scripts/extension-upload-test.sh
          fi

      - name: Rebuild DuckDB without any extensions linked, but with same extension config
        if: ${{ matrix.osx_arch == 'arm64' }}
        shell: bash
        env:
          EXTENSION_TESTS_ONLY: 1
          ENABLE_EXTENSION_AUTOLOADING: 1
          ENABLE_EXTENSION_AUTOINSTALL: 1
        run: |
          rm -rf build/release
          make

      # Run autoload unittests (including the out-of-tree tests) without any extensions linked, relying on the autoloader
      - name: Run tests with auto loading
        if: ${{ matrix.osx_arch == 'arm64' && inputs.skip_tests != 'true' }}
        shell: bash
        env:
          LOCAL_EXTENSION_REPO: ${{ github.workspace }}
        run: |
          python3 scripts/get_test_list.py --file-contains 'require ' --list '"*.test"' > test.list
          python3 scripts/get_test_list.py --file-contains 'require-env LOCAL_EXTENSION_REPO' --list '"*.test"' >> test.list
          python3 scripts/run_tests_one_by_one.py ./build/release/test/unittest '-f test.list'