From fd456b63c483bb6420d1979943b1adc7799e02a5 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 24 Nov 2023 10:15:40 +0100 Subject: [PATCH] Deploy Wasm artifacts --- .../workflows/MainDistributionPipeline.yml | 12 ++ .github/workflows/_extension_deploy_wasm.yml | 122 ++++++++++++++++++ scripts/extension-upload-wasm.sh | 61 +++++++++ 3 files changed, 195 insertions(+) create mode 100644 .github/workflows/_extension_deploy_wasm.yml create mode 100755 scripts/extension-upload-wasm.sh diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index c757460..e40f9e3 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -19,6 +19,7 @@ jobs: vcpkg_commit: a42af01b72c28a8e1d7b48107b33e4f286a55ef6 duckdb_version: v0.9.2 extension_name: quack + exclude_archs: "" duckdb-stable-deploy: name: Deploy extension binaries @@ -30,3 +31,14 @@ jobs: extension_name: quack deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} deploy_versioned: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} + + duckdb-wasm-stable-deploy: + name: Deploy Wasm extension binaries + needs: duckdb-stable-build + uses: ./.github/workflows/_extension_deploy_wasm.yml + secrets: inherit + with: + duckdb_version: v0.9.2 + extension_name: quack + deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} + deploy_versioned: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} diff --git a/.github/workflows/_extension_deploy_wasm.yml b/.github/workflows/_extension_deploy_wasm.yml new file mode 100644 index 0000000..b33352a --- /dev/null +++ b/.github/workflows/_extension_deploy_wasm.yml @@ -0,0 +1,122 @@ +# +# Reusable workflow that deploys the artifacts produced by github.com/duckdb/duckdb/.github/workflows/_extension_distribution.yml +# +# note: this workflow needs to be located in the extension repository, as it requires secrets to be passed to the +# deploy script. However, it should generally not be necessary to modify this workflow in your extension repository, as +# this workflow can be configured to use a custom deploy script. + + +name: Extension Deployment Wasm +on: + workflow_call: + inputs: + # The name of the extension + extension_name: + required: true + type: string + # DuckDB version to build against + duckdb_version: + required: true + type: string + # ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64' + exclude_archs: + required: false + type: string + default: "" + # Whether to upload this deployment as the latest. This may overwrite a previous deployment. + deploy_latest: + required: false + type: boolean + default: false + # Whether to upload this deployment under a versioned path. These will not be deleted automatically + deploy_versioned: + required: false + type: boolean + default: false + # Postfix added to artifact names. Can be used to guarantee unique names when this workflow is called multiple times + artifact_postfix: + required: false + type: string + default: "" + # Override the default deploy script with a custom script + deploy_script: + required: false + type: string + default: "./scripts/extension-upload-wasm.sh" + # Override the default matrix parse script with a custom script + matrix_parse_script: + required: false + type: string + default: "./duckdb/scripts/modify_distribution_matrix.py" + +jobs: + generate_matrix: + name: Generate matrix + runs-on: ubuntu-latest + outputs: + deploy_matrix: ${{ steps.parse-matrices.outputs.deploy_matrix }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'true' + + - name: Checkout DuckDB to version + run: | + cd duckdb + git checkout ${{ inputs.duckdb_version }} + + - id: parse-matrices + run: | + echo '{"wasm":{"include":[{"duckdb_arch":"wasm_mvp","vcpkg_triplet":"wasm32-emscripten"},{"duckdb_arch":"wasm_eh","vcpkg_triplet":"wasm32-emscripten"},{"duckdb_arch":"wasm_threads","vcpkg_triplet":"wasm32-emscripten"}]}}' > wasm_matrix_input.json + python3 ${{ inputs.matrix_parse_script }} --input wasm_matrix_input.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + deploy_matrix="`cat deploy_matrix.json`" + echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT + echo `cat $GITHUB_OUTPUT` + + deploy: + name: Deploy + runs-on: ubuntu-latest + needs: generate_matrix + if: ${{ needs.generate_matrix.outputs.deploy_matrix != '{}' && needs.generate_matrix.outputs.deploy_matrix != '' }} + strategy: + matrix: ${{fromJson(needs.generate_matrix.outputs.deploy_matrix)}} + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'true' + + - name: Checkout DuckDB to version + run: | + cd duckdb + git checkout ${{ inputs.duckdb_version }} + + - uses: actions/download-artifact@v2 + with: + name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} + path: | + /tmp/extension + + - name: Deploy + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_ORG_DEPLOY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_ORG_DEPLOY_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.S3_DUCKDB_ORG_REGION }} + BUCKET_NAME: ${{ secrets.S3_DUCKDB_ORG_BUCKET }} + DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.S3_DUCKDB_ORG_EXTENSION_SIGNING_PK }} + run: | + pwd + python3 -m pip install pip awscli + git config --global --add safe.directory '*' + cd duckdb + git fetch --tags + export DUCKDB_VERSION=`git tag --points-at HEAD` + export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} + cd .. + git fetch --tags + export EXT_VERSION=`git tag --points-at HEAD` + export EXT_VERSION=${EXT_VERSION:=`git log -1 --format=%h`} + ${{ inputs.deploy_script }} ${{ inputs.extension_name }} $EXT_VERSION $DUCKDB_VERSION ${{ matrix.duckdb_arch }} $BUCKET_NAME ${{inputs.deploy_latest || 'true' && 'false'}} ${{inputs.deploy_versioned || 'true' && 'false'}} diff --git a/scripts/extension-upload-wasm.sh b/scripts/extension-upload-wasm.sh new file mode 100755 index 0000000..f392397 --- /dev/null +++ b/scripts/extension-upload-wasm.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# Extension upload script + +# Usage: ./extension-upload-wasm.sh +# : Name of the extension +# : Version (commit / version tag) of the extension +# : Version (commit / version tag) of DuckDB +# : Architecture target of the extension binary +# : S3 bucket to upload to +# : Set this as the latest version ("true" / "false", default: "false") +# : Set this as a versioned version that will prevent its deletion + +set -e + +f="/tmp/extension/$1.duckdb_extension.wasm" + +script_dir="$(dirname "$(readlink -f "$0")")" + +# calculate SHA256 hash of extension binary +cat $f > $f.append +# 0 for custom section +# 113 in hex = 275 in decimal, total lenght of what follows (1 + 16 + 2 + 256) +# [1(continuation) + 0010011(payload) = \x93, 0(continuation) + 10(payload) = \x02] +echo -n -e '\x00' >> $f.append +echo -n -e '\x93\x02' >> $f.append +# 10 in hex = 16 in decimal, lenght of name, 1 byte +echo -n -e '\x10' >> $f.append +echo -n -e 'duckdb_signature' >> $f.append +# the name of the WebAssembly custom section, 16 bytes +# 100 in hex, 256 in decimal +# [1(continuation) + 0000000(payload) = ff, 0(continuation) + 10(payload)], +# for a grand total of 2 bytes +echo -n -e '\x80\x02' >> $f.append + +# (Optionally) Sign binary +if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then + echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem + $script_dir/../duckdb/scripts/compute-extension-hash.sh $f.append > $f.hash + openssl pkeyutl -sign -in $f.hash -inkey private.pem -pkeyopt digest:sha256 -out $f.sign + rm -f private.pem +fi + +truncate -s 256 $f.sign + +# append signature to extension binary +cat $f.sign >> $f.append +# compress extension binary +brotli < $f.append > "$f.brotli" +set -e + +# Abort if AWS key is not set +if [ -z "$AWS_ACCESS_KEY_ID" ]; then + echo "No AWS key found, skipping.." + exit 0 +fi + +# upload to latest version +if [[ $6 = 'true' ]]; then + aws s3 cp $ext.gz s3://$5/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" +fi