From 0cbbfbd970e4a2cb039688770f364099f1d99f16 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 23 Nov 2023 16:08:33 +0100 Subject: [PATCH 01/10] Makefile changes to make it work with DuckDB-Wasm / Emscripten --- Makefile | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 44ffe48..637f634 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all clean format debug release duckdb_debug duckdb_release pull update +.PHONY: all clean format debug release duckdb_debug duckdb_release pull update wasm_mvp wasm_eh wasm_threads all: release @@ -31,6 +31,8 @@ ifeq ($(GEN),ninja) GENERATOR=-G "Ninja" -DFORCE_COLORED_OUTPUT=1 endif +EXT_NAME=quack + #### Configuration for this extension EXTENSION_NAME=QUACK EXTENSION_FLAGS=\ @@ -109,3 +111,23 @@ clean: rm -rf testext cd duckdb && make clean cd duckdb && make clean-python + +WASM_LINK_TIME_FLAGS= + +wasm_mvp: + mkdir -p build/wasm_mvp + emcmake cmake $(GENERATOR) -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_mvp -DCMAKE_CXX_FLAGS="-DDUCKDB_CUSTOM_PLATFORM=wasm_mvp" -DSKIP_EXTENSIONS="parquet" -S duckdb $(TOOLCHAIN_FLAGS) $(EXTENSION_FLAGS) -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake + emmake make -j8 -Cbuild/wasm_mvp + cd build/wasm_mvp/extension/${EXT_NAME} && emcc $f -sSIDE_MODULE=1 -o ../../${EXT_NAME}.duckdb_extension.wasm -O3 ${EXT_NAME}.duckdb_extension $(WASM_LINK_TIME_FLAGS) + +wasm_eh: + mkdir -p build/wasm_eh + emcmake cmake $(GENERATOR) -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_eh -DCMAKE_CXX_FLAGS="-fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1 -DDUCKDB_CUSTOM_PLATFORM=wasm_eh" -DSKIP_EXTENSIONS="parquet" -S duckdb $(TOOLCHAIN_FLAGS) $(EXTENSION_FLAGS) -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake + emmake make -j8 -Cbuild/wasm_eh + cd build/wasm_eh/extension/${EXT_NAME} && emcc $f -sSIDE_MODULE=1 -o ../../${EXT_NAME}.duckdb_extension.wasm -O3 ${EXT_NAME}.duckdb_extension $(WASM_LINK_TIME_FLAGS) + +wasm_threads: + mkdir -p ./build/wasm_threads + emcmake cmake $(GENERATOR) -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_threads -DCMAKE_CXX_FLAGS="-fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1 -DWITH_WASM_THREADS=1 -DWITH_WASM_SIMD=1 -DWITH_WASM_BULK_MEMORY=1 -DDUCKDB_CUSTOM_PLATFORM=wasm_threads" -DSKIP_EXTENSIONS="parquet" -S duckdb $(TOOLCHAIN_FLAGS) $(EXTENSION_FLAGS) -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake + emmake make -j8 -Cbuild/wasm_threads + cd build/wasm_threads/extension/${EXT_NAME} && emcc $f -sSIDE_MODULE=1 -o ../../${EXT_NAME}.duckdb_extension.wasm -O3 ${EXT_NAME}.duckdb_extension $(WASM_LINK_TIME_FLAGS) From fd456b63c483bb6420d1979943b1adc7799e02a5 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 24 Nov 2023 10:15:40 +0100 Subject: [PATCH 02/10] Deploy Wasm artifacts --- .../workflows/MainDistributionPipeline.yml | 12 ++ .github/workflows/_extension_deploy_wasm.yml | 122 ++++++++++++++++++ scripts/extension-upload-wasm.sh | 61 +++++++++ 3 files changed, 195 insertions(+) create mode 100644 .github/workflows/_extension_deploy_wasm.yml create mode 100755 scripts/extension-upload-wasm.sh diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index c757460..e40f9e3 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -19,6 +19,7 @@ jobs: vcpkg_commit: a42af01b72c28a8e1d7b48107b33e4f286a55ef6 duckdb_version: v0.9.2 extension_name: quack + exclude_archs: "" duckdb-stable-deploy: name: Deploy extension binaries @@ -30,3 +31,14 @@ jobs: extension_name: quack deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} deploy_versioned: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} + + duckdb-wasm-stable-deploy: + name: Deploy Wasm extension binaries + needs: duckdb-stable-build + uses: ./.github/workflows/_extension_deploy_wasm.yml + secrets: inherit + with: + duckdb_version: v0.9.2 + extension_name: quack + deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} + deploy_versioned: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} diff --git a/.github/workflows/_extension_deploy_wasm.yml b/.github/workflows/_extension_deploy_wasm.yml new file mode 100644 index 0000000..b33352a --- /dev/null +++ b/.github/workflows/_extension_deploy_wasm.yml @@ -0,0 +1,122 @@ +# +# Reusable workflow that deploys the artifacts produced by github.com/duckdb/duckdb/.github/workflows/_extension_distribution.yml +# +# note: this workflow needs to be located in the extension repository, as it requires secrets to be passed to the +# deploy script. However, it should generally not be necessary to modify this workflow in your extension repository, as +# this workflow can be configured to use a custom deploy script. + + +name: Extension Deployment Wasm +on: + workflow_call: + inputs: + # The name of the extension + extension_name: + required: true + type: string + # DuckDB version to build against + duckdb_version: + required: true + type: string + # ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64' + exclude_archs: + required: false + type: string + default: "" + # Whether to upload this deployment as the latest. This may overwrite a previous deployment. + deploy_latest: + required: false + type: boolean + default: false + # Whether to upload this deployment under a versioned path. These will not be deleted automatically + deploy_versioned: + required: false + type: boolean + default: false + # Postfix added to artifact names. Can be used to guarantee unique names when this workflow is called multiple times + artifact_postfix: + required: false + type: string + default: "" + # Override the default deploy script with a custom script + deploy_script: + required: false + type: string + default: "./scripts/extension-upload-wasm.sh" + # Override the default matrix parse script with a custom script + matrix_parse_script: + required: false + type: string + default: "./duckdb/scripts/modify_distribution_matrix.py" + +jobs: + generate_matrix: + name: Generate matrix + runs-on: ubuntu-latest + outputs: + deploy_matrix: ${{ steps.parse-matrices.outputs.deploy_matrix }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'true' + + - name: Checkout DuckDB to version + run: | + cd duckdb + git checkout ${{ inputs.duckdb_version }} + + - id: parse-matrices + run: | + echo '{"wasm":{"include":[{"duckdb_arch":"wasm_mvp","vcpkg_triplet":"wasm32-emscripten"},{"duckdb_arch":"wasm_eh","vcpkg_triplet":"wasm32-emscripten"},{"duckdb_arch":"wasm_threads","vcpkg_triplet":"wasm32-emscripten"}]}}' > wasm_matrix_input.json + python3 ${{ inputs.matrix_parse_script }} --input wasm_matrix_input.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + deploy_matrix="`cat deploy_matrix.json`" + echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT + echo `cat $GITHUB_OUTPUT` + + deploy: + name: Deploy + runs-on: ubuntu-latest + needs: generate_matrix + if: ${{ needs.generate_matrix.outputs.deploy_matrix != '{}' && needs.generate_matrix.outputs.deploy_matrix != '' }} + strategy: + matrix: ${{fromJson(needs.generate_matrix.outputs.deploy_matrix)}} + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'true' + + - name: Checkout DuckDB to version + run: | + cd duckdb + git checkout ${{ inputs.duckdb_version }} + + - uses: actions/download-artifact@v2 + with: + name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} + path: | + /tmp/extension + + - name: Deploy + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_ORG_DEPLOY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_ORG_DEPLOY_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.S3_DUCKDB_ORG_REGION }} + BUCKET_NAME: ${{ secrets.S3_DUCKDB_ORG_BUCKET }} + DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.S3_DUCKDB_ORG_EXTENSION_SIGNING_PK }} + run: | + pwd + python3 -m pip install pip awscli + git config --global --add safe.directory '*' + cd duckdb + git fetch --tags + export DUCKDB_VERSION=`git tag --points-at HEAD` + export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} + cd .. + git fetch --tags + export EXT_VERSION=`git tag --points-at HEAD` + export EXT_VERSION=${EXT_VERSION:=`git log -1 --format=%h`} + ${{ inputs.deploy_script }} ${{ inputs.extension_name }} $EXT_VERSION $DUCKDB_VERSION ${{ matrix.duckdb_arch }} $BUCKET_NAME ${{inputs.deploy_latest || 'true' && 'false'}} ${{inputs.deploy_versioned || 'true' && 'false'}} diff --git a/scripts/extension-upload-wasm.sh b/scripts/extension-upload-wasm.sh new file mode 100755 index 0000000..f392397 --- /dev/null +++ b/scripts/extension-upload-wasm.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# Extension upload script + +# Usage: ./extension-upload-wasm.sh +# : Name of the extension +# : Version (commit / version tag) of the extension +# : Version (commit / version tag) of DuckDB +# : Architecture target of the extension binary +# : S3 bucket to upload to +# : Set this as the latest version ("true" / "false", default: "false") +# : Set this as a versioned version that will prevent its deletion + +set -e + +f="/tmp/extension/$1.duckdb_extension.wasm" + +script_dir="$(dirname "$(readlink -f "$0")")" + +# calculate SHA256 hash of extension binary +cat $f > $f.append +# 0 for custom section +# 113 in hex = 275 in decimal, total lenght of what follows (1 + 16 + 2 + 256) +# [1(continuation) + 0010011(payload) = \x93, 0(continuation) + 10(payload) = \x02] +echo -n -e '\x00' >> $f.append +echo -n -e '\x93\x02' >> $f.append +# 10 in hex = 16 in decimal, lenght of name, 1 byte +echo -n -e '\x10' >> $f.append +echo -n -e 'duckdb_signature' >> $f.append +# the name of the WebAssembly custom section, 16 bytes +# 100 in hex, 256 in decimal +# [1(continuation) + 0000000(payload) = ff, 0(continuation) + 10(payload)], +# for a grand total of 2 bytes +echo -n -e '\x80\x02' >> $f.append + +# (Optionally) Sign binary +if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then + echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem + $script_dir/../duckdb/scripts/compute-extension-hash.sh $f.append > $f.hash + openssl pkeyutl -sign -in $f.hash -inkey private.pem -pkeyopt digest:sha256 -out $f.sign + rm -f private.pem +fi + +truncate -s 256 $f.sign + +# append signature to extension binary +cat $f.sign >> $f.append +# compress extension binary +brotli < $f.append > "$f.brotli" +set -e + +# Abort if AWS key is not set +if [ -z "$AWS_ACCESS_KEY_ID" ]; then + echo "No AWS key found, skipping.." + exit 0 +fi + +# upload to latest version +if [[ $6 = 'true' ]]; then + aws s3 cp $ext.gz s3://$5/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" +fi From 7df882e43a57b9cfb065ee6b34fffcf33c7b67f7 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 24 Nov 2023 10:16:04 +0100 Subject: [PATCH 03/10] Bump to main, with the updated workflow --- .github/workflows/MainDistributionPipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index e40f9e3..d129dce 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -14,7 +14,7 @@ concurrency: jobs: duckdb-stable-build: name: Build extension binaries - uses: duckdb/duckdb/.github/workflows/_extension_distribution.yml@v0.9.2 + uses: duckdb/duckdb/.github/workflows/_extension_distribution.yml@6812703823d1d66566bc7eaac2b6e4b273c85333 with: vcpkg_commit: a42af01b72c28a8e1d7b48107b33e4f286a55ef6 duckdb_version: v0.9.2 From b804f8ee2bad315a324e9a91502cf6c4b367950d Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 24 Nov 2023 12:35:34 +0100 Subject: [PATCH 04/10] Properly put extensions in the duckdb-wasm subfolder --- scripts/extension-upload-wasm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/extension-upload-wasm.sh b/scripts/extension-upload-wasm.sh index f392397..2208135 100755 --- a/scripts/extension-upload-wasm.sh +++ b/scripts/extension-upload-wasm.sh @@ -57,5 +57,5 @@ fi # upload to latest version if [[ $6 = 'true' ]]; then - aws s3 cp $ext.gz s3://$5/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" + aws s3 cp $ext.gz s3://$5/duckdb-wasm/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" fi From ad2a2aa49a1b6520a17e01710de1943c1da05d13 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 24 Nov 2023 12:44:11 +0100 Subject: [PATCH 05/10] Make extension-upload's scritps more similar --- scripts/extension-upload-wasm.sh | 31 +++++++++++++++++++------------ scripts/extension-upload.sh | 32 ++++++++++++++++++-------------- 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/scripts/extension-upload-wasm.sh b/scripts/extension-upload-wasm.sh index 2208135..b4de92a 100755 --- a/scripts/extension-upload-wasm.sh +++ b/scripts/extension-upload-wasm.sh @@ -18,35 +18,37 @@ f="/tmp/extension/$1.duckdb_extension.wasm" script_dir="$(dirname "$(readlink -f "$0")")" # calculate SHA256 hash of extension binary -cat $f > $f.append +cat $ext > $ext.append # 0 for custom section # 113 in hex = 275 in decimal, total lenght of what follows (1 + 16 + 2 + 256) # [1(continuation) + 0010011(payload) = \x93, 0(continuation) + 10(payload) = \x02] -echo -n -e '\x00' >> $f.append -echo -n -e '\x93\x02' >> $f.append +echo -n -e '\x00' >> $ext.append +echo -n -e '\x93\x02' >> $ext.append # 10 in hex = 16 in decimal, lenght of name, 1 byte -echo -n -e '\x10' >> $f.append -echo -n -e 'duckdb_signature' >> $f.append +echo -n -e '\x10' >> $ext.append +echo -n -e 'duckdb_signature' >> $ext.append # the name of the WebAssembly custom section, 16 bytes # 100 in hex, 256 in decimal # [1(continuation) + 0000000(payload) = ff, 0(continuation) + 10(payload)], # for a grand total of 2 bytes -echo -n -e '\x80\x02' >> $f.append +echo -n -e '\x80\x02' >> $ext.append # (Optionally) Sign binary if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem - $script_dir/../duckdb/scripts/compute-extension-hash.sh $f.append > $f.hash - openssl pkeyutl -sign -in $f.hash -inkey private.pem -pkeyopt digest:sha256 -out $f.sign + $script_dir/../duckdb/scripts/compute-extension-hash.sh $ext.append > $ext.hash + openssl pkeyutl -sign -in $ext.hash -inkey private.pem -pkeyopt digest:sha256 -out $ext.sign rm -f private.pem fi -truncate -s 256 $f.sign +# Signature is always there, potentially defaulting to 256 zeros +truncate -s 256 $ext.sign # append signature to extension binary -cat $f.sign >> $f.append +cat $ext.sign >> $ext.append + # compress extension binary -brotli < $f.append > "$f.brotli" +brotli < $ext.append > "$ext.brotli" set -e # Abort if AWS key is not set @@ -55,7 +57,12 @@ if [ -z "$AWS_ACCESS_KEY_ID" ]; then exit 0 fi +# upload versioned version +if [[ $7 = 'true' ]]; then + aws s3 cp $ext.gz s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read +fi + # upload to latest version if [[ $6 = 'true' ]]; then - aws s3 cp $ext.gz s3://$5/duckdb-wasm/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" + aws s3 cp $ext.brotli s3://$5/duckdb-wasm/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" fi diff --git a/scripts/extension-upload.sh b/scripts/extension-upload.sh index 3eb3b67..e1c99e0 100755 --- a/scripts/extension-upload.sh +++ b/scripts/extension-upload.sh @@ -2,7 +2,7 @@ # Extension upload script -# Usage: ./extension-upload-oote.sh +# Usage: ./extension-upload.sh # : Name of the extension # : Version (commit / version tag) of the extension # : Version (commit / version tag) of DuckDB @@ -17,24 +17,32 @@ ext="/tmp/extension/$1.duckdb_extension" script_dir="$(dirname "$(readlink -f "$0")")" -# Abort if AWS key is not set -if [ -z "$AWS_ACCESS_KEY_ID" ]; then - echo "No AWS key found, skipping.." - exit 0 -fi +# calculate SHA256 hash of extension binary +cat $ext > $ext.append # (Optionally) Sign binary if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem - $script_dir/../duckdb/scripts/compute-extension-hash.sh $ext > $ext.hash + $script_dir/../duckdb/scripts/compute-extension-hash.sh $ext.append > $ext.hash openssl pkeyutl -sign -in $ext.hash -inkey private.pem -pkeyopt digest:sha256 -out $ext.sign - cat $ext.sign >> $ext + rm -f private.pem fi -set -e +# Signature is always there, potentially defaulting to 256 zeros +truncate -s 256 $ext.sign + +# append signature to extension binary +cat $ext.sign >> $ext.append # compress extension binary -gzip < "${ext}" > "$ext.gz" +gzip < $ext.append > "$ext.gz" +set -e + +# Abort if AWS key is not set +if [ -z "$AWS_ACCESS_KEY_ID" ]; then + echo "No AWS key found, skipping.." + exit 0 +fi # upload versioned version if [[ $7 = 'true' ]]; then @@ -45,7 +53,3 @@ fi if [[ $6 = 'true' ]]; then aws s3 cp $ext.gz s3://$5/$3/$4/$1.duckdb_extension.gz --acl public-read fi - -if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then - rm private.pem -fi \ No newline at end of file From 5acebcc0040059dcfe3d90d069879501e46f955a Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 28 Nov 2023 12:54:47 +0100 Subject: [PATCH 06/10] Fix extension-upload-wasm --- scripts/extension-upload-wasm.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scripts/extension-upload-wasm.sh b/scripts/extension-upload-wasm.sh index b4de92a..c33b4e0 100755 --- a/scripts/extension-upload-wasm.sh +++ b/scripts/extension-upload-wasm.sh @@ -57,11 +57,6 @@ if [ -z "$AWS_ACCESS_KEY_ID" ]; then exit 0 fi -# upload versioned version -if [[ $7 = 'true' ]]; then - aws s3 cp $ext.gz s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read -fi - # upload to latest version if [[ $6 = 'true' ]]; then aws s3 cp $ext.brotli s3://$5/duckdb-wasm/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" From d91d1c4ba91ca2ffa09ebb8ca9e745a33629ed17 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 28 Nov 2023 12:59:21 +0100 Subject: [PATCH 07/10] Fix script --- scripts/bootstrap-template.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/bootstrap-template.py b/scripts/bootstrap-template.py index ef079e2..bae21b8 100755 --- a/scripts/bootstrap-template.py +++ b/scripts/bootstrap-template.py @@ -50,6 +50,7 @@ def replace_everywhere(to_find, to_replace): for path in files_to_search: replace(path, to_find, to_replace) replace(path, to_find.capitalize(), to_camel_case(to_replace)) + replace(path, to_find.upper(), to_replace.upper()) replace("./CMakeLists.txt", to_find, to_replace) replace("./Makefile", to_find, to_replace) From 075d8557f14f0d0d0a6420ed881cbaec15489918 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 28 Nov 2023 15:19:19 +0100 Subject: [PATCH 08/10] Unify scripts --- .github/workflows/_extension_deploy_wasm.yml | 2 +- scripts/extension-upload-wasm.sh | 63 -------------------- scripts/extension-upload.sh | 41 +++++++++++-- 3 files changed, 38 insertions(+), 68 deletions(-) delete mode 100755 scripts/extension-upload-wasm.sh diff --git a/.github/workflows/_extension_deploy_wasm.yml b/.github/workflows/_extension_deploy_wasm.yml index b33352a..f319819 100644 --- a/.github/workflows/_extension_deploy_wasm.yml +++ b/.github/workflows/_extension_deploy_wasm.yml @@ -42,7 +42,7 @@ on: deploy_script: required: false type: string - default: "./scripts/extension-upload-wasm.sh" + default: "./scripts/extension-upload.sh" # Override the default matrix parse script with a custom script matrix_parse_script: required: false diff --git a/scripts/extension-upload-wasm.sh b/scripts/extension-upload-wasm.sh deleted file mode 100755 index c33b4e0..0000000 --- a/scripts/extension-upload-wasm.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -# Extension upload script - -# Usage: ./extension-upload-wasm.sh -# : Name of the extension -# : Version (commit / version tag) of the extension -# : Version (commit / version tag) of DuckDB -# : Architecture target of the extension binary -# : S3 bucket to upload to -# : Set this as the latest version ("true" / "false", default: "false") -# : Set this as a versioned version that will prevent its deletion - -set -e - -f="/tmp/extension/$1.duckdb_extension.wasm" - -script_dir="$(dirname "$(readlink -f "$0")")" - -# calculate SHA256 hash of extension binary -cat $ext > $ext.append -# 0 for custom section -# 113 in hex = 275 in decimal, total lenght of what follows (1 + 16 + 2 + 256) -# [1(continuation) + 0010011(payload) = \x93, 0(continuation) + 10(payload) = \x02] -echo -n -e '\x00' >> $ext.append -echo -n -e '\x93\x02' >> $ext.append -# 10 in hex = 16 in decimal, lenght of name, 1 byte -echo -n -e '\x10' >> $ext.append -echo -n -e 'duckdb_signature' >> $ext.append -# the name of the WebAssembly custom section, 16 bytes -# 100 in hex, 256 in decimal -# [1(continuation) + 0000000(payload) = ff, 0(continuation) + 10(payload)], -# for a grand total of 2 bytes -echo -n -e '\x80\x02' >> $ext.append - -# (Optionally) Sign binary -if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then - echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem - $script_dir/../duckdb/scripts/compute-extension-hash.sh $ext.append > $ext.hash - openssl pkeyutl -sign -in $ext.hash -inkey private.pem -pkeyopt digest:sha256 -out $ext.sign - rm -f private.pem -fi - -# Signature is always there, potentially defaulting to 256 zeros -truncate -s 256 $ext.sign - -# append signature to extension binary -cat $ext.sign >> $ext.append - -# compress extension binary -brotli < $ext.append > "$ext.brotli" -set -e - -# Abort if AWS key is not set -if [ -z "$AWS_ACCESS_KEY_ID" ]; then - echo "No AWS key found, skipping.." - exit 0 -fi - -# upload to latest version -if [[ $6 = 'true' ]]; then - aws s3 cp $ext.brotli s3://$5/duckdb-wasm/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" -fi diff --git a/scripts/extension-upload.sh b/scripts/extension-upload.sh index e1c99e0..9a787ed 100755 --- a/scripts/extension-upload.sh +++ b/scripts/extension-upload.sh @@ -13,13 +13,33 @@ set -e -ext="/tmp/extension/$1.duckdb_extension" +if [[ $ext == "wasm_*" ]]; then + ext="/tmp/extension/$1.duckdb_extension.wasm" +else + ext="/tmp/extension/$1.duckdb_extension" +fi script_dir="$(dirname "$(readlink -f "$0")")" # calculate SHA256 hash of extension binary cat $ext > $ext.append +if [[ $4 == "wasm_*" ]]; then + # 0 for custom section + # 113 in hex = 275 in decimal, total lenght of what follows (1 + 16 + 2 + 256) + # [1(continuation) + 0010011(payload) = \x93, 0(continuation) + 10(payload) = \x02] + echo -n -e '\x00' >> $ext.append + echo -n -e '\x93\x02' >> $ext.append + # 10 in hex = 16 in decimal, lenght of name, 1 byte + echo -n -e '\x10' >> $ext.append + echo -n -e 'duckdb_signature' >> $ext.append + # the name of the WebAssembly custom section, 16 bytes + # 100 in hex, 256 in decimal + # [1(continuation) + 0000000(payload) = ff, 0(continuation) + 10(payload)], + # for a grand total of 2 bytes + echo -n -e '\x80\x02' >> $ext.append +fi + # (Optionally) Sign binary if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem @@ -35,7 +55,12 @@ truncate -s 256 $ext.sign cat $ext.sign >> $ext.append # compress extension binary -gzip < $ext.append > "$ext.gz" +if [[ $4 == "wasm_*" ]]; then + gzip < $ext.append > "$ext.compressed" +else + brotli < $ext.append > "$ext.compressed" +fi + set -e # Abort if AWS key is not set @@ -46,10 +71,18 @@ fi # upload versioned version if [[ $7 = 'true' ]]; then - aws s3 cp $ext.gz s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read + if [[ $4 == "wasm_*" ]]; then + aws s3 cp $ext.compressed s3://$5/duckdb-wasm/$1/$2/duckdb-wasm/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" + else + aws s3 cp $ext.compressed s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read + fi fi # upload to latest version if [[ $6 = 'true' ]]; then - aws s3 cp $ext.gz s3://$5/$3/$4/$1.duckdb_extension.gz --acl public-read + if [[ $4 == "wasm_*" ]]; then + aws s3 cp $ext.compressed s3://$5/duckdb-wasm/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" + else + aws s3 cp $ext.compressed s3://$5/$3/$4/$1.duckdb_extension.gz --acl public-read + fi fi From 1e093bbf48c7154f6af01ec9a85b1022c95a1b86 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 28 Nov 2023 16:10:40 +0100 Subject: [PATCH 09/10] Unify yml files --- .../workflows/MainDistributionPipeline.yml | 11 -- .github/workflows/_extension_deploy.yml | 8 +- .github/workflows/_extension_deploy_wasm.yml | 122 ------------------ 3 files changed, 5 insertions(+), 136 deletions(-) delete mode 100644 .github/workflows/_extension_deploy_wasm.yml diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index d129dce..1dcf763 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -31,14 +31,3 @@ jobs: extension_name: quack deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} deploy_versioned: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} - - duckdb-wasm-stable-deploy: - name: Deploy Wasm extension binaries - needs: duckdb-stable-build - uses: ./.github/workflows/_extension_deploy_wasm.yml - secrets: inherit - with: - duckdb_version: v0.9.2 - extension_name: quack - deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} - deploy_versioned: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} diff --git a/.github/workflows/_extension_deploy.yml b/.github/workflows/_extension_deploy.yml index 5d6b020..53493dc 100644 --- a/.github/workflows/_extension_deploy.yml +++ b/.github/workflows/_extension_deploy.yml @@ -68,7 +68,9 @@ jobs: - id: parse-matrices run: | - python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + cat ./duckdb/.github/config/distribution_matrix.json > distribution_matrix.json + grep wasm distribution_matrix.json || (head -n -1 ./duckdb/.github/config/distribution_matrix.json > distribution_matrix.json && echo ',"wasm":{"include":[{"duckdb_arch":"wasm_mvp","vcpkg_triplet":"wasm32-emscripten"},{"duckdb_arch":"wasm_eh","vcpkg_triplet":"wasm32-emscripten"},{"duckdb_arch":"wasm_threads","vcpkg_triplet":"wasm32-emscripten"}]}}' >> distribution_matrix.json) + python3 ${{ inputs.matrix_parse_script }} --input distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty deploy_matrix="`cat deploy_matrix.json`" echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT echo `cat $GITHUB_OUTPUT` @@ -94,7 +96,7 @@ jobs: - uses: actions/download-artifact@v2 with: - name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} + name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}${{startsWith(matrix.duckdb, 'wasm') && '.wasm' || ''}} path: | /tmp/extension @@ -118,4 +120,4 @@ jobs: git fetch --tags export EXT_VERSION=`git tag --points-at HEAD` export EXT_VERSION=${EXT_VERSION:=`git log -1 --format=%h`} - ${{ inputs.deploy_script }} ${{ inputs.extension_name }} $EXT_VERSION $DUCKDB_VERSION ${{ matrix.duckdb_arch }} $BUCKET_NAME ${{inputs.deploy_latest || 'true' && 'false'}} ${{inputs.deploy_versioned || 'true' && 'false'}} \ No newline at end of file + ${{ inputs.deploy_script }} ${{ inputs.extension_name }} $EXT_VERSION $DUCKDB_VERSION ${{ matrix.duckdb_arch }} $BUCKET_NAME ${{inputs.deploy_latest || 'true' && 'false'}} ${{inputs.deploy_versioned || 'true' && 'false'}} diff --git a/.github/workflows/_extension_deploy_wasm.yml b/.github/workflows/_extension_deploy_wasm.yml deleted file mode 100644 index f319819..0000000 --- a/.github/workflows/_extension_deploy_wasm.yml +++ /dev/null @@ -1,122 +0,0 @@ -# -# Reusable workflow that deploys the artifacts produced by github.com/duckdb/duckdb/.github/workflows/_extension_distribution.yml -# -# note: this workflow needs to be located in the extension repository, as it requires secrets to be passed to the -# deploy script. However, it should generally not be necessary to modify this workflow in your extension repository, as -# this workflow can be configured to use a custom deploy script. - - -name: Extension Deployment Wasm -on: - workflow_call: - inputs: - # The name of the extension - extension_name: - required: true - type: string - # DuckDB version to build against - duckdb_version: - required: true - type: string - # ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64' - exclude_archs: - required: false - type: string - default: "" - # Whether to upload this deployment as the latest. This may overwrite a previous deployment. - deploy_latest: - required: false - type: boolean - default: false - # Whether to upload this deployment under a versioned path. These will not be deleted automatically - deploy_versioned: - required: false - type: boolean - default: false - # Postfix added to artifact names. Can be used to guarantee unique names when this workflow is called multiple times - artifact_postfix: - required: false - type: string - default: "" - # Override the default deploy script with a custom script - deploy_script: - required: false - type: string - default: "./scripts/extension-upload.sh" - # Override the default matrix parse script with a custom script - matrix_parse_script: - required: false - type: string - default: "./duckdb/scripts/modify_distribution_matrix.py" - -jobs: - generate_matrix: - name: Generate matrix - runs-on: ubuntu-latest - outputs: - deploy_matrix: ${{ steps.parse-matrices.outputs.deploy_matrix }} - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Checkout DuckDB to version - run: | - cd duckdb - git checkout ${{ inputs.duckdb_version }} - - - id: parse-matrices - run: | - echo '{"wasm":{"include":[{"duckdb_arch":"wasm_mvp","vcpkg_triplet":"wasm32-emscripten"},{"duckdb_arch":"wasm_eh","vcpkg_triplet":"wasm32-emscripten"},{"duckdb_arch":"wasm_threads","vcpkg_triplet":"wasm32-emscripten"}]}}' > wasm_matrix_input.json - python3 ${{ inputs.matrix_parse_script }} --input wasm_matrix_input.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty - deploy_matrix="`cat deploy_matrix.json`" - echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT - echo `cat $GITHUB_OUTPUT` - - deploy: - name: Deploy - runs-on: ubuntu-latest - needs: generate_matrix - if: ${{ needs.generate_matrix.outputs.deploy_matrix != '{}' && needs.generate_matrix.outputs.deploy_matrix != '' }} - strategy: - matrix: ${{fromJson(needs.generate_matrix.outputs.deploy_matrix)}} - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Checkout DuckDB to version - run: | - cd duckdb - git checkout ${{ inputs.duckdb_version }} - - - uses: actions/download-artifact@v2 - with: - name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} - path: | - /tmp/extension - - - name: Deploy - shell: bash - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_ORG_DEPLOY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_ORG_DEPLOY_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.S3_DUCKDB_ORG_REGION }} - BUCKET_NAME: ${{ secrets.S3_DUCKDB_ORG_BUCKET }} - DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.S3_DUCKDB_ORG_EXTENSION_SIGNING_PK }} - run: | - pwd - python3 -m pip install pip awscli - git config --global --add safe.directory '*' - cd duckdb - git fetch --tags - export DUCKDB_VERSION=`git tag --points-at HEAD` - export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} - cd .. - git fetch --tags - export EXT_VERSION=`git tag --points-at HEAD` - export EXT_VERSION=${EXT_VERSION:=`git log -1 --format=%h`} - ${{ inputs.deploy_script }} ${{ inputs.extension_name }} $EXT_VERSION $DUCKDB_VERSION ${{ matrix.duckdb_arch }} $BUCKET_NAME ${{inputs.deploy_latest || 'true' && 'false'}} ${{inputs.deploy_versioned || 'true' && 'false'}} From a24f7a9fd882c11f98f978baf5f9580aa188ad29 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 29 Nov 2023 11:30:23 +0100 Subject: [PATCH 10/10] Fix deploy --- .github/workflows/MainDistributionPipeline.yml | 1 - scripts/extension-upload.sh | 12 +++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 1dcf763..19e81d9 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -19,7 +19,6 @@ jobs: vcpkg_commit: a42af01b72c28a8e1d7b48107b33e4f286a55ef6 duckdb_version: v0.9.2 extension_name: quack - exclude_archs: "" duckdb-stable-deploy: name: Deploy extension binaries diff --git a/scripts/extension-upload.sh b/scripts/extension-upload.sh index 9a787ed..32097f2 100755 --- a/scripts/extension-upload.sh +++ b/scripts/extension-upload.sh @@ -13,18 +13,20 @@ set -e -if [[ $ext == "wasm_*" ]]; then +if [[ $4 == wasm* ]]; then ext="/tmp/extension/$1.duckdb_extension.wasm" else ext="/tmp/extension/$1.duckdb_extension" fi +echo $ext + script_dir="$(dirname "$(readlink -f "$0")")" # calculate SHA256 hash of extension binary cat $ext > $ext.append -if [[ $4 == "wasm_*" ]]; then +if [[ $4 == wasm* ]]; then # 0 for custom section # 113 in hex = 275 in decimal, total lenght of what follows (1 + 16 + 2 + 256) # [1(continuation) + 0010011(payload) = \x93, 0(continuation) + 10(payload) = \x02] @@ -55,7 +57,7 @@ truncate -s 256 $ext.sign cat $ext.sign >> $ext.append # compress extension binary -if [[ $4 == "wasm_*" ]]; then +if [[ $4 == wasm_* ]]; then gzip < $ext.append > "$ext.compressed" else brotli < $ext.append > "$ext.compressed" @@ -71,7 +73,7 @@ fi # upload versioned version if [[ $7 = 'true' ]]; then - if [[ $4 == "wasm_*" ]]; then + if [[ $4 == wasm* ]]; then aws s3 cp $ext.compressed s3://$5/duckdb-wasm/$1/$2/duckdb-wasm/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" else aws s3 cp $ext.compressed s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read @@ -80,7 +82,7 @@ fi # upload to latest version if [[ $6 = 'true' ]]; then - if [[ $4 == "wasm_*" ]]; then + if [[ $4 == wasm* ]]; then aws s3 cp $ext.compressed s3://$5/duckdb-wasm/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" else aws s3 cp $ext.compressed s3://$5/$3/$4/$1.duckdb_extension.gz --acl public-read