diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index c757460..19e81d9 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -14,7 +14,7 @@ concurrency: jobs: duckdb-stable-build: name: Build extension binaries - uses: duckdb/duckdb/.github/workflows/_extension_distribution.yml@v0.9.2 + uses: duckdb/duckdb/.github/workflows/_extension_distribution.yml@6812703823d1d66566bc7eaac2b6e4b273c85333 with: vcpkg_commit: a42af01b72c28a8e1d7b48107b33e4f286a55ef6 duckdb_version: v0.9.2 diff --git a/.github/workflows/_extension_deploy.yml b/.github/workflows/_extension_deploy.yml index 5d6b020..53493dc 100644 --- a/.github/workflows/_extension_deploy.yml +++ b/.github/workflows/_extension_deploy.yml @@ -68,7 +68,9 @@ jobs: - id: parse-matrices run: | - python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + cat ./duckdb/.github/config/distribution_matrix.json > distribution_matrix.json + grep wasm distribution_matrix.json || (head -n -1 ./duckdb/.github/config/distribution_matrix.json > distribution_matrix.json && echo ',"wasm":{"include":[{"duckdb_arch":"wasm_mvp","vcpkg_triplet":"wasm32-emscripten"},{"duckdb_arch":"wasm_eh","vcpkg_triplet":"wasm32-emscripten"},{"duckdb_arch":"wasm_threads","vcpkg_triplet":"wasm32-emscripten"}]}}' >> distribution_matrix.json) + python3 ${{ inputs.matrix_parse_script }} --input distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty deploy_matrix="`cat deploy_matrix.json`" echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT echo `cat $GITHUB_OUTPUT` @@ -94,7 +96,7 @@ jobs: - uses: actions/download-artifact@v2 with: - name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} + name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}${{startsWith(matrix.duckdb, 'wasm') && '.wasm' || ''}} path: | /tmp/extension @@ -118,4 +120,4 @@ jobs: git fetch --tags export EXT_VERSION=`git tag --points-at HEAD` export EXT_VERSION=${EXT_VERSION:=`git log -1 --format=%h`} - ${{ inputs.deploy_script }} ${{ inputs.extension_name }} $EXT_VERSION $DUCKDB_VERSION ${{ matrix.duckdb_arch }} $BUCKET_NAME ${{inputs.deploy_latest || 'true' && 'false'}} ${{inputs.deploy_versioned || 'true' && 'false'}} \ No newline at end of file + ${{ inputs.deploy_script }} ${{ inputs.extension_name }} $EXT_VERSION $DUCKDB_VERSION ${{ matrix.duckdb_arch }} $BUCKET_NAME ${{inputs.deploy_latest || 'true' && 'false'}} ${{inputs.deploy_versioned || 'true' && 'false'}} diff --git a/Makefile b/Makefile index 44ffe48..637f634 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all clean format debug release duckdb_debug duckdb_release pull update +.PHONY: all clean format debug release duckdb_debug duckdb_release pull update wasm_mvp wasm_eh wasm_threads all: release @@ -31,6 +31,8 @@ ifeq ($(GEN),ninja) GENERATOR=-G "Ninja" -DFORCE_COLORED_OUTPUT=1 endif +EXT_NAME=quack + #### Configuration for this extension EXTENSION_NAME=QUACK EXTENSION_FLAGS=\ @@ -109,3 +111,23 @@ clean: rm -rf testext cd duckdb && make clean cd duckdb && make clean-python + +WASM_LINK_TIME_FLAGS= + +wasm_mvp: + mkdir -p build/wasm_mvp + emcmake cmake $(GENERATOR) -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_mvp -DCMAKE_CXX_FLAGS="-DDUCKDB_CUSTOM_PLATFORM=wasm_mvp" -DSKIP_EXTENSIONS="parquet" -S duckdb $(TOOLCHAIN_FLAGS) $(EXTENSION_FLAGS) -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake + emmake make -j8 -Cbuild/wasm_mvp + cd build/wasm_mvp/extension/${EXT_NAME} && emcc $f -sSIDE_MODULE=1 -o ../../${EXT_NAME}.duckdb_extension.wasm -O3 ${EXT_NAME}.duckdb_extension $(WASM_LINK_TIME_FLAGS) + +wasm_eh: + mkdir -p build/wasm_eh + emcmake cmake $(GENERATOR) -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_eh -DCMAKE_CXX_FLAGS="-fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1 -DDUCKDB_CUSTOM_PLATFORM=wasm_eh" -DSKIP_EXTENSIONS="parquet" -S duckdb $(TOOLCHAIN_FLAGS) $(EXTENSION_FLAGS) -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake + emmake make -j8 -Cbuild/wasm_eh + cd build/wasm_eh/extension/${EXT_NAME} && emcc $f -sSIDE_MODULE=1 -o ../../${EXT_NAME}.duckdb_extension.wasm -O3 ${EXT_NAME}.duckdb_extension $(WASM_LINK_TIME_FLAGS) + +wasm_threads: + mkdir -p ./build/wasm_threads + emcmake cmake $(GENERATOR) -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_threads -DCMAKE_CXX_FLAGS="-fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1 -DWITH_WASM_THREADS=1 -DWITH_WASM_SIMD=1 -DWITH_WASM_BULK_MEMORY=1 -DDUCKDB_CUSTOM_PLATFORM=wasm_threads" -DSKIP_EXTENSIONS="parquet" -S duckdb $(TOOLCHAIN_FLAGS) $(EXTENSION_FLAGS) -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake + emmake make -j8 -Cbuild/wasm_threads + cd build/wasm_threads/extension/${EXT_NAME} && emcc $f -sSIDE_MODULE=1 -o ../../${EXT_NAME}.duckdb_extension.wasm -O3 ${EXT_NAME}.duckdb_extension $(WASM_LINK_TIME_FLAGS) diff --git a/scripts/bootstrap-template.py b/scripts/bootstrap-template.py index ef079e2..bae21b8 100755 --- a/scripts/bootstrap-template.py +++ b/scripts/bootstrap-template.py @@ -50,6 +50,7 @@ def replace_everywhere(to_find, to_replace): for path in files_to_search: replace(path, to_find, to_replace) replace(path, to_find.capitalize(), to_camel_case(to_replace)) + replace(path, to_find.upper(), to_replace.upper()) replace("./CMakeLists.txt", to_find, to_replace) replace("./Makefile", to_find, to_replace) diff --git a/scripts/extension-upload.sh b/scripts/extension-upload.sh index 3eb3b67..32097f2 100755 --- a/scripts/extension-upload.sh +++ b/scripts/extension-upload.sh @@ -2,7 +2,7 @@ # Extension upload script -# Usage: ./extension-upload-oote.sh +# Usage: ./extension-upload.sh # : Name of the extension # : Version (commit / version tag) of the extension # : Version (commit / version tag) of DuckDB @@ -13,39 +13,78 @@ set -e -ext="/tmp/extension/$1.duckdb_extension" +if [[ $4 == wasm* ]]; then + ext="/tmp/extension/$1.duckdb_extension.wasm" +else + ext="/tmp/extension/$1.duckdb_extension" +fi + +echo $ext script_dir="$(dirname "$(readlink -f "$0")")" -# Abort if AWS key is not set -if [ -z "$AWS_ACCESS_KEY_ID" ]; then - echo "No AWS key found, skipping.." - exit 0 +# calculate SHA256 hash of extension binary +cat $ext > $ext.append + +if [[ $4 == wasm* ]]; then + # 0 for custom section + # 113 in hex = 275 in decimal, total lenght of what follows (1 + 16 + 2 + 256) + # [1(continuation) + 0010011(payload) = \x93, 0(continuation) + 10(payload) = \x02] + echo -n -e '\x00' >> $ext.append + echo -n -e '\x93\x02' >> $ext.append + # 10 in hex = 16 in decimal, lenght of name, 1 byte + echo -n -e '\x10' >> $ext.append + echo -n -e 'duckdb_signature' >> $ext.append + # the name of the WebAssembly custom section, 16 bytes + # 100 in hex, 256 in decimal + # [1(continuation) + 0000000(payload) = ff, 0(continuation) + 10(payload)], + # for a grand total of 2 bytes + echo -n -e '\x80\x02' >> $ext.append fi # (Optionally) Sign binary if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem - $script_dir/../duckdb/scripts/compute-extension-hash.sh $ext > $ext.hash + $script_dir/../duckdb/scripts/compute-extension-hash.sh $ext.append > $ext.hash openssl pkeyutl -sign -in $ext.hash -inkey private.pem -pkeyopt digest:sha256 -out $ext.sign - cat $ext.sign >> $ext + rm -f private.pem fi -set -e +# Signature is always there, potentially defaulting to 256 zeros +truncate -s 256 $ext.sign + +# append signature to extension binary +cat $ext.sign >> $ext.append # compress extension binary -gzip < "${ext}" > "$ext.gz" +if [[ $4 == wasm_* ]]; then + gzip < $ext.append > "$ext.compressed" +else + brotli < $ext.append > "$ext.compressed" +fi + +set -e + +# Abort if AWS key is not set +if [ -z "$AWS_ACCESS_KEY_ID" ]; then + echo "No AWS key found, skipping.." + exit 0 +fi # upload versioned version if [[ $7 = 'true' ]]; then - aws s3 cp $ext.gz s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read + if [[ $4 == wasm* ]]; then + aws s3 cp $ext.compressed s3://$5/duckdb-wasm/$1/$2/duckdb-wasm/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" + else + aws s3 cp $ext.compressed s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read + fi fi # upload to latest version if [[ $6 = 'true' ]]; then - aws s3 cp $ext.gz s3://$5/$3/$4/$1.duckdb_extension.gz --acl public-read + if [[ $4 == wasm* ]]; then + aws s3 cp $ext.compressed s3://$5/duckdb-wasm/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" + else + aws s3 cp $ext.compressed s3://$5/$3/$4/$1.duckdb_extension.gz --acl public-read + fi fi - -if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then - rm private.pem -fi \ No newline at end of file