From dfcb80d78f761234c8eb7eccef0b66d1e534e3c6 Mon Sep 17 00:00:00 2001 From: Marcin Pastecki <33489038+mpastecki@users.noreply.github.com> Date: Thu, 24 Oct 2024 16:08:40 +0200 Subject: [PATCH] Added deterministic wasm build (#1746) * Added deterministic wasm build * Updated ci workflow dependencies * We need compressed wasm --- .circleci/config.yml | 164 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 158 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 202aa273e..bee19a3d6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -84,9 +84,10 @@ jobs: - run: name: Copy binary to assets command: cp ./target/release/polymesh ./assets/polymesh - - run: - name: Copy wasm to assets - command: cp ./target/release/wbuild/polymesh*/*wasm ./assets/ + ## This has been commented out due to deterministic wasm build step added + # - run: + # name: Copy wasm to assets + # command: cp ./target/release/wbuild/polymesh*/*wasm ./assets/ - persist_to_workspace: root: ./assets paths: @@ -169,6 +170,105 @@ jobs: paths: - polymesh-benchmarks - save-sccache-cache + + deterministic-wasm-build: + parameters: + chain: + type: enum + enum: + - "testnet" + - "mainnet" + - "develop" + default: "mainnet" + description: "Name of the chain, ie. mainnet" + package: + type: string + default: "wasm" + description: > + Runtime package to build, ie. polkadot-runtime. + If your runtime follows this pattern, you don't have and should not provide this input. + + If not provided, it will be set to -runtime + image: + type: string + default: "paritytech/srtool" + description: > + You can use an alternate image, use with caution! + tag: + type: string + default: "latest" + description: "Tag of the srtool image to use. Omit to use the latest (recommended)" + workdir: + type: string + default: "." + description: "Path of the project, this is where your main Cargo.toml is located." + runtime_dir: + type: string + default: "runtime/testnet" + description: "Location of the runtime in your repo. The default is 'runtime/'" + profile: + type: string + default: "release" + description: "Which profile to use with cargo build." + build_opts: + type: string + default: " " + description: "Additional options to pass to cargo build." + parachain_pallet_id: + type: string + default: "0x01" + description: "The patachain's pallet id." + authorize_upgrade_prefix: + type: string + default: "0x02" + description: "The authorize_upgrade call ID." + + docker: + - image: <>:<> + resource_class: xlarge + working_directory: /build + environment: + # https://github.com/paritytech/polkadot-sdk/pull/2217 + WASM_BUILD_STD: "0" + + PACKAGE: <> + RUNTIME_DIR: <> + BUILD_OPTS: <> + PARACHAIN_PALLET_ID: <> + AUTHORIZE_UPGRADE_PREFIX: <> + PROFILE: <> + steps: + - checkout + - attach_workspace: + at: ./assets + - run: + name: Use rust-toolchain.toml + command: cp rust-toolchain.toml pallets/runtime/<>/rust-toolchain.toml + - run: + name: Build the deterministic wasm binary + command: | + CMD="/srtool/build --app --json -cM" + stdbuf -oL $CMD | { + while IFS= read -r line + do + echo ║ $line + JSON="$line" + done + + echo "=========================================" + echo $JSON | jq . + echo "=========================================" + echo $JSON | jq . > ./assets/<>-srtool-digest.json + echo "prop: $(echo $JSON | jq -r .runtimes.compressed.prop)" + cp $(echo $JSON | jq -r .runtimes.compressed.wasm) ./assets/polymesh_runtime_<>.compact.compressed.wasm + echo "=========================================" + } + - persist_to_workspace: + root: ./assets + paths: + - . + + benchmark: docker: - image: debian:stable @@ -473,11 +573,27 @@ jobs: go install github.com/tcnksm/ghr@latest cd assets VERSION=$(./polymesh --version | cut -d " " -f2) + + # polymesh x86_64 mv ./polymesh ./polymesh-${VERSION} + sha256sum ./polymesh-${VERSION} > ./polymesh-${VERSION}.sha256 + zip -rm ./polymesh-${VERSION}.zip ./polymesh-${VERSION} ./polymesh-${VERSION}.sha256 + zip -T ./polymesh-${VERSION}.zip + + # polymesh arm64 mv ./polymesh-arm64 ./polymesh-${VERSION}-arm64 - zip -rm ./polymesh-${VERSION}.zip ./polymesh-${VERSION} - zip -rm ./polymesh-${VERSION}-linux-arm64.zip ./polymesh-${VERSION}-arm64 - zip -rm ./polymesh-${VERSION}-wasm.zip ./*.wasm + sha256sum ./polymesh-${VERSION}-arm64 > ./polymesh-${VERSION}-arm64.sha256 + zip -rm ./polymesh-${VERSION}-linux-arm64.zip ./polymesh-${VERSION}-arm64 ./polymesh-${VERSION}-arm64.sha256 + zip -T ./polymesh-${VERSION}-linux-arm64.zip + + # polymesh wasm + for file in *.wasm; do sha256sum "$file" > "${file}.sha256"; done + zip -rm ./polymesh-${VERSION}-wasm.zip ./*.wasm ./*.wasm.sha256 ./*-srtool-digest.json + zip -T ./polymesh-${VERSION}-wasm.zip + + # checksum file + sha256sum ./polymesh-${VERSION}.zip ./polymesh-${VERSION}-linux-arm64.zip ./polymesh-${VERSION}-wasm.zip | sed 's/^/sha256: /' > ./polymesh-${VERSION}.CHECKSUMS + sha512sum ./polymesh-${VERSION}.zip ./polymesh-${VERSION}-linux-arm64.zip ./polymesh-${VERSION}-wasm.zip | sed 's/^/sha512: /' >> ./polymesh-${VERSION}.CHECKSUMS ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -prerelease -delete v${VERSION} ./ workflows: @@ -559,9 +675,45 @@ workflows: requires: - build - build-arm64 + - deterministic-wasm-build-mainnet + - deterministic-wasm-build-testnet + filters: + branches: + only: + - mainnet + - testnet + - staging + - deterministic-wasm-build: + name: deterministic-wasm-build-testnet filters: branches: only: - mainnet - testnet - staging + chain: testnet + package: polymesh-runtime-testnet + runtime_dir: pallets/runtime/testnet + tag: 1.81.0 + - deterministic-wasm-build: + name: deterministic-wasm-build-mainnet + filters: + branches: + only: + - mainnet + - testnet + - staging + chain: mainnet + package: polymesh-runtime-mainnet + runtime_dir: pallets/runtime/mainnet + tag: 1.81.0 + - deterministic-wasm-build: + name: deterministic-wasm-build-develop + filters: + branches: + only: + - develop + chain: develop + package: polymesh-runtime-develop + runtime_dir: pallets/runtime/develop + tag: 1.81.0