Benchmark of pallet-contracts and pallet-evm #58
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark of pallet-contracts and pallet-evm | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' | |
| workflow_dispatch: | |
| env: | |
| MOONBEAM_ARTIFACT: moonbeam | |
| MOONBEAM_DIR: moonbeam_release | |
| MOONBEAM_BIN: moonbeam_release/*/target/release/moonbeam | |
| MOONBEAM_VERSION: version | |
| BENCHMARK_DIR: stats | |
| TEST_PARAMS: --instance-count 1 --call-count 2000 | |
| BENCHMARK_URI: https://raw.githubusercontent.com/paritytech/smart-bench/gh-pages | |
| jobs: | |
| build_dev_moonbeam: | |
| runs-on: ubuntu-20.04 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Clean worker | |
| run: | | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| - uses: actions/checkout@v4 | |
| with: | |
| sparse-checkout: | | |
| launch/moonbeam.patch | |
| sparse-checkout-cone-mode: false | |
| - name: Download Moonbeam Release | |
| run: | | |
| API_URL="https://api.github.com/repos/moonbeam-foundation/moonbeam/releases/131808906" | |
| RESPONSE=$(curl -s "${API_URL}") | |
| # Remove control characters causing error while parsing | |
| RESPONSE=$(echo $RESPONSE | tr -cd '[:print:]') | |
| DOWNLOAD_URL=$(echo $RESPONSE | jq -r '.tarball_url') | |
| RELEASE_TAG=$(echo $RESPONSE | jq -r '.tag_name') | |
| mkdir ${{ env.MOONBEAM_DIR }} | |
| echo "Download moonbeam release: ${DOWNLOAD_URL}" | |
| curl -L ${DOWNLOAD_URL} | tar -xzv -C ${{ env.MOONBEAM_DIR }} | |
| echo $RELEASE_TAG > ${{ env.MOONBEAM_DIR }}/${{ env.MOONBEAM_VERSION }} | |
| - name: Patch | |
| run: | | |
| # Add Dev RPC support | |
| cd moonbeam_release/*/ | |
| patch -p1 < ../../launch/moonbeam.patch | |
| - name: Build | |
| uses: docker://paritytech/ci-unified:bullseye-1.73.0 | |
| with: | |
| args: /bin/bash -c "cd moonbeam_release/*/ && cargo build --release" | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.MOONBEAM_ARTIFACT }} | |
| path: | | |
| ./${{ env.MOONBEAM_BIN }} | |
| ./${{ env.MOONBEAM_DIR }}/${{ env.MOONBEAM_VERSION }} | |
| retention-days: 1 | |
| smart_contract_benchmark: | |
| strategy: | |
| matrix: | |
| type: [ink-wasm, sol-wasm, evm] | |
| contract: [erc20, flipper, triangle-number, storage-read, storage-write] | |
| env: | |
| BENCHMARK_FILE: benchmark_${{ matrix.type }}_${{ matrix.contract }}.csv | |
| needs: build_dev_moonbeam | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Cache | |
| uses: Swatinem/rust-cache@v2 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ env.MOONBEAM_ARTIFACT }} | |
| path: ./${{ env.MOONBEAM_DIR }} | |
| - name: Set Moonbeam Release | |
| id: moonbeam_release | |
| run: | | |
| mkdir bin | |
| cp ${{ env.MOONBEAM_BIN }} bin/moonbeam | |
| chmod +x bin/moonbeam | |
| RELEASE_TAG=$(cat ${{ env.MOONBEAM_DIR }}/${{ env.MOONBEAM_VERSION }}) | |
| echo "tag=$(echo ${RELEASE_TAG})" >> $GITHUB_OUTPUT | |
| - name: Download Polkadot-Parachain Release | |
| id: polkadot_parachain_release | |
| run: | | |
| API_URL="https://api.github.com/repos/paritytech/cumulus/releases/latest" | |
| RESPONSE=$(curl -s "${API_URL}") | |
| # Get the download URL of the release binary from Repo | |
| DOWNLOAD_URL=$(echo $RESPONSE | jq -r '.assets | map(select(.name == "polkadot-parachain")) | .[0].browser_download_url') | |
| RELEASE_TAG=$(echo $RESPONSE | jq -r '.tag_name') | |
| curl -L -o bin/polkadot-parachain ${DOWNLOAD_URL} && chmod +x bin/polkadot-parachain | |
| echo "tag=$(echo ${RELEASE_TAG})" >> $GITHUB_OUTPUT | |
| - name: Get Ink contract language verision | |
| id: ink_version | |
| uses: ./.github/actions/get-contract-language | |
| with: | |
| contracts-directory: './contracts/ink' | |
| - name: Get Solang contract language verision | |
| id: solang_version | |
| uses: ./.github/actions/get-contract-language | |
| with: | |
| contracts-directory: './contracts/solidity/wasm' | |
| - name: Get Solc contract language verision | |
| id: solc_version | |
| run: | | |
| language="" | |
| # Iterate over each contract in the directory | |
| for contract in ./contracts/solidity/evm/contracts/*/*.dbg.json; do | |
| build_info=$(jq -r '.buildInfo' "$contract") | |
| build_info_path="./contracts/solidity/evm/contracts/${build_info#../}" | |
| solc_version=$(jq -r '.solcVersion' "$build_info_path") | |
| current_language="solc $solc_version" | |
| # Check if the current language is different from the previous one | |
| if [ -n "$language" ] && [ "$current_language" != "$language" ]; then | |
| echo "Error: Different language detected in contract $contract: $language != $current_language" | |
| exit 1 | |
| fi | |
| language="$current_language" | |
| echo "Contract: $contract, Language: $current_language" | |
| done | |
| echo "language=$language" >> $GITHUB_OUTPUT | |
| - name: Display variables | |
| run: | | |
| echo moonbeam_tag: ${{ steps.moonbeam_release.outputs.tag }} | |
| echo polkadot_parachain_tag: ${{ steps.polkadot_parachain_release.outputs.tag }} | |
| - name: Execute tests | |
| id: run_smart_bench | |
| run: | | |
| cd launch | |
| ./download-bins.sh | |
| # overwrite parachains | |
| cp -f ../bin/* ./bin | |
| ./build.sh | |
| STATS=$(./run.sh -- ${{ matrix.type }} ${{ matrix.contract }} ${TEST_PARAMS}) | |
| BLOCKS=$(echo ${STATS} | grep -o 'Total Blocks: [0-9]*' | awk '{print $3}') | |
| EXTRINSICS=$(echo ${STATS} | grep -o 'Total Extrinsics: [0-9]*' | awk '{print $3}') | |
| TPS=$(echo ${STATS} | grep -o 'sTPS: [0-9]*' | awk '{print $2}') | |
| echo "Blocks: ${BLOCKS}" | |
| echo "Extrinsics: ${EXTRINSICS}" | |
| echo "TPS: ${TPS}" | |
| echo "tps=$(echo ${TPS})" >> $GITHUB_OUTPUT | |
| - name: Extract Ink benchmark stats | |
| if: matrix.type == 'ink-wasm' | |
| run: | | |
| CURRENT_DATE=$(date +"%s") | |
| # date, parachain tag, contract, contract language, TPS | |
| echo "${CURRENT_DATE}, ${{ matrix.type }}, ${{ steps.polkadot_parachain_release.outputs.tag }}, \ | |
| ${{ matrix.contract }}, ${{ steps.run_smart_bench.outputs.tps }}, ${{ steps.ink_version.outputs.language }}, \ | |
| ${{ env.TEST_PARAMS }}" > ${BENCHMARK_DIR}/${BENCHMARK_FILE} | |
| - name: Extract Solang benchmark stats | |
| if: matrix.type == 'sol-wasm' | |
| run: | | |
| CURRENT_DATE=$(date +"%s") | |
| # date, parachain tag, contract, contract language, TPS | |
| echo "${CURRENT_DATE}, ${{ matrix.type }}, ${{ steps.polkadot_parachain_release.outputs.tag }}, \ | |
| ${{ matrix.contract }}, ${{ steps.run_smart_bench.outputs.tps }}, ${{ steps.solang_version.outputs.language }}, \ | |
| ${{ env.TEST_PARAMS }}" > ${BENCHMARK_DIR}/${BENCHMARK_FILE} | |
| - name: Extract Solidity benchmark stats | |
| if: matrix.type == 'evm' | |
| run: | | |
| CURRENT_DATE=$(date +"%s") | |
| # date, parachain tag, contract, contract language, TPS | |
| echo "${CURRENT_DATE}, ${{ matrix.type }}, ${{ steps.moonbeam_release.outputs.tag }}, \ | |
| ${{ matrix.contract }}, ${{ steps.run_smart_bench.outputs.tps }}, ${{ steps.solc_version.outputs.language }}, \ | |
| ${{ env.TEST_PARAMS }}" > ${BENCHMARK_DIR}/${BENCHMARK_FILE} | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.BENCHMARK_FILE }} | |
| path: ${{ env.BENCHMARK_DIR }}/${{ env.BENCHMARK_FILE }} | |
| retention-days: 1 | |
| collect: | |
| runs-on: ubuntu-latest | |
| needs: [smart_contract_benchmark] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Download artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: ${{ env.BENCHMARK_DIR }} | |
| - name: Merge CSV | |
| run: | | |
| for file in ${{ env.BENCHMARK_DIR }}/*/*.csv; do | |
| # Extract contract name | |
| contract_name=$(basename "$file" | sed 's/^.*_\(.*\)\.csv$/\1/') | |
| benchmark_file=bench_${contract_name}.csv | |
| if [ ! -f ${{ env.BENCHMARK_DIR }}/${benchmark_file} ]; then | |
| curl -L -o ${{ env.BENCHMARK_DIR }}/${benchmark_file} ${{ env.BENCHMARK_URI }}/${benchmark_file} || exit 1 | |
| fi | |
| cat $file >> ${{ env.BENCHMARK_DIR }}/${benchmark_file} | |
| done | |
| - name: Generate graph | |
| run: | | |
| for file in ${{ env.BENCHMARK_DIR }}/bench_*.csv; do | |
| contract_name=$(basename "$file" | sed 's/^.*_\(.*\)\.csv$/\1/') | |
| pushd stats | |
| ./get_graph.sh --panel-id=2 --csv-data=../${file} --output=../${{ env.BENCHMARK_DIR }}/stps_${contract_name}.png | |
| popd | |
| done | |
| - name: Commit benchmark stats | |
| run: | | |
| CURRENT_DATE=$(date +"%Y%m%d") | |
| # Set git config | |
| git config user.email "[email protected]" | |
| git config user.name "paritytech-ci" | |
| git fetch origin gh-pages | |
| # saving stats | |
| mkdir /tmp/stats | |
| mv ${{ env.BENCHMARK_DIR }}/bench_*.csv /tmp/stats | |
| mv ${{ env.BENCHMARK_DIR }}/stps_*.png /tmp/stats | |
| git checkout gh-pages | |
| mv /tmp/stats/* . | |
| # Upload files | |
| git add *.csv *.png --force | |
| git status | |
| git commit -m "Updated stats in ${CURRENT_DATE} and pushed to gh-pages" | |
| git push origin gh-pages --force | |
| rm -rf .git/ ./* |