Attach Sui binaries to a release #466
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Attach Sui binaries to a release | |
| run-name: Attach Sui binaries to a ${{ inputs.sui_tag }} release | |
| on: | |
| release: | |
| types: [ created, prereleased, released, published ] | |
| workflow_dispatch: | |
| inputs: | |
| sui_tag: | |
| description: 'Sui repo tag to build from' | |
| type: string | |
| required: true | |
| concurrency: ${{ github.workflow }}-${{ inputs.sui_tag || github.ref }} | |
| env: | |
| TAG_NAME: "${{ github.event.inputs.sui_tag || github.ref }}" | |
| BINARY_LIST_FILE: "./binary-build-list.json" | |
| CARGO_TERM_COLOR: always | |
| # Disable incremental compilation. | |
| # | |
| # Incremental compilation is useful as part of an edit-build-test-edit cycle, | |
| # as it lets the compiler avoid recompiling code that hasn't changed. However, | |
| # on CI, we're not making small edits; we're almost always building the entire | |
| # project from scratch. Thus, incremental compilation on CI actually | |
| # introduces *additional* overhead to support making future builds | |
| # faster...but no future builds will ever occur in any given CI environment. | |
| # | |
| # See https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow | |
| # for details. | |
| CARGO_INCREMENTAL: 0 | |
| # Allow more retries for network requests in cargo (downloading crates) and | |
| # rustup (installing toolchains). This should help to reduce flaky CI failures | |
| # from transient network timeouts or other issues. | |
| CARGO_NET_RETRY: 10 | |
| RUSTUP_MAX_RETRIES: 10 | |
| # Don't emit giant backtraces in the CI logs. | |
| RUST_BACKTRACE: short | |
| TMP_BUILD_DIR: './tmp/release' | |
| jobs: | |
| release-build: | |
| name: Build & Publish Binaries | |
| timeout-minutes: 120 | |
| strategy: | |
| matrix: | |
| os: | |
| [ | |
| ubuntu-ghcloud, # ubuntu-x86_64 | |
| ubuntu-arm64, # ubuntu-arm64 | |
| windows-ghcloud, # windows-x86_64 | |
| macos-latest-xl, # macos-x86_64 | |
| macos-latest-xlarge # macos-arm64 | |
| ] | |
| fail-fast: false | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Clean up and validate ${{ env.TAG_NAME }} tag name | |
| shell: bash | |
| run: | | |
| export sui_tag=$(echo ${{ env.TAG_NAME }} | sed s/'refs\/tags\/'//) | |
| [[ "${sui_tag}" == "main" ]] && echo "tag cannot be equals to 'main'" && exit 1 | |
| echo "sui_tag=${sui_tag}" >> $GITHUB_ENV | |
| export sui_version=$(echo ${sui_tag} | sed -e 's/mainnet-v//' -e 's/testnet-v//') | |
| echo "sui_version=${sui_version}" >> $GITHUB_ENV | |
| - name: Check out ${{ env.sui_tag }} | |
| uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # pin@v3 | |
| with: | |
| ref: ${{ env.sui_tag }} | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # pin v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-west-2 | |
| - name: Set os/arch variables (Windows) | |
| if: ${{ matrix.os == 'windows-ghcloud' }} | |
| shell: bash | |
| run: | | |
| export arch=$(uname -m) | |
| export os_type="windows-${arch}" | |
| echo "os_type=${os_type}" >> $GITHUB_ENV | |
| echo "extention=$(echo ".exe")" >> $GITHUB_ENV | |
| - name: Set os/arch variables | |
| if: ${{ matrix.os != 'windows-ghcloud' }} | |
| shell: bash | |
| run: | | |
| export arch=$(uname -m) | |
| export system_os=$(echo ${{ matrix.os }} | cut -d- -f1) | |
| export os_type="${system_os}-${arch}" | |
| echo "os_type=${system_os}-${arch}" >> $GITHUB_ENV | |
| - name: Check if archive have already been published | |
| continue-on-error: true | |
| shell: bash | |
| run: | | |
| echo "s3_archive_exist=$(curl -Is https://sui-releases.s3.us-east-1.amazonaws.com/releases/sui-${{ env.sui_tag }}-${{ env.os_type }}.tgz | head -n 1 | grep '200 OK')" >> $GITHUB_ENV | |
| - name: Download archive, if it exists | |
| if: ${{ env.s3_archive_exist != '' }} | |
| shell: bash | |
| run: | | |
| mkdir -p ${{ env.TMP_BUILD_DIR }} | |
| aws s3 cp s3://sui-releases/releases/sui-${{ env.sui_tag }}-${os_type}.tgz ./tmp/sui-${{ env.sui_tag }}-${os_type}.tgz | |
| tar -xf ./tmp/sui-${{ env.sui_tag }}-${os_type}.tgz -C ${{ env.TMP_BUILD_DIR }} | |
| - name: Setup caching | |
| if: ${{ env.s3_archive_exist == '' }} | |
| uses: bmwill/rust-cache@v1 # Fork of 'Swatinem/rust-cache' which allows caching additional paths | |
| - name: Install nexttest (Windows) | |
| if: ${{ matrix.os == 'windows-ghcloud' && env.s3_archive_exist == '' }} | |
| uses: taiki-e/install-action@33022ba120c3f523d134bbbee12278fc11a3df1a # pin@nextest | |
| - name: Install postgres (Windows) | |
| if: ${{ matrix.os == 'windows-ghcloud' && env.s3_archive_exist == '' }} | |
| shell: bash | |
| run: | | |
| choco install postgresql12 --force --params '/Password:root' | |
| echo "C:\Program Files\PostgreSQL\12\bin" >> $GITHUB_PATH | |
| echo "C:\Program Files\PostgreSQL\12\lib" >> $GITHUB_PATH | |
| echo "PQ_LIB_DIR=C:\Program Files\PostgreSQL\12\lib" >> $GITHUB_ENV | |
| echo "PG_DATABASE_URL=postgres://postgres:root@localhost/" >> $GITHUB_ENV | |
| echo "PG_EXAMPLE_DATABASE_URL=postgres://postgres:root@localhost/diesel_example" >> $GITHUB_ENV | |
| - name: Install postgres (MacOS arm64) | |
| if: ${{ matrix.os == 'macos-latest-xlarge' && env.gcloud_archive_exist == '' }} | |
| shell: bash | |
| env: | |
| PQ_LIB_DIR: "$(brew --prefix libpq)/lib" | |
| LIBRARY_PATH: "/opt/homebrew/lib:$LIBRARY_PATH" | |
| PKG_CONFIG_PATH: "/opt/homebrew/lib/pkgconfig:$PKG_CONFIG_PATH" | |
| PATH: "/opt/homebrew/bin:$PATH" | |
| run: | | |
| brew install postgresql | |
| - name: Install postgres (Ubuntu arm64) | |
| if: ${{ matrix.os == 'ubuntu-arm64' && env.s3_archive_exist == '' }} | |
| shell: bash | |
| run: | | |
| sudo apt install libpq-dev | |
| - name: Remove unused apps (MacOS arm64) | |
| if: ${{ matrix.os == 'macos-latest-xlarge' && env.gcloud_archive_exist == '' }} | |
| continue-on-error: true | |
| shell: bash | |
| run: | | |
| # MacOS arm64 runner only has 14GB avaialble, which is too small for our builds, so removing unused softwared. | |
| df -h / | |
| sudo rm -rf /Applications/Xcode*.app | |
| sudo rm -rf ~/Library/Developer/Xcode/DerivedData | |
| sudo rm -rf ~/Library/Developer/CoreSimulator/Caches/* | |
| sudo rm -rf ~/Library/Developer/Xcode/iOS\ DeviceSupport/* | |
| df -h / | |
| - name: Cargo build for ${{ matrix.os }} platform | |
| if: ${{ env.s3_archive_exist == '' }} | |
| shell: bash | |
| run: | | |
| [ -f ~/.cargo/env ] && source ~/.cargo/env ; cargo build --bin sui --release --features indexer && mv target/release/sui target/release/sui-pg | |
| [ -f ~/.cargo/env ] && source ~/.cargo/env ; cargo build --release && cargo build --profile=dev --bin sui | |
| - name: Rename binaries for ${{ matrix.os }} | |
| if: ${{ env.s3_archive_exist == '' }} | |
| shell: bash | |
| run: | | |
| mkdir -p ${{ env.TMP_BUILD_DIR }} | |
| [ ! -f ${{ env.BINARY_LIST_FILE }} ] && echo "${{ env.BINARY_LIST_FILE }} cannot be found" && exit 1 | |
| for binary in $(cat ${{ env.BINARY_LIST_FILE }} | jq -r '.release_binaries[]'); do | |
| export binary=$(echo ${binary} | tr -d $'\r') | |
| mv ./target/release/${binary}${{ env.extention }} ${{ env.TMP_BUILD_DIR }}/${binary}${{ env.extention }} | |
| done | |
| # sui-pg is a special binary that is built with the indexer feature for sui start cmd | |
| export binary=$(echo "sui-pg" | tr -d $'\r') | |
| mv ./target/release/${binary}${{ env.extention }} ${{ env.TMP_BUILD_DIR }}/${binary}${{ env.extention }} | |
| mv ./target/debug/sui${{ env.extention }} ${{ env.TMP_BUILD_DIR }}/sui-debug${{ env.extention }} | |
| tar -cvzf ./tmp/sui-${{ env.sui_tag }}-${{ env.os_type }}.tgz -C ${{ env.TMP_BUILD_DIR }} . | |
| [[ ${{ env.sui_tag }} == *"testnet"* ]] && aws s3 cp ./tmp/sui-${{ env.sui_tag }}-${{ env.os_type }}.tgz s3://sui-releases/releases/sui-${{ env.sui_tag }}-${{ env.os_type }}.tgz || true | |
| - name: Publish Windows sui binary to Chocolatey | |
| if: ${{ matrix.os == 'windows-ghcloud' && contains(env.sui_tag, 'testnet') }} | |
| shell: bash | |
| run: | | |
| choco install checksum | |
| export sui_sha=$(checksum -t sha256 ${{ env.TMP_BUILD_DIR }}/sui.exe) | |
| cd chocolatey | |
| cat <<EOF >>VERIFICATION.txt | |
| Sui Binary verification steps | |
| 1. Download https://github.com/MystenLabs/sui/releases/download/${{ env.sui_tag }}/sui-${{ env.sui_tag }}-windows-x86_64.tgz | |
| 2. Extract sui.exe | |
| 3. Verify binary: checksum.exe -t sha256 sui.exe: ${sui_sha} | |
| File 'LICENSE.txt' is obtained from: https://github.com/MystenLabs/sui/blob/main/LICENSE | |
| EOF | |
| choco pack --version ${{ env.sui_version }} configuration=release | |
| choco apikey --api-key ${{ secrets.CHOCO_API_KEY }} --source https://push.chocolatey.org/ | |
| choco push sui.${{ env.sui_version }}.nupkg --source https://push.chocolatey.org/ | |
| - name: Upload release artifacts for ${{ matrix.os }} platform | |
| uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # pin@v3 | |
| with: | |
| name: sui-binaries-${{ matrix.os }} | |
| if-no-files-found: error | |
| path: | | |
| ./tmp/sui-${{ env.sui_tag }}-${{ env.os_type }}.tgz | |
| - name: Attach artifacts to ${{ env.sui_tag }} release in GH | |
| uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # pin@v1 | |
| with: | |
| tag_name: ${{ env.sui_tag }} | |
| files: | | |
| ./tmp/sui-${{ env.sui_tag }}-${{ env.os_type }}.tgz | |
| update-homebrew-formula: | |
| name: Run brew bump-formula-pr for sui on testnet releases | |
| needs: release-build | |
| runs-on: ubuntu-latest | |
| # releasing sui cli on testnet releases because it lags `main` less than mainnet, but is more likely to be stable than devnet | |
| if: ${{ contains( inputs.sui_tag, 'testnet') || contains( github.ref, 'testnet') }} | |
| steps: | |
| - name: Clean up tag name ${{ env.TAG_NAME }} | |
| shell: bash | |
| run: | | |
| echo "sui_tag=$(echo ${{ env.TAG_NAME }} | sed s/'refs\/tags\/'//)" >> $GITHUB_ENV | |
| echo "versionless_tag=$(echo ${{ env.TAG_NAME }} | sed s/'refs\/tags\/'// | sed s/'testnet\-v'//)" >> $GITHUB_ENV | |
| - uses: mislav/bump-homebrew-formula-action@b3327118b2153c82da63fd9cbf58942146ee99f0 # pin@v3 | |
| with: | |
| formula-name: sui | |
| create-pullrequest: true | |
| tag-name: "${{ env.sui_tag }}" | |
| commit-message: | | |
| {{formulaName}} ${{ env.versionless_tag }} | |
| Created by https://github.com/mislav/bump-homebrew-formula-action | |
| From release: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| env: | |
| # https://github.com/settings/tokens/new?scopes=public_repo,workflow | |
| COMMITTER_TOKEN: ${{ secrets.HOMEBREW_GH_FORMULA_BUMP }} | |
| update-homebrew-tap: | |
| name: Update mysten-tap sui.rb file | |
| needs: release-build | |
| runs-on: ubuntu-latest | |
| if: ${{ contains( inputs.sui_tag, 'testnet') || contains( github.ref, 'testnet') }} | |
| steps: | |
| - name: Clean up tag name ${{ env.TAG_NAME }} | |
| shell: bash | |
| run: | | |
| echo "sui_tag=$(echo ${{ env.TAG_NAME }} | sed s/'refs\/tags\/'//)" >> $GITHUB_ENV | |
| # Checkout MystenLabs/homebrew-tap | |
| - name: Checkout Target Repository | |
| uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # pin@v3 | |
| with: | |
| repository: MystenLabs/homebrew-tap | |
| # @john's PAT, needs to be rotated jan 5 2025 | |
| token: ${{ secrets.HOMEBREW_TAP_REPO_READ_WRITE }} | |
| ref: main | |
| fetch-depth: 0 | |
| # Download all artifacts from the previous job | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2 | |
| - name: Fetch Ubuntu x86_64 Binary and Compute SHA256 | |
| shell: bash | |
| run: | | |
| echo "sha256_ubuntu_release=$(sha256sum sui-binaries-ubuntu-ghcloud/sui-${{ env.sui_tag }}-ubuntu-x86_64.tgz | awk '{print $1}')" >> $GITHUB_ENV | |
| echo "sha256_macos_x86_release=$(sha256sum sui-binaries-macos-latest-xl/sui-${{ env.sui_tag }}-macos-x86_64.tgz | awk '{print $1}')" >> $GITHUB_ENV | |
| echo "sha256_macos_arm_release=$(sha256sum sui-binaries-macos-latest-xlarge/sui-${{ env.sui_tag }}-macos-arm64.tgz | awk '{print $1}' )" >> $GITHUB_ENV | |
| # Install Jinja2 for templating | |
| - name: Install Jinja2 | |
| run: pip install jinja2 | |
| - name: Apply Jinja2 Template and Update Formula | |
| run: | | |
| python3 - <<EOF | |
| import jinja2 | |
| import os | |
| # Load the template | |
| template_loader = jinja2.FileSystemLoader(searchpath="./template") | |
| template_env = jinja2.Environment(loader=template_loader) | |
| template = template_env.get_template("sui.rb.j2") | |
| version = "${{ env.sui_tag }}".removeprefix("testnet-v") | |
| # Render the template with variables | |
| output = template.render( | |
| version=version, | |
| linux_sha256="${sha256_ubuntu_release}", | |
| macos_intel_sha256="${sha256_macos_x86_release}", | |
| macos_arm_sha256="${sha256_macos_arm_release}" | |
| ) | |
| # Write the output to the formula file | |
| with open("Formula/sui.rb", "w") as file: | |
| file.write(output) | |
| EOF | |
| # Commit and Push to Target Repository | |
| - name: Commit and Push to Target Repository | |
| uses: github-actions-x/commit@722d56b8968bf00ced78407bbe2ead81062d8baa # [email protected] | |
| with: | |
| github-token: ${{ secrets.HOMEBREW_TAP_REPO_READ_WRITE }} | |
| commit-message: "Update Homebrew formula for release ${{ env.sui_tag }}" | |
| files: Formula/sui.rb | |
| push-branch: main | |
| rebase: 'true' | |
| force-add: 'true' | |
| name: ${{ github.triggering_actor }} | |
| email: ${{ github.triggering_actor }}@users.noreply.github.com | |
| # Tag all sui images with release tag, so that they can be easily found | |
| tag-docker-hub-images: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Dispatch Tagging of images in DockerHub, in MystenLabs/sui-operations | |
| uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # [email protected] | |
| with: | |
| repository: MystenLabs/sui-operations | |
| token: ${{ secrets.DOCKER_BINARY_BUILDS_DISPATCH }} | |
| event-type: tag-docker-images | |
| client-payload: '{"sui_commit": "${{ github.sha }}", "repo_name": "all", "tag": "${{ env.TAG_NAME }}"}' |