test(bench): Playing with flamegraph generation [wip] #11
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| push: | |
| branches: [develop] | |
| jobs: | |
| bench: | |
| name: Benchmark the code | |
| runs-on: warp-ubuntu-latest-x64-16x | |
| strategy: | |
| matrix: | |
| toolchain: | |
| - stable | |
| env: | |
| PR_NUMBER: ${{ github.event.number }} | |
| BENCH_AGAINST_BASE: 1 | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Used to get hash of base branch | |
| # https://github.com/dtolnay/rust-toolchain | |
| - name: Setup rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: ${{ matrix.toolchain }} | |
| # https://github.com/swatinem/rust-cache | |
| - name: Run Swatinem/rust-cache@v2 | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| cache-on-failure: true | |
| - name: Set (and display) useful variables | |
| id: vars | |
| run: | | |
| source scripts/ci/env-vars.sh | |
| echo "HEAD_SHA: ${HEAD_SHA}" | |
| echo "HEAD_SHA_SHORT: ${HEAD_SHA_SHORT}" | |
| echo "BASE_SHA: ${BASE_SHA}" | |
| echo "BASE_SHA_SHORT: ${BASE_SHA_SHORT}" | |
| echo "HEAD_SHA=${HEAD_SHA}" >> "$GITHUB_OUTPUT" | |
| echo "HEAD_SHA_SHORT=${HEAD_SHA_SHORT}" >> "$GITHUB_OUTPUT" | |
| echo "BASE_SHA=${BASE_SHA}" >> "$GITHUB_OUTPUT" | |
| echo "BASE_SHA_SHORT=${BASE_SHA_SHORT}" >> "$GITHUB_OUTPUT" | |
| # S3 upload directory, depending if it includes a comparison or not | |
| if [ "$HEAD_SHA" == "$BASE_SHA" ]; then | |
| # No comparison (i.e. running on a branch like develop directly) | |
| S3_UPLOAD_DIR="benchmark/${HEAD_SHA_SHORT}" | |
| else | |
| # Comparison (i.e. running on a PR) | |
| S3_UPLOAD_DIR="benchmark/${HEAD_SHA_SHORT}-${BASE_SHA_SHORT}" | |
| fi | |
| echo "S3_UPLOAD_DIR: ${S3_UPLOAD_DIR}" | |
| echo "S3_UPLOAD_DIR=${S3_UPLOAD_DIR}" >> "$GITHUB_OUTPUT" | |
| echo "S3_UPLOAD_DIR=${S3_UPLOAD_DIR}" >> "$GITHUB_ENV" | |
| # | |
| # RUN BENCHMARKS (and upload the report) | |
| # | |
| - run: make bench-in-ci | |
| # Upload as artifact first | |
| - name: Upload report as artifact | |
| uses: actions/[email protected] | |
| with: | |
| name: benchmark-report | |
| path: target/benchmark-in-ci/benchmark-report/ | |
| - name: Zip the report and add to folder (for S3 upload) | |
| working-directory: target/benchmark-in-ci | |
| run: | | |
| zip_fn="report.zip" | |
| zip -r $zip_fn benchmark-report | |
| mv $zip_fn benchmark-report/ | |
| # Upload S3 (using https://github.com/shallwefootball/upload-s3-action) | |
| - name: Upload report to S3 | |
| uses: shallwefootball/s3-upload-action@master | |
| id: S3 | |
| with: | |
| aws_key_id: ${{secrets.AWS_KEY_ID}} | |
| aws_secret_access_key: ${{secrets.AWS_SECRET_ACCESS_KEY}} | |
| aws_bucket: flashbots-rbuilder-ci-stats | |
| source_dir: target/benchmark-in-ci/benchmark-report | |
| destination_dir: ${{ steps.vars.outputs.S3_UPLOAD_DIR }} | |
| # | |
| # POST SUMMARY (to PR comment and CI job summary) | |
| # | |
| - name: Add summary to CI job summary | |
| run: | | |
| BENCH_URL="https://flashbots-rbuilder-ci-stats.s3.us-east-2.amazonaws.com/${{steps.S3.outputs.object_key}}/report/index.html" | |
| sed -i "s|__BENCH_URL__|${BENCH_URL}|" target/benchmark-in-ci/benchmark-summary.md | |
| sed -i "s|__BENCH_URL__|${BENCH_URL}|" target/benchmark-in-ci/benchmark-pr-comment.md | |
| cat target/benchmark-in-ci/benchmark-summary.md >> $GITHUB_STEP_SUMMARY | |
| # https://github.com/peter-evans/find-comment | |
| - name: Find previous PR comment | |
| uses: peter-evans/find-comment@v3 | |
| if: github.event_name == 'pull_request' | |
| id: fc | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| comment-author: 'github-actions[bot]' | |
| body-includes: Benchmark results | |
| # https://github.com/peter-evans/create-or-update-comment | |
| - name: Create or update PR comment | |
| uses: peter-evans/create-or-update-comment@v4 | |
| if: github.event_name == 'pull_request' | |
| with: | |
| comment-id: ${{ steps.fc.outputs.comment-id }} | |
| issue-number: ${{ github.event.pull_request.number }} | |
| edit-mode: replace | |
| body-path: target/benchmark-in-ci/benchmark-pr-comment.md |