benchmark.yaml

name: Run Benchmarks

permissions:
  id-token: write
  contents: write
  deployments: write

on:
  workflow_dispatch:
  push:
    branches:
      - main

jobs:
  build_ios_app:
    runs-on: macos-latest
    strategy:
      matrix:
        engine: [hermes, jsc]
    steps:
      - name: check Xcode version
        run: /usr/bin/xcodebuild -version

      - name: checkout repository
        uses: actions/checkout@v4

      - name: Use Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 22

      - name: Install the Apple certificate and provisioning profile
        env:
          BUILD_CERTIFICATE_BASE64: ${{ secrets.BUILD_CERTIFICATE_BASE64 }}
          P12_PASSWORD: ${{ secrets.P12_PASSWORD }}
          BUILD_PROVISION_PROFILE_BASE64: ${{ secrets.BUILD_PROVISION_PROFILE_BASE64 }}
          KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
        run: |
          # create variables
          CERTIFICATE_PATH=$RUNNER_TEMP/build_certificate.p12
          PP_PATH=$RUNNER_TEMP/build_pp.mobileprovision
          KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db

          # import certificate and provisioning profile from secrets
          echo -n "$BUILD_CERTIFICATE_BASE64" | base64 --decode -o $CERTIFICATE_PATH
          echo -n "$BUILD_PROVISION_PROFILE_BASE64" | base64 --decode -o $PP_PATH

          # create temporary keychain
          security create-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH
          security set-keychain-settings -lut 21600 $KEYCHAIN_PATH
          security unlock-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH

          # import certificate to keychain
          security import $CERTIFICATE_PATH -P "$P12_PASSWORD" -A -t cert -f pkcs12 -k $KEYCHAIN_PATH
          security list-keychain -d user -s $KEYCHAIN_PATH

          # apply provisioning profile
          mkdir -p ~/Library/MobileDevice/Provisioning\ Profiles
          cp $PP_PATH ~/Library/MobileDevice/Provisioning\ Profiles

      - name: build javascript bundle
        run: |
          cd apps/benchmarking-test-app
          npm run install:${{ matrix.engine }}
          npm run build:ios:release

      - uses: actions/cache/restore@v4
        id: cache-xcarchive
        with:
          path: ./apps/benchmarking-test-app/BenchmarkingTestApp.xcarchive
          key: ${{ runner.os }}-${{ matrix.engine }}-${{ hashFiles('**/Podfile.lock') }}

      - name: build archive
        if: steps.cache-xcarchive.outputs.cache-hit != 'true'
        run: |
          cd apps/benchmarking-test-app
          npm run install:ios:${{ matrix.engine }}

          xcodebuild -scheme "benchmarking-test-app" \
          -archivePath "BenchmarkingTestApp.xcarchive" \
          -workspace ios/benchmarking-test-app.xcworkspace \
          -configuration Release \
          -sdk iphoneos \
          -destination generic/platform=iOS \
          -allowProvisioningUpdates \
          DEVELOPMENT_TEAM=${{ secrets.DEVELOPMENT_TEAM_ID }} \
          clean archive

          cp dist/main.ios.jsbundle BenchmarkingTestApp.xcarchive/Products/Applications/ReactTestApp.app

      - uses: actions/cache/save@v4
        with:
          path: ./apps/benchmarking-test-app/BenchmarkingTestApp.xcarchive
          key: ${{ steps.cache-xcarchive.outputs.cache-primary-key }}

      - name: export ipa
        run: |
          cd apps/benchmarking-test-app
          ls
          xcodebuild -exportArchive -archivePath BenchmarkingTestApp.xcarchive -exportOptionsPlist BenchmarkingTestApp.xcarchive/Info.plist -exportPath ${{ runner.temp }}/build
          mv ${{ runner.temp }}/build/ReactTestApp.ipa ${{ runner.temp }}/build/${{matrix.engine}}Benchmarking.ipa

      - name: Upload application
        uses: actions/upload-artifact@v4
        with:
          name: ${{matrix.engine}}App
          path: ${{ runner.temp }}/build/${{matrix.engine}}Benchmarking.ipa
          retention-days: 1

      - name: Upload bundle
        uses: actions/upload-artifact@v4
        with:
          name: ${{matrix.engine}}Bundle
          path: apps/benchmarking-test-app/dist/main.ios.jsbundle
          retention-days: 1

  benchmark:
    runs-on: ubuntu-latest
    needs: build_ios_app
    strategy:
      matrix:
        engine: [hermes, jsc]
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Use Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 22

      - name: Install dependencies
        run: npm ci

      - name: Create Appium Test Zip
        run: npm run bundle -w benchmarking-scripts

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v2
        with:
          role-to-assume: ${{ secrets.DEVICE_FARM_IAM_ARN }}
          aws-region: us-west-2

      - name: Download ${{matrix.engine}} artifact
        uses: actions/download-artifact@v4
        with:
          name: ${{matrix.engine}}App
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Run Benchmark with JavaScriptCore Engine
        id: run-benchmark
        uses: aws-actions/aws-devicefarm-mobile-device-testing@v2.3
        with:
          run-settings-json: |
            {
              "name": "GitHubAction-${{ github.workflow }}_${{ github.run_id }}_${{ github.run_attempt }}",
              "projectArn": "TestApp",
              "appArn": "${{matrix.engine}}Benchmarking.ipa",
              "devicePoolArn": "SmallDevicePool",
              "executionConfiguration": {
                "jobTimeoutMinutes": 20
              },
              "test": {
                "type": "APPIUM_NODE",
                "testPackageArn": "packages/benchmarking/MyTests.zip",
                "testSpecArn": "packages/benchmarking/testSpec.yaml"
              }
            }
          artifact-types: ALL

      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: ${{matrix.engine}}OutputFiles
          path: ${{ steps.run-benchmark.outputs.artifact-folder }}

  parse-benchmarks:
    runs-on: ubuntu-latest
    needs: benchmark
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Use Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 22

      - name: Install dependencies
        run: npm ci

      - name: Build benchmark parser
        run: npm run build -w benchmark-parser

      - name: Download JSC Artifacts
        uses: actions/download-artifact@v4
        with:
          name: "jscOutputFiles"
          path: "jscOutput"
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Download Hermes Artifacts
        uses: actions/download-artifact@v4
        with:
          name: "hermesOutputFiles"
          path: "hermesOutput"
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Extract benchmark results from artifacts
        run: |
          ./scripts/unzipArchive.sh jscOutput benchmarks/jsc
          ./scripts/unzipArchive.sh hermesOutput benchmarks/hermes

      - name: Download hermes bundle
        uses: actions/download-artifact@v4
        with:
          name: hermesBundle
          path: bundleDist
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Parse Benchmarks
        run: npm run parse-benchmarks

      - name: Report Benchmarks
        run: cat benchmark.json

      - name: Store benchmark result
        uses: benchmark-action/github-action-benchmark@v1
        with:
          name: "React Native Benchmarks"
          tool: "customSmallerIsBetter"
          output-file-path: benchmark.json
          github-token: ${{ secrets.GITHUB_TOKEN }}
          auto-push: true